feat(db): use wa-sqlite
This commit is contained in:
parent
c9aaf2ecef
commit
a2bc4115a2
14 changed files with 232 additions and 177 deletions
|
@ -8,7 +8,8 @@
|
|||
"dev": "vite",
|
||||
"build": "vite build",
|
||||
"serve": "vite preview",
|
||||
"prepare": "husky"
|
||||
"prepare": "husky",
|
||||
"postinstall": "cp ./node_modules/@subframe7536/sqlite-wasm/dist/*.wasm ./src/assets/"
|
||||
},
|
||||
"license": "MIT",
|
||||
"devDependencies": {
|
||||
|
@ -40,6 +41,7 @@
|
|||
"@solidjs/meta": "^0.29.4",
|
||||
"@solidjs/router": "^0.15.3",
|
||||
"@sqlite.org/sqlite-wasm": "3.48.0-build2",
|
||||
"@subframe7536/sqlite-wasm": "^0.5.1",
|
||||
"@tanstack/solid-table": "^8.20.5",
|
||||
"chart.js": "^4.4.7",
|
||||
"chartjs-chart-wordcloud": "^4.4.4",
|
||||
|
@ -58,9 +60,6 @@
|
|||
"zen-mitt": "^3.0.0"
|
||||
},
|
||||
"lint-staged": {
|
||||
"*.{ts,tsx}": [
|
||||
"biome lint",
|
||||
"biome format"
|
||||
]
|
||||
"*.{ts,tsx}": ["biome lint", "biome format"]
|
||||
}
|
||||
}
|
||||
|
|
8
pnpm-lock.yaml
generated
8
pnpm-lock.yaml
generated
|
@ -38,6 +38,9 @@ importers:
|
|||
'@sqlite.org/sqlite-wasm':
|
||||
specifier: 3.48.0-build2
|
||||
version: 3.48.0-build2
|
||||
'@subframe7536/sqlite-wasm':
|
||||
specifier: ^0.5.1
|
||||
version: 0.5.1
|
||||
'@tanstack/solid-table':
|
||||
specifier: ^8.20.5
|
||||
version: 8.20.5(solid-js@1.9.4)
|
||||
|
@ -760,6 +763,9 @@ packages:
|
|||
resolution: {integrity: sha512-nltoBHBbLZmI3VioebwUYaSugTpVcHPvL9rYa0uSkqmiLF0b9ZEM8l9NzoWAHlS6qTMqhGHFtX1lWJ/egyjohQ==}
|
||||
hasBin: true
|
||||
|
||||
'@subframe7536/sqlite-wasm@0.5.1':
|
||||
resolution: {integrity: sha512-rRszpnvcT045Jd5HkDMIxaYI95BUl8H07mhYmklNW1CbNjJtiD3dN550lRXElwcDUpTQrdhIdmA9A5QwJ+ct+A==}
|
||||
|
||||
'@swc/helpers@0.5.15':
|
||||
resolution: {integrity: sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g==}
|
||||
|
||||
|
@ -2698,6 +2704,8 @@ snapshots:
|
|||
|
||||
'@sqlite.org/sqlite-wasm@3.48.0-build2': {}
|
||||
|
||||
'@subframe7536/sqlite-wasm@0.5.1': {}
|
||||
|
||||
'@swc/helpers@0.5.15':
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
|
|
BIN
src/assets/wa-sqlite-async.wasm
Executable file
BIN
src/assets/wa-sqlite-async.wasm
Executable file
Binary file not shown.
BIN
src/assets/wa-sqlite.wasm
Executable file
BIN
src/assets/wa-sqlite.wasm
Executable file
Binary file not shown.
|
@ -1,7 +1,7 @@
|
|||
import { sql, type NotNull } from "kysely";
|
||||
import { worker, kyselyDb, SELF_ID, DB_FILENAME } from "./db";
|
||||
import { cached } from "../lib/db-cache";
|
||||
import type { MainToWorkerMsg, WorkerToMainMsg } from "~/lib/kysely-official-wasm-worker/type";
|
||||
import type { MainToWorkerMsg, WorkerToMainMsg } from "~/lib/kysely-wasqlite-worker/type";
|
||||
|
||||
export const loadDb = (statements: string[], progressCallback?: (percentage: number) => void): Promise<void> => {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
@ -23,8 +23,8 @@ export const loadDb = (statements: string[], progressCallback?: (percentage: num
|
|||
}
|
||||
};
|
||||
|
||||
worker.addEventListener("message", endListener);
|
||||
worker.addEventListener("message", progressListener);
|
||||
worker.addEventListener("message", endListener);
|
||||
|
||||
worker.postMessage([4, DB_FILENAME, true, statements] satisfies MainToWorkerMsg);
|
||||
});
|
||||
|
|
|
@ -2,8 +2,8 @@ import { makePersisted } from "@solid-primitives/storage";
|
|||
import { Kysely } from "kysely";
|
||||
import type { DB } from "./db-schema";
|
||||
import { createSignal } from "solid-js";
|
||||
import { OfficialWasmWorkerDialect } from "~/lib/kysely-official-wasm-worker";
|
||||
import wasmWorkerUrl from "~/lib/kysely-official-wasm-worker/worker?url";
|
||||
import { WaSqliteWorkerDialect } from "~/lib/kysely-wasqlite-worker";
|
||||
import wasmWorkerUrl from "~/lib/kysely-wasqlite-worker/worker?url";
|
||||
|
||||
export const SELF_ID = 2;
|
||||
|
||||
|
@ -13,7 +13,7 @@ export const worker = new Worker(wasmWorkerUrl, {
|
|||
type: "module",
|
||||
});
|
||||
|
||||
const dialect = new OfficialWasmWorkerDialect({
|
||||
const dialect = new WaSqliteWorkerDialect({
|
||||
fileName: DB_FILENAME,
|
||||
preferOPFS: true,
|
||||
worker,
|
||||
|
|
|
@ -1,38 +0,0 @@
|
|||
import type {
|
||||
DatabaseIntrospector,
|
||||
Dialect,
|
||||
DialectAdapter,
|
||||
Driver,
|
||||
Kysely,
|
||||
QueryCompiler,
|
||||
} from "kysely";
|
||||
import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely";
|
||||
import { OfficialWasmWorkerDriver } from "./driver";
|
||||
import type { OfficialWasmWorkerDialectConfig } from "./type";
|
||||
|
||||
export type {
|
||||
Promisable,
|
||||
OfficialWasmWorkerDialectConfig as WaSqliteWorkerDialectConfig,
|
||||
} from "./type";
|
||||
export { createOnMessageCallback } from "./worker/utils";
|
||||
|
||||
export class OfficialWasmWorkerDialect implements Dialect {
|
||||
constructor(private config: OfficialWasmWorkerDialectConfig) {}
|
||||
|
||||
createDriver(): Driver {
|
||||
return new OfficialWasmWorkerDriver(this.config);
|
||||
}
|
||||
|
||||
createQueryCompiler(): QueryCompiler {
|
||||
return new SqliteQueryCompiler();
|
||||
}
|
||||
|
||||
createAdapter(): DialectAdapter {
|
||||
return new SqliteAdapter();
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
createIntrospector(db: Kysely<any>): DatabaseIntrospector {
|
||||
return new SqliteIntrospector(db);
|
||||
}
|
||||
}
|
|
@ -1,54 +0,0 @@
|
|||
import type { SqlValue } from "@sqlite.org/sqlite-wasm";
|
||||
import type { DatabaseConnection, QueryResult } from "kysely";
|
||||
|
||||
export type Promisable<T> = T | Promise<T>;
|
||||
|
||||
export interface OfficialWasmWorkerDialectConfig {
|
||||
/**
|
||||
* db file name
|
||||
*/
|
||||
fileName: string;
|
||||
/**
|
||||
* prefer to store data in OPFS
|
||||
* @default true
|
||||
*/
|
||||
preferOPFS?: boolean;
|
||||
/**
|
||||
* official wasm worker
|
||||
*/
|
||||
worker?: Worker;
|
||||
onCreateConnection?: (connection: DatabaseConnection) => Promisable<void>;
|
||||
}
|
||||
|
||||
type InitMsg = [type: 0, fileName: string, useOPFS: boolean];
|
||||
|
||||
type RunMsg = [type: 1, isSelect: boolean, sql: string, parameters?: readonly unknown[]];
|
||||
|
||||
type CloseMsg = [2];
|
||||
|
||||
type StreamMsg = [type: 3, sql: string, parameters?: readonly unknown[]];
|
||||
|
||||
type LoadDbMsg = [type: 4, filename: string, useOPFS: boolean, statements: string[]];
|
||||
|
||||
export type MainToWorkerMsg = InitMsg | RunMsg | CloseMsg | StreamMsg | LoadDbMsg;
|
||||
|
||||
type Events = {
|
||||
0: null;
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
1: QueryResult<any> | null;
|
||||
2: null;
|
||||
3: {
|
||||
[columnName: string]: SqlValue;
|
||||
}[];
|
||||
4: null;
|
||||
5: number;
|
||||
6: null;
|
||||
};
|
||||
|
||||
export type WorkerToMainMsg = {
|
||||
[K in keyof Events]: [type: K, data: Events[K], err: unknown];
|
||||
}[keyof Events];
|
||||
|
||||
export type EventWithError = {
|
||||
[K in keyof Events]: [data: Events[K], err: unknown];
|
||||
};
|
|
@ -1,45 +1,52 @@
|
|||
import type { DatabaseConnection, Driver, QueryResult } from "kysely";
|
||||
import { CompiledQuery, SelectQueryNode } from "kysely";
|
||||
import type { Emitter } from "zen-mitt";
|
||||
import type { EventWithError, MainToWorkerMsg, WaSqliteWorkerDialectConfig, WorkerToMainMsg } from "./type";
|
||||
import { isModuleWorkerSupport, isOpfsSupported } from "@subframe7536/sqlite-wasm";
|
||||
import { CompiledQuery, SelectQueryNode } from "kysely";
|
||||
import { mitt } from "zen-mitt";
|
||||
import type { EventWithError, MainToWorkerMsg, OfficialWasmWorkerDialectConfig, WorkerToMainMsg } from "./type";
|
||||
import workerUrl from "./worker?url";
|
||||
import { defaultWasmURL, defaultWorker, parseWorkerOrURL } from "./utils";
|
||||
|
||||
export class OfficialWasmWorkerDriver implements Driver {
|
||||
export class WaSqliteWorkerDriver implements Driver {
|
||||
private worker?: Worker;
|
||||
private connection?: DatabaseConnection;
|
||||
private connectionMutex = new ConnectionMutex();
|
||||
private mitt?: Emitter<EventWithError>;
|
||||
constructor(private config: OfficialWasmWorkerDialectConfig) {}
|
||||
constructor(private config: WaSqliteWorkerDialectConfig) {}
|
||||
|
||||
async init(): Promise<void> {
|
||||
// try to persist storage, https://web.dev/articles/persistent-storage#request_persistent_storage
|
||||
try {
|
||||
if (navigator.storage?.persist && !(await navigator.storage.persisted())) {
|
||||
if (!(await navigator.storage.persisted())) {
|
||||
await navigator.storage.persist();
|
||||
}
|
||||
// biome-ignore lint/suspicious/noEmptyBlockStatements: <explanation>
|
||||
} catch {}
|
||||
|
||||
const useOPFS = (this.config.preferOPFS ?? true) ? await isOpfsSupported() : false;
|
||||
|
||||
this.mitt = mitt<EventWithError>();
|
||||
|
||||
this.worker =
|
||||
this.config.worker ??
|
||||
new Worker(workerUrl, {
|
||||
type: "module",
|
||||
});
|
||||
this.worker = parseWorkerOrURL(this.config.worker || defaultWorker, useOPFS || isModuleWorkerSupport());
|
||||
|
||||
this.worker.onmessage = ({ data: [type, ...msg] }: MessageEvent<WorkerToMainMsg>) => {
|
||||
// biome-ignore lint/style/noNonNullAssertion: <explanation>
|
||||
this.worker!.onmessage = ({ data: [type, ...msg] }: MessageEvent<WorkerToMainMsg>) => {
|
||||
this.mitt?.emit(type, ...msg);
|
||||
};
|
||||
|
||||
this.worker.postMessage([0, this.config.fileName, this.config.preferOPFS ?? false] satisfies MainToWorkerMsg);
|
||||
this.worker?.postMessage([
|
||||
0,
|
||||
this.config.fileName,
|
||||
// if use OPFS, wasm should use sync version
|
||||
parseWorkerOrURL(this.config.url ?? defaultWasmURL, !useOPFS) as string,
|
||||
useOPFS,
|
||||
] satisfies MainToWorkerMsg);
|
||||
|
||||
await new Promise<void>((resolve, reject) => {
|
||||
this.mitt?.once(0, (_, err) => (err ? reject(err) : resolve()));
|
||||
});
|
||||
|
||||
this.connection = new OfficialWasmWorkerConnection(this.worker, this.mitt);
|
||||
// biome-ignore lint/style/noNonNullAssertion: <explanation>
|
||||
this.connection = new WaSqliteWorkerConnection(this.worker!, this.mitt);
|
||||
await this.config.onCreateConnection?.(this.connection);
|
||||
}
|
||||
|
||||
|
@ -47,7 +54,6 @@ export class OfficialWasmWorkerDriver implements Driver {
|
|||
// SQLite only has one single connection. We use a mutex here to wait
|
||||
// until the single connection has been released.
|
||||
await this.connectionMutex.lock();
|
||||
|
||||
// biome-ignore lint/style/noNonNullAssertion: <explanation>
|
||||
return this.connection!;
|
||||
}
|
||||
|
@ -64,12 +70,9 @@ export class OfficialWasmWorkerDriver implements Driver {
|
|||
await connection.executeQuery(CompiledQuery.raw("rollback"));
|
||||
}
|
||||
|
||||
releaseConnection(): Promise<void> {
|
||||
return new Promise((resolve) => {
|
||||
this.connectionMutex.unlock();
|
||||
|
||||
resolve();
|
||||
});
|
||||
// biome-ignore lint/suspicious/useAwait: <explanation>
|
||||
async releaseConnection(): Promise<void> {
|
||||
this.connectionMutex.unlock();
|
||||
}
|
||||
|
||||
async destroy(): Promise<void> {
|
||||
|
@ -116,7 +119,7 @@ class ConnectionMutex {
|
|||
}
|
||||
}
|
||||
|
||||
class OfficialWasmWorkerConnection implements DatabaseConnection {
|
||||
class WaSqliteWorkerConnection implements DatabaseConnection {
|
||||
readonly worker: Worker;
|
||||
readonly mitt?: Emitter<EventWithError>;
|
||||
constructor(worker: Worker, mitt?: Emitter<EventWithError>) {
|
||||
|
@ -127,18 +130,20 @@ class OfficialWasmWorkerConnection implements DatabaseConnection {
|
|||
async *streamQuery<R>(compiledQuery: CompiledQuery): AsyncIterableIterator<QueryResult<R>> {
|
||||
const { parameters, sql, query } = compiledQuery;
|
||||
if (!SelectQueryNode.is(query)) {
|
||||
throw new Error("official wasm worker dialect only supports SELECT queries for streaming");
|
||||
throw new Error("WaSqlite dialect only supported SELECT queries");
|
||||
}
|
||||
this.worker.postMessage([3, sql, parameters] satisfies MainToWorkerMsg);
|
||||
let done = false;
|
||||
let resolveFn: (value: IteratorResult<QueryResult<R>>) => void;
|
||||
let rejectFn: (reason?: unknown) => void;
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
let rejectFn: (reason?: any) => void;
|
||||
|
||||
this.mitt?.on(3 /* data */, (data, err): void => {
|
||||
if (err) {
|
||||
rejectFn(err);
|
||||
} else {
|
||||
resolveFn({ value: { rows: data as R[] }, done: false });
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
resolveFn({ value: { rows: data as any }, done: false });
|
||||
}
|
||||
});
|
||||
|
||||
|
@ -167,10 +172,8 @@ class OfficialWasmWorkerConnection implements DatabaseConnection {
|
|||
}
|
||||
|
||||
async executeQuery<R>(compiledQuery: CompiledQuery<unknown>): Promise<QueryResult<R>> {
|
||||
const { sql, parameters, query } = compiledQuery;
|
||||
|
||||
const { parameters, sql, query } = compiledQuery;
|
||||
const isSelect = SelectQueryNode.is(query);
|
||||
|
||||
this.worker.postMessage([1, isSelect, sql, parameters] satisfies MainToWorkerMsg);
|
||||
return new Promise((resolve, reject) => {
|
||||
if (!this.mitt) {
|
48
src/lib/kysely-wasqlite-worker/index.ts
Normal file
48
src/lib/kysely-wasqlite-worker/index.ts
Normal file
|
@ -0,0 +1,48 @@
|
|||
import type { DatabaseIntrospector, Dialect, DialectAdapter, Driver, Kysely, QueryCompiler } from "kysely";
|
||||
import type { WaSqliteWorkerDialectConfig } from "./type";
|
||||
import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely";
|
||||
import { WaSqliteWorkerDriver } from "./driver";
|
||||
|
||||
export type { Promisable, WaSqliteWorkerDialectConfig } from "./type";
|
||||
export { createOnMessageCallback } from "./worker/utils";
|
||||
|
||||
export {
|
||||
customFunction,
|
||||
isIdbSupported,
|
||||
isModuleWorkerSupport,
|
||||
isOpfsSupported,
|
||||
type SQLiteDB,
|
||||
} from "@subframe7536/sqlite-wasm";
|
||||
|
||||
export class WaSqliteWorkerDialect implements Dialect {
|
||||
/**
|
||||
* dialect for [`wa-sqlite`](https://github.com/rhashimoto/wa-sqlite),
|
||||
* execute sql in `Web Worker`,
|
||||
* store data in [OPFS](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system) or IndexedDB
|
||||
*
|
||||
* @example
|
||||
* import { WaSqliteWorkerDialect } from 'kysely-wasqlite-worker'
|
||||
*
|
||||
* const dialect = new WaSqliteWorkerDialect({
|
||||
* fileName: 'test',
|
||||
* })
|
||||
*/
|
||||
constructor(private config: WaSqliteWorkerDialectConfig) {}
|
||||
|
||||
createDriver(): Driver {
|
||||
return new WaSqliteWorkerDriver(this.config);
|
||||
}
|
||||
|
||||
createQueryCompiler(): QueryCompiler {
|
||||
return new SqliteQueryCompiler();
|
||||
}
|
||||
|
||||
createAdapter(): DialectAdapter {
|
||||
return new SqliteAdapter();
|
||||
}
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
createIntrospector(db: Kysely<any>): DatabaseIntrospector {
|
||||
return new SqliteIntrospector(db);
|
||||
}
|
||||
}
|
81
src/lib/kysely-wasqlite-worker/type.ts
Normal file
81
src/lib/kysely-wasqlite-worker/type.ts
Normal file
|
@ -0,0 +1,81 @@
|
|||
import type { SqlValue } from "@sqlite.org/sqlite-wasm";
|
||||
import type { DatabaseConnection, QueryResult } from "kysely";
|
||||
|
||||
export type Promisable<T> = T | Promise<T>;
|
||||
|
||||
export interface WaSqliteWorkerDialectConfig {
|
||||
/**
|
||||
* db file name
|
||||
*/
|
||||
fileName: string;
|
||||
/**
|
||||
* prefer to store data in OPFS
|
||||
* @default true
|
||||
*/
|
||||
preferOPFS?: boolean;
|
||||
/**
|
||||
* wasqlite worker
|
||||
*
|
||||
* built-in: {@link useDefaultWorker}
|
||||
* @param supportModuleWorker if support `{ type: 'module' }` in worker options
|
||||
* @example
|
||||
* import { useDefaultWorker } from 'kysely-wasqlite-worker'
|
||||
* @example
|
||||
* (supportModuleWorker) => supportModuleWorker
|
||||
* ? new Worker(
|
||||
* new URL('kysely-wasqlite-worker/worker-module', import.meta.url),
|
||||
* { type: 'module', credentials: 'same-origin' }
|
||||
* )
|
||||
* : new Worker(
|
||||
* new URL('kysely-wasqlite-worker/worker-classic', import.meta.url),
|
||||
* { type: 'classic', name: 'test' }
|
||||
* )
|
||||
*/
|
||||
worker?: Worker | ((supportModuleWorker: boolean) => Worker);
|
||||
/**
|
||||
* wasm URL
|
||||
*
|
||||
* built-in: {@link useDefaultWasmURL}
|
||||
* @param useAsyncWasm if need to use wa-sqlite-async.wasm
|
||||
* @example
|
||||
* import { useDefaultWasmURL } from 'kysely-wasqlite-worker'
|
||||
* @example
|
||||
* (useAsyncWasm) => useAsyncWasm
|
||||
* ? 'https://cdn.jsdelivr.net/gh/rhashimoto/wa-sqlite@v1.0.0/dist/wa-sqlite-async.wasm'
|
||||
* : new URL('kysely-wasqlite-worker/wasm-sync', import.meta.url).href
|
||||
*/
|
||||
url?: string | ((useAsyncWasm: boolean) => string);
|
||||
onCreateConnection?: (connection: DatabaseConnection) => Promisable<void>;
|
||||
}
|
||||
|
||||
type RunMsg = [type: 1, isSelect: boolean, sql: string, parameters?: readonly unknown[]];
|
||||
type StreamMsg = [type: 3, sql: string, parameters?: readonly unknown[]];
|
||||
|
||||
type InitMsg = [type: 0, url: string, fileName: string, useOPFS: boolean];
|
||||
|
||||
type CloseMsg = [2];
|
||||
|
||||
type LoadDbMsg = [type: 4, filename: string, useOPFS: boolean, statements: string[]];
|
||||
|
||||
export type MainToWorkerMsg = InitMsg | RunMsg | CloseMsg | StreamMsg | LoadDbMsg;
|
||||
|
||||
export type WorkerToMainMsg = {
|
||||
[K in keyof Events]: [type: K, data: Events[K], err: unknown];
|
||||
}[keyof Events];
|
||||
|
||||
export type EventWithError = {
|
||||
[K in keyof Events]: [data: Events[K], err: unknown];
|
||||
};
|
||||
|
||||
type Events = {
|
||||
0: null;
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
1: QueryResult<any> | null;
|
||||
2: null;
|
||||
3: {
|
||||
[columnName: string]: SqlValue;
|
||||
}[];
|
||||
4: null;
|
||||
5: number;
|
||||
6: null;
|
||||
};
|
24
src/lib/kysely-wasqlite-worker/utils.ts
Normal file
24
src/lib/kysely-wasqlite-worker/utils.ts
Normal file
|
@ -0,0 +1,24 @@
|
|||
import asyncWasmUrl from "~/assets/wa-sqlite-async.wasm?url";
|
||||
import syncWasmUrl from "~/assets/wa-sqlite.wasm?url";
|
||||
|
||||
export function parseWorkerOrURL<T, P extends T | ((is: boolean) => T)>(obj: P, is: boolean): T {
|
||||
return typeof obj === "function" ? obj(is) : obj;
|
||||
}
|
||||
|
||||
/**
|
||||
* auto load target worker
|
||||
*
|
||||
* **only basic worker options**
|
||||
*/
|
||||
export function defaultWorker(support: boolean): Worker {
|
||||
return support
|
||||
? new Worker(new URL("worker.mjs", import.meta.url), { type: "module" })
|
||||
: new Worker(new URL("worker.js", import.meta.url));
|
||||
}
|
||||
|
||||
/**
|
||||
* auto load target wasm
|
||||
*/
|
||||
export function defaultWasmURL(useAsyncWasm: boolean): string {
|
||||
return useAsyncWasm ? asyncWasmUrl : syncWasmUrl;
|
||||
}
|
|
@ -1,70 +1,52 @@
|
|||
import sqlite3InitModule, {
|
||||
type BindingSpec,
|
||||
type Database,
|
||||
type OpfsDatabase,
|
||||
type Sqlite3Static,
|
||||
type SqlValue,
|
||||
} from "@sqlite.org/sqlite-wasm";
|
||||
import type { SQLiteDB } from "@subframe7536/sqlite-wasm";
|
||||
import type { QueryResult } from "kysely";
|
||||
import type { MainToWorkerMsg, WorkerToMainMsg } from "../type";
|
||||
import { initSQLite } from "@subframe7536/sqlite-wasm";
|
||||
import { defaultWasmURL, parseWorkerOrURL } from "../utils";
|
||||
|
||||
let sqlite3: Sqlite3Static;
|
||||
let db: Database | OpfsDatabase;
|
||||
let db: SQLiteDB;
|
||||
|
||||
async function init(
|
||||
fileName: string,
|
||||
preferOpfs: boolean,
|
||||
afterInit?: (sqliteDB: Database | OpfsDatabase) => Promise<void>,
|
||||
url: string,
|
||||
useOPFS: boolean,
|
||||
afterInit?: (sqliteDB: SQLiteDB) => Promise<void>,
|
||||
): Promise<void> {
|
||||
sqlite3 = await sqlite3InitModule();
|
||||
|
||||
db = preferOpfs && "opfs" in sqlite3.oo1 ? new sqlite3.oo1.OpfsDb(fileName) : new sqlite3.oo1.DB(fileName);
|
||||
|
||||
db = await initSQLite(
|
||||
(useOPFS
|
||||
? (await import("@subframe7536/sqlite-wasm/opfs")).useOpfsStorage
|
||||
: (await import("@subframe7536/sqlite-wasm/idb")).useIdbStorage)(fileName, { url }),
|
||||
);
|
||||
await afterInit?.(db);
|
||||
}
|
||||
|
||||
function exec(isSelect: boolean, sql: string, parameters?: readonly unknown[]): QueryResult<SqlValue[]> {
|
||||
const rows = db.exec(sql, {
|
||||
bind: parameters as BindingSpec,
|
||||
returnValue: "resultRows",
|
||||
});
|
||||
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
async function exec(isSelect: boolean, sql: string, parameters?: readonly unknown[]): Promise<QueryResult<any>> {
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
const rows = await db.run(sql, parameters as any[]);
|
||||
return isSelect || rows.length
|
||||
? { rows }
|
||||
: {
|
||||
rows,
|
||||
insertId: BigInt(sqlite3.capi.sqlite3_last_insert_rowid(db)),
|
||||
insertId: BigInt(db.lastInsertRowId()),
|
||||
numAffectedRows: BigInt(db.changes()),
|
||||
};
|
||||
}
|
||||
|
||||
function stream(
|
||||
onData: (data: { [columnName: string]: SqlValue }) => void,
|
||||
sql: string,
|
||||
parameters?: readonly unknown[],
|
||||
): void {
|
||||
const stmt = db.prepare(sql);
|
||||
|
||||
if (parameters) {
|
||||
stmt.bind(parameters as BindingSpec);
|
||||
}
|
||||
|
||||
while (stmt.step()) {
|
||||
onData(stmt.get({}));
|
||||
}
|
||||
|
||||
stmt.finalize();
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
async function stream(onData: (data: any) => void, sql: string, parameters?: readonly unknown[]): Promise<void> {
|
||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||
await db.stream(onData, sql, parameters as any[]);
|
||||
}
|
||||
|
||||
async function loadDb(onData: (percentage: number) => void, fileName: string, useOPFS: boolean, statements: string[]) {
|
||||
if (!db) {
|
||||
await init(fileName, useOPFS);
|
||||
await init(fileName, parseWorkerOrURL(defaultWasmURL, !useOPFS) as string, useOPFS);
|
||||
}
|
||||
|
||||
const length = statements.length;
|
||||
let percentage = 0;
|
||||
|
||||
for (let i = 0; i < length; i++) {
|
||||
for (let i = 0; i < length; i += 1000) {
|
||||
const newPercentage = Math.round((i / length) * 100);
|
||||
|
||||
if (newPercentage !== percentage) {
|
||||
|
@ -75,8 +57,10 @@ async function loadDb(onData: (percentage: number) => void, fileName: string, us
|
|||
|
||||
console.log("executing statement");
|
||||
|
||||
db.exec(statements[i]);
|
||||
await db.run(statements.slice(i, i + 1000).join(";"));
|
||||
}
|
||||
|
||||
// await db.run(statements.join(";"));
|
||||
}
|
||||
|
||||
/**
|
||||
|
@ -92,7 +76,7 @@ async function loadDb(onData: (percentage: number) => void, fileName: string, us
|
|||
* )
|
||||
*/
|
||||
export function createOnMessageCallback(
|
||||
afterInit?: (sqliteDB: Database | OpfsDatabase) => Promise<void>,
|
||||
afterInit?: (sqliteDB: SQLiteDB) => Promise<void>,
|
||||
): (event: MessageEvent<MainToWorkerMsg>) => Promise<void> {
|
||||
return async ({ data: [msg, data1, data2, data3] }: MessageEvent<MainToWorkerMsg>) => {
|
||||
const ret: WorkerToMainMsg = [msg, null, null];
|
||||
|
@ -100,16 +84,16 @@ export function createOnMessageCallback(
|
|||
try {
|
||||
switch (msg) {
|
||||
case 0:
|
||||
await init(data1, data2, afterInit);
|
||||
await init(data1, data2, data3, afterInit);
|
||||
break;
|
||||
case 1:
|
||||
ret[1] = exec(data1, data2, data3);
|
||||
ret[1] = await exec(data1, data2, data3);
|
||||
break;
|
||||
case 2:
|
||||
db.close();
|
||||
await db.close();
|
||||
break;
|
||||
case 3:
|
||||
stream((val) => postMessage([3, [val], null] satisfies WorkerToMainMsg), data1, data2);
|
||||
await stream((val) => postMessage([3, [val], null] satisfies WorkerToMainMsg), data1, data2);
|
||||
ret[0] = 4;
|
||||
break;
|
||||
case 4:
|
Loading…
Add table
Add a link
Reference in a new issue