feat(db): use wa-sqlite

This commit is contained in:
Samuel 2025-01-20 17:51:56 +01:00
parent c9aaf2ecef
commit a2bc4115a2
14 changed files with 232 additions and 177 deletions

View file

@ -8,7 +8,8 @@
"dev": "vite", "dev": "vite",
"build": "vite build", "build": "vite build",
"serve": "vite preview", "serve": "vite preview",
"prepare": "husky" "prepare": "husky",
"postinstall": "cp ./node_modules/@subframe7536/sqlite-wasm/dist/*.wasm ./src/assets/"
}, },
"license": "MIT", "license": "MIT",
"devDependencies": { "devDependencies": {
@ -40,6 +41,7 @@
"@solidjs/meta": "^0.29.4", "@solidjs/meta": "^0.29.4",
"@solidjs/router": "^0.15.3", "@solidjs/router": "^0.15.3",
"@sqlite.org/sqlite-wasm": "3.48.0-build2", "@sqlite.org/sqlite-wasm": "3.48.0-build2",
"@subframe7536/sqlite-wasm": "^0.5.1",
"@tanstack/solid-table": "^8.20.5", "@tanstack/solid-table": "^8.20.5",
"chart.js": "^4.4.7", "chart.js": "^4.4.7",
"chartjs-chart-wordcloud": "^4.4.4", "chartjs-chart-wordcloud": "^4.4.4",
@ -58,9 +60,6 @@
"zen-mitt": "^3.0.0" "zen-mitt": "^3.0.0"
}, },
"lint-staged": { "lint-staged": {
"*.{ts,tsx}": [ "*.{ts,tsx}": ["biome lint", "biome format"]
"biome lint",
"biome format"
]
} }
} }

8
pnpm-lock.yaml generated
View file

@ -38,6 +38,9 @@ importers:
'@sqlite.org/sqlite-wasm': '@sqlite.org/sqlite-wasm':
specifier: 3.48.0-build2 specifier: 3.48.0-build2
version: 3.48.0-build2 version: 3.48.0-build2
'@subframe7536/sqlite-wasm':
specifier: ^0.5.1
version: 0.5.1
'@tanstack/solid-table': '@tanstack/solid-table':
specifier: ^8.20.5 specifier: ^8.20.5
version: 8.20.5(solid-js@1.9.4) version: 8.20.5(solid-js@1.9.4)
@ -760,6 +763,9 @@ packages:
resolution: {integrity: sha512-nltoBHBbLZmI3VioebwUYaSugTpVcHPvL9rYa0uSkqmiLF0b9ZEM8l9NzoWAHlS6qTMqhGHFtX1lWJ/egyjohQ==} resolution: {integrity: sha512-nltoBHBbLZmI3VioebwUYaSugTpVcHPvL9rYa0uSkqmiLF0b9ZEM8l9NzoWAHlS6qTMqhGHFtX1lWJ/egyjohQ==}
hasBin: true hasBin: true
'@subframe7536/sqlite-wasm@0.5.1':
resolution: {integrity: sha512-rRszpnvcT045Jd5HkDMIxaYI95BUl8H07mhYmklNW1CbNjJtiD3dN550lRXElwcDUpTQrdhIdmA9A5QwJ+ct+A==}
'@swc/helpers@0.5.15': '@swc/helpers@0.5.15':
resolution: {integrity: sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g==} resolution: {integrity: sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g==}
@ -2698,6 +2704,8 @@ snapshots:
'@sqlite.org/sqlite-wasm@3.48.0-build2': {} '@sqlite.org/sqlite-wasm@3.48.0-build2': {}
'@subframe7536/sqlite-wasm@0.5.1': {}
'@swc/helpers@0.5.15': '@swc/helpers@0.5.15':
dependencies: dependencies:
tslib: 2.8.1 tslib: 2.8.1

BIN
src/assets/wa-sqlite-async.wasm Executable file

Binary file not shown.

BIN
src/assets/wa-sqlite.wasm Executable file

Binary file not shown.

View file

@ -1,7 +1,7 @@
import { sql, type NotNull } from "kysely"; import { sql, type NotNull } from "kysely";
import { worker, kyselyDb, SELF_ID, DB_FILENAME } from "./db"; import { worker, kyselyDb, SELF_ID, DB_FILENAME } from "./db";
import { cached } from "../lib/db-cache"; import { cached } from "../lib/db-cache";
import type { MainToWorkerMsg, WorkerToMainMsg } from "~/lib/kysely-official-wasm-worker/type"; import type { MainToWorkerMsg, WorkerToMainMsg } from "~/lib/kysely-wasqlite-worker/type";
export const loadDb = (statements: string[], progressCallback?: (percentage: number) => void): Promise<void> => { export const loadDb = (statements: string[], progressCallback?: (percentage: number) => void): Promise<void> => {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
@ -23,8 +23,8 @@ export const loadDb = (statements: string[], progressCallback?: (percentage: num
} }
}; };
worker.addEventListener("message", endListener);
worker.addEventListener("message", progressListener); worker.addEventListener("message", progressListener);
worker.addEventListener("message", endListener);
worker.postMessage([4, DB_FILENAME, true, statements] satisfies MainToWorkerMsg); worker.postMessage([4, DB_FILENAME, true, statements] satisfies MainToWorkerMsg);
}); });

View file

@ -2,8 +2,8 @@ import { makePersisted } from "@solid-primitives/storage";
import { Kysely } from "kysely"; import { Kysely } from "kysely";
import type { DB } from "./db-schema"; import type { DB } from "./db-schema";
import { createSignal } from "solid-js"; import { createSignal } from "solid-js";
import { OfficialWasmWorkerDialect } from "~/lib/kysely-official-wasm-worker"; import { WaSqliteWorkerDialect } from "~/lib/kysely-wasqlite-worker";
import wasmWorkerUrl from "~/lib/kysely-official-wasm-worker/worker?url"; import wasmWorkerUrl from "~/lib/kysely-wasqlite-worker/worker?url";
export const SELF_ID = 2; export const SELF_ID = 2;
@ -13,7 +13,7 @@ export const worker = new Worker(wasmWorkerUrl, {
type: "module", type: "module",
}); });
const dialect = new OfficialWasmWorkerDialect({ const dialect = new WaSqliteWorkerDialect({
fileName: DB_FILENAME, fileName: DB_FILENAME,
preferOPFS: true, preferOPFS: true,
worker, worker,

View file

@ -1,38 +0,0 @@
import type {
DatabaseIntrospector,
Dialect,
DialectAdapter,
Driver,
Kysely,
QueryCompiler,
} from "kysely";
import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely";
import { OfficialWasmWorkerDriver } from "./driver";
import type { OfficialWasmWorkerDialectConfig } from "./type";
export type {
Promisable,
OfficialWasmWorkerDialectConfig as WaSqliteWorkerDialectConfig,
} from "./type";
export { createOnMessageCallback } from "./worker/utils";
export class OfficialWasmWorkerDialect implements Dialect {
constructor(private config: OfficialWasmWorkerDialectConfig) {}
createDriver(): Driver {
return new OfficialWasmWorkerDriver(this.config);
}
createQueryCompiler(): QueryCompiler {
return new SqliteQueryCompiler();
}
createAdapter(): DialectAdapter {
return new SqliteAdapter();
}
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
createIntrospector(db: Kysely<any>): DatabaseIntrospector {
return new SqliteIntrospector(db);
}
}

View file

@ -1,54 +0,0 @@
import type { SqlValue } from "@sqlite.org/sqlite-wasm";
import type { DatabaseConnection, QueryResult } from "kysely";
export type Promisable<T> = T | Promise<T>;
export interface OfficialWasmWorkerDialectConfig {
/**
* db file name
*/
fileName: string;
/**
* prefer to store data in OPFS
* @default true
*/
preferOPFS?: boolean;
/**
* official wasm worker
*/
worker?: Worker;
onCreateConnection?: (connection: DatabaseConnection) => Promisable<void>;
}
type InitMsg = [type: 0, fileName: string, useOPFS: boolean];
type RunMsg = [type: 1, isSelect: boolean, sql: string, parameters?: readonly unknown[]];
type CloseMsg = [2];
type StreamMsg = [type: 3, sql: string, parameters?: readonly unknown[]];
type LoadDbMsg = [type: 4, filename: string, useOPFS: boolean, statements: string[]];
export type MainToWorkerMsg = InitMsg | RunMsg | CloseMsg | StreamMsg | LoadDbMsg;
type Events = {
0: null;
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
1: QueryResult<any> | null;
2: null;
3: {
[columnName: string]: SqlValue;
}[];
4: null;
5: number;
6: null;
};
export type WorkerToMainMsg = {
[K in keyof Events]: [type: K, data: Events[K], err: unknown];
}[keyof Events];
export type EventWithError = {
[K in keyof Events]: [data: Events[K], err: unknown];
};

View file

@ -1,45 +1,52 @@
import type { DatabaseConnection, Driver, QueryResult } from "kysely"; import type { DatabaseConnection, Driver, QueryResult } from "kysely";
import { CompiledQuery, SelectQueryNode } from "kysely";
import type { Emitter } from "zen-mitt"; import type { Emitter } from "zen-mitt";
import type { EventWithError, MainToWorkerMsg, WaSqliteWorkerDialectConfig, WorkerToMainMsg } from "./type";
import { isModuleWorkerSupport, isOpfsSupported } from "@subframe7536/sqlite-wasm";
import { CompiledQuery, SelectQueryNode } from "kysely";
import { mitt } from "zen-mitt"; import { mitt } from "zen-mitt";
import type { EventWithError, MainToWorkerMsg, OfficialWasmWorkerDialectConfig, WorkerToMainMsg } from "./type"; import { defaultWasmURL, defaultWorker, parseWorkerOrURL } from "./utils";
import workerUrl from "./worker?url";
export class OfficialWasmWorkerDriver implements Driver { export class WaSqliteWorkerDriver implements Driver {
private worker?: Worker; private worker?: Worker;
private connection?: DatabaseConnection; private connection?: DatabaseConnection;
private connectionMutex = new ConnectionMutex(); private connectionMutex = new ConnectionMutex();
private mitt?: Emitter<EventWithError>; private mitt?: Emitter<EventWithError>;
constructor(private config: OfficialWasmWorkerDialectConfig) {} constructor(private config: WaSqliteWorkerDialectConfig) {}
async init(): Promise<void> { async init(): Promise<void> {
// try to persist storage, https://web.dev/articles/persistent-storage#request_persistent_storage // try to persist storage, https://web.dev/articles/persistent-storage#request_persistent_storage
try { try {
if (navigator.storage?.persist && !(await navigator.storage.persisted())) { if (!(await navigator.storage.persisted())) {
await navigator.storage.persist(); await navigator.storage.persist();
} }
// biome-ignore lint/suspicious/noEmptyBlockStatements: <explanation> // biome-ignore lint/suspicious/noEmptyBlockStatements: <explanation>
} catch {} } catch {}
const useOPFS = (this.config.preferOPFS ?? true) ? await isOpfsSupported() : false;
this.mitt = mitt<EventWithError>(); this.mitt = mitt<EventWithError>();
this.worker = this.worker = parseWorkerOrURL(this.config.worker || defaultWorker, useOPFS || isModuleWorkerSupport());
this.config.worker ??
new Worker(workerUrl, {
type: "module",
});
this.worker.onmessage = ({ data: [type, ...msg] }: MessageEvent<WorkerToMainMsg>) => { // biome-ignore lint/style/noNonNullAssertion: <explanation>
this.worker!.onmessage = ({ data: [type, ...msg] }: MessageEvent<WorkerToMainMsg>) => {
this.mitt?.emit(type, ...msg); this.mitt?.emit(type, ...msg);
}; };
this.worker.postMessage([0, this.config.fileName, this.config.preferOPFS ?? false] satisfies MainToWorkerMsg); this.worker?.postMessage([
0,
this.config.fileName,
// if use OPFS, wasm should use sync version
parseWorkerOrURL(this.config.url ?? defaultWasmURL, !useOPFS) as string,
useOPFS,
] satisfies MainToWorkerMsg);
await new Promise<void>((resolve, reject) => { await new Promise<void>((resolve, reject) => {
this.mitt?.once(0, (_, err) => (err ? reject(err) : resolve())); this.mitt?.once(0, (_, err) => (err ? reject(err) : resolve()));
}); });
this.connection = new OfficialWasmWorkerConnection(this.worker, this.mitt); // biome-ignore lint/style/noNonNullAssertion: <explanation>
this.connection = new WaSqliteWorkerConnection(this.worker!, this.mitt);
await this.config.onCreateConnection?.(this.connection); await this.config.onCreateConnection?.(this.connection);
} }
@ -47,7 +54,6 @@ export class OfficialWasmWorkerDriver implements Driver {
// SQLite only has one single connection. We use a mutex here to wait // SQLite only has one single connection. We use a mutex here to wait
// until the single connection has been released. // until the single connection has been released.
await this.connectionMutex.lock(); await this.connectionMutex.lock();
// biome-ignore lint/style/noNonNullAssertion: <explanation> // biome-ignore lint/style/noNonNullAssertion: <explanation>
return this.connection!; return this.connection!;
} }
@ -64,12 +70,9 @@ export class OfficialWasmWorkerDriver implements Driver {
await connection.executeQuery(CompiledQuery.raw("rollback")); await connection.executeQuery(CompiledQuery.raw("rollback"));
} }
releaseConnection(): Promise<void> { // biome-ignore lint/suspicious/useAwait: <explanation>
return new Promise((resolve) => { async releaseConnection(): Promise<void> {
this.connectionMutex.unlock(); this.connectionMutex.unlock();
resolve();
});
} }
async destroy(): Promise<void> { async destroy(): Promise<void> {
@ -116,7 +119,7 @@ class ConnectionMutex {
} }
} }
class OfficialWasmWorkerConnection implements DatabaseConnection { class WaSqliteWorkerConnection implements DatabaseConnection {
readonly worker: Worker; readonly worker: Worker;
readonly mitt?: Emitter<EventWithError>; readonly mitt?: Emitter<EventWithError>;
constructor(worker: Worker, mitt?: Emitter<EventWithError>) { constructor(worker: Worker, mitt?: Emitter<EventWithError>) {
@ -127,18 +130,20 @@ class OfficialWasmWorkerConnection implements DatabaseConnection {
async *streamQuery<R>(compiledQuery: CompiledQuery): AsyncIterableIterator<QueryResult<R>> { async *streamQuery<R>(compiledQuery: CompiledQuery): AsyncIterableIterator<QueryResult<R>> {
const { parameters, sql, query } = compiledQuery; const { parameters, sql, query } = compiledQuery;
if (!SelectQueryNode.is(query)) { if (!SelectQueryNode.is(query)) {
throw new Error("official wasm worker dialect only supports SELECT queries for streaming"); throw new Error("WaSqlite dialect only supported SELECT queries");
} }
this.worker.postMessage([3, sql, parameters] satisfies MainToWorkerMsg); this.worker.postMessage([3, sql, parameters] satisfies MainToWorkerMsg);
let done = false; let done = false;
let resolveFn: (value: IteratorResult<QueryResult<R>>) => void; let resolveFn: (value: IteratorResult<QueryResult<R>>) => void;
let rejectFn: (reason?: unknown) => void; // biome-ignore lint/suspicious/noExplicitAny: <explanation>
let rejectFn: (reason?: any) => void;
this.mitt?.on(3 /* data */, (data, err): void => { this.mitt?.on(3 /* data */, (data, err): void => {
if (err) { if (err) {
rejectFn(err); rejectFn(err);
} else { } else {
resolveFn({ value: { rows: data as R[] }, done: false }); // biome-ignore lint/suspicious/noExplicitAny: <explanation>
resolveFn({ value: { rows: data as any }, done: false });
} }
}); });
@ -167,10 +172,8 @@ class OfficialWasmWorkerConnection implements DatabaseConnection {
} }
async executeQuery<R>(compiledQuery: CompiledQuery<unknown>): Promise<QueryResult<R>> { async executeQuery<R>(compiledQuery: CompiledQuery<unknown>): Promise<QueryResult<R>> {
const { sql, parameters, query } = compiledQuery; const { parameters, sql, query } = compiledQuery;
const isSelect = SelectQueryNode.is(query); const isSelect = SelectQueryNode.is(query);
this.worker.postMessage([1, isSelect, sql, parameters] satisfies MainToWorkerMsg); this.worker.postMessage([1, isSelect, sql, parameters] satisfies MainToWorkerMsg);
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
if (!this.mitt) { if (!this.mitt) {

View file

@ -0,0 +1,48 @@
import type { DatabaseIntrospector, Dialect, DialectAdapter, Driver, Kysely, QueryCompiler } from "kysely";
import type { WaSqliteWorkerDialectConfig } from "./type";
import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely";
import { WaSqliteWorkerDriver } from "./driver";
export type { Promisable, WaSqliteWorkerDialectConfig } from "./type";
export { createOnMessageCallback } from "./worker/utils";
export {
customFunction,
isIdbSupported,
isModuleWorkerSupport,
isOpfsSupported,
type SQLiteDB,
} from "@subframe7536/sqlite-wasm";
export class WaSqliteWorkerDialect implements Dialect {
/**
* dialect for [`wa-sqlite`](https://github.com/rhashimoto/wa-sqlite),
* execute sql in `Web Worker`,
* store data in [OPFS](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system) or IndexedDB
*
* @example
* import { WaSqliteWorkerDialect } from 'kysely-wasqlite-worker'
*
* const dialect = new WaSqliteWorkerDialect({
* fileName: 'test',
* })
*/
constructor(private config: WaSqliteWorkerDialectConfig) {}
createDriver(): Driver {
return new WaSqliteWorkerDriver(this.config);
}
createQueryCompiler(): QueryCompiler {
return new SqliteQueryCompiler();
}
createAdapter(): DialectAdapter {
return new SqliteAdapter();
}
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
createIntrospector(db: Kysely<any>): DatabaseIntrospector {
return new SqliteIntrospector(db);
}
}

View file

@ -0,0 +1,81 @@
import type { SqlValue } from "@sqlite.org/sqlite-wasm";
import type { DatabaseConnection, QueryResult } from "kysely";
export type Promisable<T> = T | Promise<T>;
export interface WaSqliteWorkerDialectConfig {
/**
* db file name
*/
fileName: string;
/**
* prefer to store data in OPFS
* @default true
*/
preferOPFS?: boolean;
/**
* wasqlite worker
*
* built-in: {@link useDefaultWorker}
* @param supportModuleWorker if support `{ type: 'module' }` in worker options
* @example
* import { useDefaultWorker } from 'kysely-wasqlite-worker'
* @example
* (supportModuleWorker) => supportModuleWorker
* ? new Worker(
* new URL('kysely-wasqlite-worker/worker-module', import.meta.url),
* { type: 'module', credentials: 'same-origin' }
* )
* : new Worker(
* new URL('kysely-wasqlite-worker/worker-classic', import.meta.url),
* { type: 'classic', name: 'test' }
* )
*/
worker?: Worker | ((supportModuleWorker: boolean) => Worker);
/**
* wasm URL
*
* built-in: {@link useDefaultWasmURL}
* @param useAsyncWasm if need to use wa-sqlite-async.wasm
* @example
* import { useDefaultWasmURL } from 'kysely-wasqlite-worker'
* @example
* (useAsyncWasm) => useAsyncWasm
* ? 'https://cdn.jsdelivr.net/gh/rhashimoto/wa-sqlite@v1.0.0/dist/wa-sqlite-async.wasm'
* : new URL('kysely-wasqlite-worker/wasm-sync', import.meta.url).href
*/
url?: string | ((useAsyncWasm: boolean) => string);
onCreateConnection?: (connection: DatabaseConnection) => Promisable<void>;
}
type RunMsg = [type: 1, isSelect: boolean, sql: string, parameters?: readonly unknown[]];
type StreamMsg = [type: 3, sql: string, parameters?: readonly unknown[]];
type InitMsg = [type: 0, url: string, fileName: string, useOPFS: boolean];
type CloseMsg = [2];
type LoadDbMsg = [type: 4, filename: string, useOPFS: boolean, statements: string[]];
export type MainToWorkerMsg = InitMsg | RunMsg | CloseMsg | StreamMsg | LoadDbMsg;
export type WorkerToMainMsg = {
[K in keyof Events]: [type: K, data: Events[K], err: unknown];
}[keyof Events];
export type EventWithError = {
[K in keyof Events]: [data: Events[K], err: unknown];
};
type Events = {
0: null;
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
1: QueryResult<any> | null;
2: null;
3: {
[columnName: string]: SqlValue;
}[];
4: null;
5: number;
6: null;
};

View file

@ -0,0 +1,24 @@
import asyncWasmUrl from "~/assets/wa-sqlite-async.wasm?url";
import syncWasmUrl from "~/assets/wa-sqlite.wasm?url";
export function parseWorkerOrURL<T, P extends T | ((is: boolean) => T)>(obj: P, is: boolean): T {
return typeof obj === "function" ? obj(is) : obj;
}
/**
* auto load target worker
*
* **only basic worker options**
*/
export function defaultWorker(support: boolean): Worker {
return support
? new Worker(new URL("worker.mjs", import.meta.url), { type: "module" })
: new Worker(new URL("worker.js", import.meta.url));
}
/**
* auto load target wasm
*/
export function defaultWasmURL(useAsyncWasm: boolean): string {
return useAsyncWasm ? asyncWasmUrl : syncWasmUrl;
}

View file

@ -1,70 +1,52 @@
import sqlite3InitModule, { import type { SQLiteDB } from "@subframe7536/sqlite-wasm";
type BindingSpec,
type Database,
type OpfsDatabase,
type Sqlite3Static,
type SqlValue,
} from "@sqlite.org/sqlite-wasm";
import type { QueryResult } from "kysely"; import type { QueryResult } from "kysely";
import type { MainToWorkerMsg, WorkerToMainMsg } from "../type"; import type { MainToWorkerMsg, WorkerToMainMsg } from "../type";
import { initSQLite } from "@subframe7536/sqlite-wasm";
import { defaultWasmURL, parseWorkerOrURL } from "../utils";
let sqlite3: Sqlite3Static; let db: SQLiteDB;
let db: Database | OpfsDatabase;
async function init( async function init(
fileName: string, fileName: string,
preferOpfs: boolean, url: string,
afterInit?: (sqliteDB: Database | OpfsDatabase) => Promise<void>, useOPFS: boolean,
afterInit?: (sqliteDB: SQLiteDB) => Promise<void>,
): Promise<void> { ): Promise<void> {
sqlite3 = await sqlite3InitModule(); db = await initSQLite(
(useOPFS
db = preferOpfs && "opfs" in sqlite3.oo1 ? new sqlite3.oo1.OpfsDb(fileName) : new sqlite3.oo1.DB(fileName); ? (await import("@subframe7536/sqlite-wasm/opfs")).useOpfsStorage
: (await import("@subframe7536/sqlite-wasm/idb")).useIdbStorage)(fileName, { url }),
);
await afterInit?.(db); await afterInit?.(db);
} }
function exec(isSelect: boolean, sql: string, parameters?: readonly unknown[]): QueryResult<SqlValue[]> { // biome-ignore lint/suspicious/noExplicitAny: <explanation>
const rows = db.exec(sql, { async function exec(isSelect: boolean, sql: string, parameters?: readonly unknown[]): Promise<QueryResult<any>> {
bind: parameters as BindingSpec, // biome-ignore lint/suspicious/noExplicitAny: <explanation>
returnValue: "resultRows", const rows = await db.run(sql, parameters as any[]);
});
return isSelect || rows.length return isSelect || rows.length
? { rows } ? { rows }
: { : {
rows, rows,
insertId: BigInt(sqlite3.capi.sqlite3_last_insert_rowid(db)), insertId: BigInt(db.lastInsertRowId()),
numAffectedRows: BigInt(db.changes()), numAffectedRows: BigInt(db.changes()),
}; };
} }
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
function stream( async function stream(onData: (data: any) => void, sql: string, parameters?: readonly unknown[]): Promise<void> {
onData: (data: { [columnName: string]: SqlValue }) => void, // biome-ignore lint/suspicious/noExplicitAny: <explanation>
sql: string, await db.stream(onData, sql, parameters as any[]);
parameters?: readonly unknown[],
): void {
const stmt = db.prepare(sql);
if (parameters) {
stmt.bind(parameters as BindingSpec);
}
while (stmt.step()) {
onData(stmt.get({}));
}
stmt.finalize();
} }
async function loadDb(onData: (percentage: number) => void, fileName: string, useOPFS: boolean, statements: string[]) { async function loadDb(onData: (percentage: number) => void, fileName: string, useOPFS: boolean, statements: string[]) {
if (!db) { if (!db) {
await init(fileName, useOPFS); await init(fileName, parseWorkerOrURL(defaultWasmURL, !useOPFS) as string, useOPFS);
} }
const length = statements.length; const length = statements.length;
let percentage = 0; let percentage = 0;
for (let i = 0; i < length; i++) { for (let i = 0; i < length; i += 1000) {
const newPercentage = Math.round((i / length) * 100); const newPercentage = Math.round((i / length) * 100);
if (newPercentage !== percentage) { if (newPercentage !== percentage) {
@ -75,8 +57,10 @@ async function loadDb(onData: (percentage: number) => void, fileName: string, us
console.log("executing statement"); console.log("executing statement");
db.exec(statements[i]); await db.run(statements.slice(i, i + 1000).join(";"));
} }
// await db.run(statements.join(";"));
} }
/** /**
@ -92,7 +76,7 @@ async function loadDb(onData: (percentage: number) => void, fileName: string, us
* ) * )
*/ */
export function createOnMessageCallback( export function createOnMessageCallback(
afterInit?: (sqliteDB: Database | OpfsDatabase) => Promise<void>, afterInit?: (sqliteDB: SQLiteDB) => Promise<void>,
): (event: MessageEvent<MainToWorkerMsg>) => Promise<void> { ): (event: MessageEvent<MainToWorkerMsg>) => Promise<void> {
return async ({ data: [msg, data1, data2, data3] }: MessageEvent<MainToWorkerMsg>) => { return async ({ data: [msg, data1, data2, data3] }: MessageEvent<MainToWorkerMsg>) => {
const ret: WorkerToMainMsg = [msg, null, null]; const ret: WorkerToMainMsg = [msg, null, null];
@ -100,16 +84,16 @@ export function createOnMessageCallback(
try { try {
switch (msg) { switch (msg) {
case 0: case 0:
await init(data1, data2, afterInit); await init(data1, data2, data3, afterInit);
break; break;
case 1: case 1:
ret[1] = exec(data1, data2, data3); ret[1] = await exec(data1, data2, data3);
break; break;
case 2: case 2:
db.close(); await db.close();
break; break;
case 3: case 3:
stream((val) => postMessage([3, [val], null] satisfies WorkerToMainMsg), data1, data2); await stream((val) => postMessage([3, [val], null] satisfies WorkerToMainMsg), data1, data2);
ret[0] = 4; ret[0] = 4;
break; break;
case 4: case 4: