feat(db): use wa-sqlite

This commit is contained in:
Samuel 2025-01-20 17:51:56 +01:00
parent 877fd5a806
commit 36ff7afa4a
14 changed files with 232 additions and 177 deletions

View file

@ -8,7 +8,8 @@
"dev": "vite",
"build": "vite build",
"serve": "vite preview",
"prepare": "husky"
"prepare": "husky",
"postinstall": "cp ./node_modules/@subframe7536/sqlite-wasm/dist/*.wasm ./src/assets/"
},
"license": "MIT",
"devDependencies": {
@ -40,6 +41,7 @@
"@solidjs/meta": "^0.29.4",
"@solidjs/router": "^0.15.3",
"@sqlite.org/sqlite-wasm": "3.48.0-build2",
"@subframe7536/sqlite-wasm": "^0.5.1",
"@tanstack/solid-table": "^8.20.5",
"chart.js": "^4.4.7",
"chartjs-chart-wordcloud": "^4.4.4",
@ -58,9 +60,6 @@
"zen-mitt": "^3.0.0"
},
"lint-staged": {
"*.{ts,tsx}": [
"biome lint",
"biome format"
]
"*.{ts,tsx}": ["biome lint", "biome format"]
}
}

8
pnpm-lock.yaml generated
View file

@ -38,6 +38,9 @@ importers:
'@sqlite.org/sqlite-wasm':
specifier: 3.48.0-build2
version: 3.48.0-build2
'@subframe7536/sqlite-wasm':
specifier: ^0.5.1
version: 0.5.1
'@tanstack/solid-table':
specifier: ^8.20.5
version: 8.20.5(solid-js@1.9.4)
@ -760,6 +763,9 @@ packages:
resolution: {integrity: sha512-nltoBHBbLZmI3VioebwUYaSugTpVcHPvL9rYa0uSkqmiLF0b9ZEM8l9NzoWAHlS6qTMqhGHFtX1lWJ/egyjohQ==}
hasBin: true
'@subframe7536/sqlite-wasm@0.5.1':
resolution: {integrity: sha512-rRszpnvcT045Jd5HkDMIxaYI95BUl8H07mhYmklNW1CbNjJtiD3dN550lRXElwcDUpTQrdhIdmA9A5QwJ+ct+A==}
'@swc/helpers@0.5.15':
resolution: {integrity: sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g==}
@ -2698,6 +2704,8 @@ snapshots:
'@sqlite.org/sqlite-wasm@3.48.0-build2': {}
'@subframe7536/sqlite-wasm@0.5.1': {}
'@swc/helpers@0.5.15':
dependencies:
tslib: 2.8.1

BIN
src/assets/wa-sqlite-async.wasm Executable file

Binary file not shown.

BIN
src/assets/wa-sqlite.wasm Executable file

Binary file not shown.

View file

@ -1,7 +1,7 @@
import { sql, type NotNull } from "kysely";
import { worker, kyselyDb, SELF_ID, DB_FILENAME } from "./db";
import { cached } from "../lib/db-cache";
import type { MainToWorkerMsg, WorkerToMainMsg } from "~/lib/kysely-official-wasm-worker/type";
import type { MainToWorkerMsg, WorkerToMainMsg } from "~/lib/kysely-wasqlite-worker/type";
export const loadDb = (statements: string[], progressCallback?: (percentage: number) => void): Promise<void> => {
return new Promise((resolve, reject) => {
@ -23,8 +23,8 @@ export const loadDb = (statements: string[], progressCallback?: (percentage: num
}
};
worker.addEventListener("message", endListener);
worker.addEventListener("message", progressListener);
worker.addEventListener("message", endListener);
worker.postMessage([4, DB_FILENAME, true, statements] satisfies MainToWorkerMsg);
});

View file

@ -2,8 +2,8 @@ import { makePersisted } from "@solid-primitives/storage";
import { Kysely } from "kysely";
import type { DB } from "./db-schema";
import { createSignal } from "solid-js";
import { OfficialWasmWorkerDialect } from "~/lib/kysely-official-wasm-worker";
import wasmWorkerUrl from "~/lib/kysely-official-wasm-worker/worker?url";
import { WaSqliteWorkerDialect } from "~/lib/kysely-wasqlite-worker";
import wasmWorkerUrl from "~/lib/kysely-wasqlite-worker/worker?url";
export const SELF_ID = 2;
@ -13,7 +13,7 @@ export const worker = new Worker(wasmWorkerUrl, {
type: "module",
});
const dialect = new OfficialWasmWorkerDialect({
const dialect = new WaSqliteWorkerDialect({
fileName: DB_FILENAME,
preferOPFS: true,
worker,

View file

@ -1,38 +0,0 @@
import type {
DatabaseIntrospector,
Dialect,
DialectAdapter,
Driver,
Kysely,
QueryCompiler,
} from "kysely";
import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely";
import { OfficialWasmWorkerDriver } from "./driver";
import type { OfficialWasmWorkerDialectConfig } from "./type";
export type {
Promisable,
OfficialWasmWorkerDialectConfig as WaSqliteWorkerDialectConfig,
} from "./type";
export { createOnMessageCallback } from "./worker/utils";
export class OfficialWasmWorkerDialect implements Dialect {
constructor(private config: OfficialWasmWorkerDialectConfig) {}
createDriver(): Driver {
return new OfficialWasmWorkerDriver(this.config);
}
createQueryCompiler(): QueryCompiler {
return new SqliteQueryCompiler();
}
createAdapter(): DialectAdapter {
return new SqliteAdapter();
}
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
createIntrospector(db: Kysely<any>): DatabaseIntrospector {
return new SqliteIntrospector(db);
}
}

View file

@ -1,54 +0,0 @@
import type { SqlValue } from "@sqlite.org/sqlite-wasm";
import type { DatabaseConnection, QueryResult } from "kysely";
export type Promisable<T> = T | Promise<T>;
export interface OfficialWasmWorkerDialectConfig {
/**
* db file name
*/
fileName: string;
/**
* prefer to store data in OPFS
* @default true
*/
preferOPFS?: boolean;
/**
* official wasm worker
*/
worker?: Worker;
onCreateConnection?: (connection: DatabaseConnection) => Promisable<void>;
}
type InitMsg = [type: 0, fileName: string, useOPFS: boolean];
type RunMsg = [type: 1, isSelect: boolean, sql: string, parameters?: readonly unknown[]];
type CloseMsg = [2];
type StreamMsg = [type: 3, sql: string, parameters?: readonly unknown[]];
type LoadDbMsg = [type: 4, filename: string, useOPFS: boolean, statements: string[]];
export type MainToWorkerMsg = InitMsg | RunMsg | CloseMsg | StreamMsg | LoadDbMsg;
type Events = {
0: null;
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
1: QueryResult<any> | null;
2: null;
3: {
[columnName: string]: SqlValue;
}[];
4: null;
5: number;
6: null;
};
export type WorkerToMainMsg = {
[K in keyof Events]: [type: K, data: Events[K], err: unknown];
}[keyof Events];
export type EventWithError = {
[K in keyof Events]: [data: Events[K], err: unknown];
};

View file

@ -1,45 +1,52 @@
import type { DatabaseConnection, Driver, QueryResult } from "kysely";
import { CompiledQuery, SelectQueryNode } from "kysely";
import type { Emitter } from "zen-mitt";
import type { EventWithError, MainToWorkerMsg, WaSqliteWorkerDialectConfig, WorkerToMainMsg } from "./type";
import { isModuleWorkerSupport, isOpfsSupported } from "@subframe7536/sqlite-wasm";
import { CompiledQuery, SelectQueryNode } from "kysely";
import { mitt } from "zen-mitt";
import type { EventWithError, MainToWorkerMsg, OfficialWasmWorkerDialectConfig, WorkerToMainMsg } from "./type";
import workerUrl from "./worker?url";
import { defaultWasmURL, defaultWorker, parseWorkerOrURL } from "./utils";
export class OfficialWasmWorkerDriver implements Driver {
export class WaSqliteWorkerDriver implements Driver {
private worker?: Worker;
private connection?: DatabaseConnection;
private connectionMutex = new ConnectionMutex();
private mitt?: Emitter<EventWithError>;
constructor(private config: OfficialWasmWorkerDialectConfig) {}
constructor(private config: WaSqliteWorkerDialectConfig) {}
async init(): Promise<void> {
// try to persist storage, https://web.dev/articles/persistent-storage#request_persistent_storage
try {
if (navigator.storage?.persist && !(await navigator.storage.persisted())) {
if (!(await navigator.storage.persisted())) {
await navigator.storage.persist();
}
// biome-ignore lint/suspicious/noEmptyBlockStatements: <explanation>
} catch {}
const useOPFS = (this.config.preferOPFS ?? true) ? await isOpfsSupported() : false;
this.mitt = mitt<EventWithError>();
this.worker =
this.config.worker ??
new Worker(workerUrl, {
type: "module",
});
this.worker = parseWorkerOrURL(this.config.worker || defaultWorker, useOPFS || isModuleWorkerSupport());
this.worker.onmessage = ({ data: [type, ...msg] }: MessageEvent<WorkerToMainMsg>) => {
// biome-ignore lint/style/noNonNullAssertion: <explanation>
this.worker!.onmessage = ({ data: [type, ...msg] }: MessageEvent<WorkerToMainMsg>) => {
this.mitt?.emit(type, ...msg);
};
this.worker.postMessage([0, this.config.fileName, this.config.preferOPFS ?? false] satisfies MainToWorkerMsg);
this.worker?.postMessage([
0,
this.config.fileName,
// if use OPFS, wasm should use sync version
parseWorkerOrURL(this.config.url ?? defaultWasmURL, !useOPFS) as string,
useOPFS,
] satisfies MainToWorkerMsg);
await new Promise<void>((resolve, reject) => {
this.mitt?.once(0, (_, err) => (err ? reject(err) : resolve()));
});
this.connection = new OfficialWasmWorkerConnection(this.worker, this.mitt);
// biome-ignore lint/style/noNonNullAssertion: <explanation>
this.connection = new WaSqliteWorkerConnection(this.worker!, this.mitt);
await this.config.onCreateConnection?.(this.connection);
}
@ -47,7 +54,6 @@ export class OfficialWasmWorkerDriver implements Driver {
// SQLite only has one single connection. We use a mutex here to wait
// until the single connection has been released.
await this.connectionMutex.lock();
// biome-ignore lint/style/noNonNullAssertion: <explanation>
return this.connection!;
}
@ -64,12 +70,9 @@ export class OfficialWasmWorkerDriver implements Driver {
await connection.executeQuery(CompiledQuery.raw("rollback"));
}
releaseConnection(): Promise<void> {
return new Promise((resolve) => {
// biome-ignore lint/suspicious/useAwait: <explanation>
async releaseConnection(): Promise<void> {
this.connectionMutex.unlock();
resolve();
});
}
async destroy(): Promise<void> {
@ -116,7 +119,7 @@ class ConnectionMutex {
}
}
class OfficialWasmWorkerConnection implements DatabaseConnection {
class WaSqliteWorkerConnection implements DatabaseConnection {
readonly worker: Worker;
readonly mitt?: Emitter<EventWithError>;
constructor(worker: Worker, mitt?: Emitter<EventWithError>) {
@ -127,18 +130,20 @@ class OfficialWasmWorkerConnection implements DatabaseConnection {
async *streamQuery<R>(compiledQuery: CompiledQuery): AsyncIterableIterator<QueryResult<R>> {
const { parameters, sql, query } = compiledQuery;
if (!SelectQueryNode.is(query)) {
throw new Error("official wasm worker dialect only supports SELECT queries for streaming");
throw new Error("WaSqlite dialect only supported SELECT queries");
}
this.worker.postMessage([3, sql, parameters] satisfies MainToWorkerMsg);
let done = false;
let resolveFn: (value: IteratorResult<QueryResult<R>>) => void;
let rejectFn: (reason?: unknown) => void;
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
let rejectFn: (reason?: any) => void;
this.mitt?.on(3 /* data */, (data, err): void => {
if (err) {
rejectFn(err);
} else {
resolveFn({ value: { rows: data as R[] }, done: false });
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
resolveFn({ value: { rows: data as any }, done: false });
}
});
@ -167,10 +172,8 @@ class OfficialWasmWorkerConnection implements DatabaseConnection {
}
async executeQuery<R>(compiledQuery: CompiledQuery<unknown>): Promise<QueryResult<R>> {
const { sql, parameters, query } = compiledQuery;
const { parameters, sql, query } = compiledQuery;
const isSelect = SelectQueryNode.is(query);
this.worker.postMessage([1, isSelect, sql, parameters] satisfies MainToWorkerMsg);
return new Promise((resolve, reject) => {
if (!this.mitt) {

View file

@ -0,0 +1,48 @@
import type { DatabaseIntrospector, Dialect, DialectAdapter, Driver, Kysely, QueryCompiler } from "kysely";
import type { WaSqliteWorkerDialectConfig } from "./type";
import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely";
import { WaSqliteWorkerDriver } from "./driver";
export type { Promisable, WaSqliteWorkerDialectConfig } from "./type";
export { createOnMessageCallback } from "./worker/utils";
export {
customFunction,
isIdbSupported,
isModuleWorkerSupport,
isOpfsSupported,
type SQLiteDB,
} from "@subframe7536/sqlite-wasm";
export class WaSqliteWorkerDialect implements Dialect {
/**
* dialect for [`wa-sqlite`](https://github.com/rhashimoto/wa-sqlite),
* execute sql in `Web Worker`,
* store data in [OPFS](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system) or IndexedDB
*
* @example
* import { WaSqliteWorkerDialect } from 'kysely-wasqlite-worker'
*
* const dialect = new WaSqliteWorkerDialect({
* fileName: 'test',
* })
*/
constructor(private config: WaSqliteWorkerDialectConfig) {}
createDriver(): Driver {
return new WaSqliteWorkerDriver(this.config);
}
createQueryCompiler(): QueryCompiler {
return new SqliteQueryCompiler();
}
createAdapter(): DialectAdapter {
return new SqliteAdapter();
}
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
createIntrospector(db: Kysely<any>): DatabaseIntrospector {
return new SqliteIntrospector(db);
}
}

View file

@ -0,0 +1,81 @@
import type { SqlValue } from "@sqlite.org/sqlite-wasm";
import type { DatabaseConnection, QueryResult } from "kysely";
export type Promisable<T> = T | Promise<T>;
export interface WaSqliteWorkerDialectConfig {
/**
* db file name
*/
fileName: string;
/**
* prefer to store data in OPFS
* @default true
*/
preferOPFS?: boolean;
/**
* wasqlite worker
*
* built-in: {@link useDefaultWorker}
* @param supportModuleWorker if support `{ type: 'module' }` in worker options
* @example
* import { useDefaultWorker } from 'kysely-wasqlite-worker'
* @example
* (supportModuleWorker) => supportModuleWorker
* ? new Worker(
* new URL('kysely-wasqlite-worker/worker-module', import.meta.url),
* { type: 'module', credentials: 'same-origin' }
* )
* : new Worker(
* new URL('kysely-wasqlite-worker/worker-classic', import.meta.url),
* { type: 'classic', name: 'test' }
* )
*/
worker?: Worker | ((supportModuleWorker: boolean) => Worker);
/**
* wasm URL
*
* built-in: {@link useDefaultWasmURL}
* @param useAsyncWasm if need to use wa-sqlite-async.wasm
* @example
* import { useDefaultWasmURL } from 'kysely-wasqlite-worker'
* @example
* (useAsyncWasm) => useAsyncWasm
* ? 'https://cdn.jsdelivr.net/gh/rhashimoto/wa-sqlite@v1.0.0/dist/wa-sqlite-async.wasm'
* : new URL('kysely-wasqlite-worker/wasm-sync', import.meta.url).href
*/
url?: string | ((useAsyncWasm: boolean) => string);
onCreateConnection?: (connection: DatabaseConnection) => Promisable<void>;
}
type RunMsg = [type: 1, isSelect: boolean, sql: string, parameters?: readonly unknown[]];
type StreamMsg = [type: 3, sql: string, parameters?: readonly unknown[]];
type InitMsg = [type: 0, url: string, fileName: string, useOPFS: boolean];
type CloseMsg = [2];
type LoadDbMsg = [type: 4, filename: string, useOPFS: boolean, statements: string[]];
export type MainToWorkerMsg = InitMsg | RunMsg | CloseMsg | StreamMsg | LoadDbMsg;
export type WorkerToMainMsg = {
[K in keyof Events]: [type: K, data: Events[K], err: unknown];
}[keyof Events];
export type EventWithError = {
[K in keyof Events]: [data: Events[K], err: unknown];
};
type Events = {
0: null;
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
1: QueryResult<any> | null;
2: null;
3: {
[columnName: string]: SqlValue;
}[];
4: null;
5: number;
6: null;
};

View file

@ -0,0 +1,24 @@
import asyncWasmUrl from "~/assets/wa-sqlite-async.wasm?url";
import syncWasmUrl from "~/assets/wa-sqlite.wasm?url";
export function parseWorkerOrURL<T, P extends T | ((is: boolean) => T)>(obj: P, is: boolean): T {
return typeof obj === "function" ? obj(is) : obj;
}
/**
* auto load target worker
*
* **only basic worker options**
*/
export function defaultWorker(support: boolean): Worker {
return support
? new Worker(new URL("worker.mjs", import.meta.url), { type: "module" })
: new Worker(new URL("worker.js", import.meta.url));
}
/**
* auto load target wasm
*/
export function defaultWasmURL(useAsyncWasm: boolean): string {
return useAsyncWasm ? asyncWasmUrl : syncWasmUrl;
}

View file

@ -1,70 +1,52 @@
import sqlite3InitModule, {
type BindingSpec,
type Database,
type OpfsDatabase,
type Sqlite3Static,
type SqlValue,
} from "@sqlite.org/sqlite-wasm";
import type { SQLiteDB } from "@subframe7536/sqlite-wasm";
import type { QueryResult } from "kysely";
import type { MainToWorkerMsg, WorkerToMainMsg } from "../type";
import { initSQLite } from "@subframe7536/sqlite-wasm";
import { defaultWasmURL, parseWorkerOrURL } from "../utils";
let sqlite3: Sqlite3Static;
let db: Database | OpfsDatabase;
let db: SQLiteDB;
async function init(
fileName: string,
preferOpfs: boolean,
afterInit?: (sqliteDB: Database | OpfsDatabase) => Promise<void>,
url: string,
useOPFS: boolean,
afterInit?: (sqliteDB: SQLiteDB) => Promise<void>,
): Promise<void> {
sqlite3 = await sqlite3InitModule();
db = preferOpfs && "opfs" in sqlite3.oo1 ? new sqlite3.oo1.OpfsDb(fileName) : new sqlite3.oo1.DB(fileName);
db = await initSQLite(
(useOPFS
? (await import("@subframe7536/sqlite-wasm/opfs")).useOpfsStorage
: (await import("@subframe7536/sqlite-wasm/idb")).useIdbStorage)(fileName, { url }),
);
await afterInit?.(db);
}
function exec(isSelect: boolean, sql: string, parameters?: readonly unknown[]): QueryResult<SqlValue[]> {
const rows = db.exec(sql, {
bind: parameters as BindingSpec,
returnValue: "resultRows",
});
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
async function exec(isSelect: boolean, sql: string, parameters?: readonly unknown[]): Promise<QueryResult<any>> {
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
const rows = await db.run(sql, parameters as any[]);
return isSelect || rows.length
? { rows }
: {
rows,
insertId: BigInt(sqlite3.capi.sqlite3_last_insert_rowid(db)),
insertId: BigInt(db.lastInsertRowId()),
numAffectedRows: BigInt(db.changes()),
};
}
function stream(
onData: (data: { [columnName: string]: SqlValue }) => void,
sql: string,
parameters?: readonly unknown[],
): void {
const stmt = db.prepare(sql);
if (parameters) {
stmt.bind(parameters as BindingSpec);
}
while (stmt.step()) {
onData(stmt.get({}));
}
stmt.finalize();
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
async function stream(onData: (data: any) => void, sql: string, parameters?: readonly unknown[]): Promise<void> {
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
await db.stream(onData, sql, parameters as any[]);
}
async function loadDb(onData: (percentage: number) => void, fileName: string, useOPFS: boolean, statements: string[]) {
if (!db) {
await init(fileName, useOPFS);
await init(fileName, parseWorkerOrURL(defaultWasmURL, !useOPFS) as string, useOPFS);
}
const length = statements.length;
let percentage = 0;
for (let i = 0; i < length; i++) {
for (let i = 0; i < length; i += 1000) {
const newPercentage = Math.round((i / length) * 100);
if (newPercentage !== percentage) {
@ -75,8 +57,10 @@ async function loadDb(onData: (percentage: number) => void, fileName: string, us
console.log("executing statement");
db.exec(statements[i]);
await db.run(statements.slice(i, i + 1000).join(";"));
}
// await db.run(statements.join(";"));
}
/**
@ -92,7 +76,7 @@ async function loadDb(onData: (percentage: number) => void, fileName: string, us
* )
*/
export function createOnMessageCallback(
afterInit?: (sqliteDB: Database | OpfsDatabase) => Promise<void>,
afterInit?: (sqliteDB: SQLiteDB) => Promise<void>,
): (event: MessageEvent<MainToWorkerMsg>) => Promise<void> {
return async ({ data: [msg, data1, data2, data3] }: MessageEvent<MainToWorkerMsg>) => {
const ret: WorkerToMainMsg = [msg, null, null];
@ -100,16 +84,16 @@ export function createOnMessageCallback(
try {
switch (msg) {
case 0:
await init(data1, data2, afterInit);
await init(data1, data2, data3, afterInit);
break;
case 1:
ret[1] = exec(data1, data2, data3);
ret[1] = await exec(data1, data2, data3);
break;
case 2:
db.close();
await db.close();
break;
case 3:
stream((val) => postMessage([3, [val], null] satisfies WorkerToMainMsg), data1, data2);
await stream((val) => postMessage([3, [val], null] satisfies WorkerToMainMsg), data1, data2);
ret[0] = 4;
break;
case 4: