feat(db): use wa-sqlite
This commit is contained in:
parent
877fd5a806
commit
36ff7afa4a
14 changed files with 232 additions and 177 deletions
|
@ -8,7 +8,8 @@
|
||||||
"dev": "vite",
|
"dev": "vite",
|
||||||
"build": "vite build",
|
"build": "vite build",
|
||||||
"serve": "vite preview",
|
"serve": "vite preview",
|
||||||
"prepare": "husky"
|
"prepare": "husky",
|
||||||
|
"postinstall": "cp ./node_modules/@subframe7536/sqlite-wasm/dist/*.wasm ./src/assets/"
|
||||||
},
|
},
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
@ -40,6 +41,7 @@
|
||||||
"@solidjs/meta": "^0.29.4",
|
"@solidjs/meta": "^0.29.4",
|
||||||
"@solidjs/router": "^0.15.3",
|
"@solidjs/router": "^0.15.3",
|
||||||
"@sqlite.org/sqlite-wasm": "3.48.0-build2",
|
"@sqlite.org/sqlite-wasm": "3.48.0-build2",
|
||||||
|
"@subframe7536/sqlite-wasm": "^0.5.1",
|
||||||
"@tanstack/solid-table": "^8.20.5",
|
"@tanstack/solid-table": "^8.20.5",
|
||||||
"chart.js": "^4.4.7",
|
"chart.js": "^4.4.7",
|
||||||
"chartjs-chart-wordcloud": "^4.4.4",
|
"chartjs-chart-wordcloud": "^4.4.4",
|
||||||
|
@ -58,9 +60,6 @@
|
||||||
"zen-mitt": "^3.0.0"
|
"zen-mitt": "^3.0.0"
|
||||||
},
|
},
|
||||||
"lint-staged": {
|
"lint-staged": {
|
||||||
"*.{ts,tsx}": [
|
"*.{ts,tsx}": ["biome lint", "biome format"]
|
||||||
"biome lint",
|
|
||||||
"biome format"
|
|
||||||
]
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
8
pnpm-lock.yaml
generated
8
pnpm-lock.yaml
generated
|
@ -38,6 +38,9 @@ importers:
|
||||||
'@sqlite.org/sqlite-wasm':
|
'@sqlite.org/sqlite-wasm':
|
||||||
specifier: 3.48.0-build2
|
specifier: 3.48.0-build2
|
||||||
version: 3.48.0-build2
|
version: 3.48.0-build2
|
||||||
|
'@subframe7536/sqlite-wasm':
|
||||||
|
specifier: ^0.5.1
|
||||||
|
version: 0.5.1
|
||||||
'@tanstack/solid-table':
|
'@tanstack/solid-table':
|
||||||
specifier: ^8.20.5
|
specifier: ^8.20.5
|
||||||
version: 8.20.5(solid-js@1.9.4)
|
version: 8.20.5(solid-js@1.9.4)
|
||||||
|
@ -760,6 +763,9 @@ packages:
|
||||||
resolution: {integrity: sha512-nltoBHBbLZmI3VioebwUYaSugTpVcHPvL9rYa0uSkqmiLF0b9ZEM8l9NzoWAHlS6qTMqhGHFtX1lWJ/egyjohQ==}
|
resolution: {integrity: sha512-nltoBHBbLZmI3VioebwUYaSugTpVcHPvL9rYa0uSkqmiLF0b9ZEM8l9NzoWAHlS6qTMqhGHFtX1lWJ/egyjohQ==}
|
||||||
hasBin: true
|
hasBin: true
|
||||||
|
|
||||||
|
'@subframe7536/sqlite-wasm@0.5.1':
|
||||||
|
resolution: {integrity: sha512-rRszpnvcT045Jd5HkDMIxaYI95BUl8H07mhYmklNW1CbNjJtiD3dN550lRXElwcDUpTQrdhIdmA9A5QwJ+ct+A==}
|
||||||
|
|
||||||
'@swc/helpers@0.5.15':
|
'@swc/helpers@0.5.15':
|
||||||
resolution: {integrity: sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g==}
|
resolution: {integrity: sha512-JQ5TuMi45Owi4/BIMAJBoSQoOJu12oOk/gADqlcUL9JEdHB8vyjUSsxqeNXnmXHjYKMi2WcYtezGEEhqUI/E2g==}
|
||||||
|
|
||||||
|
@ -2698,6 +2704,8 @@ snapshots:
|
||||||
|
|
||||||
'@sqlite.org/sqlite-wasm@3.48.0-build2': {}
|
'@sqlite.org/sqlite-wasm@3.48.0-build2': {}
|
||||||
|
|
||||||
|
'@subframe7536/sqlite-wasm@0.5.1': {}
|
||||||
|
|
||||||
'@swc/helpers@0.5.15':
|
'@swc/helpers@0.5.15':
|
||||||
dependencies:
|
dependencies:
|
||||||
tslib: 2.8.1
|
tslib: 2.8.1
|
||||||
|
|
BIN
src/assets/wa-sqlite-async.wasm
Executable file
BIN
src/assets/wa-sqlite-async.wasm
Executable file
Binary file not shown.
BIN
src/assets/wa-sqlite.wasm
Executable file
BIN
src/assets/wa-sqlite.wasm
Executable file
Binary file not shown.
|
@ -1,7 +1,7 @@
|
||||||
import { sql, type NotNull } from "kysely";
|
import { sql, type NotNull } from "kysely";
|
||||||
import { worker, kyselyDb, SELF_ID, DB_FILENAME } from "./db";
|
import { worker, kyselyDb, SELF_ID, DB_FILENAME } from "./db";
|
||||||
import { cached } from "../lib/db-cache";
|
import { cached } from "../lib/db-cache";
|
||||||
import type { MainToWorkerMsg, WorkerToMainMsg } from "~/lib/kysely-official-wasm-worker/type";
|
import type { MainToWorkerMsg, WorkerToMainMsg } from "~/lib/kysely-wasqlite-worker/type";
|
||||||
|
|
||||||
export const loadDb = (statements: string[], progressCallback?: (percentage: number) => void): Promise<void> => {
|
export const loadDb = (statements: string[], progressCallback?: (percentage: number) => void): Promise<void> => {
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
|
@ -23,8 +23,8 @@ export const loadDb = (statements: string[], progressCallback?: (percentage: num
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
worker.addEventListener("message", endListener);
|
|
||||||
worker.addEventListener("message", progressListener);
|
worker.addEventListener("message", progressListener);
|
||||||
|
worker.addEventListener("message", endListener);
|
||||||
|
|
||||||
worker.postMessage([4, DB_FILENAME, true, statements] satisfies MainToWorkerMsg);
|
worker.postMessage([4, DB_FILENAME, true, statements] satisfies MainToWorkerMsg);
|
||||||
});
|
});
|
||||||
|
|
|
@ -2,8 +2,8 @@ import { makePersisted } from "@solid-primitives/storage";
|
||||||
import { Kysely } from "kysely";
|
import { Kysely } from "kysely";
|
||||||
import type { DB } from "./db-schema";
|
import type { DB } from "./db-schema";
|
||||||
import { createSignal } from "solid-js";
|
import { createSignal } from "solid-js";
|
||||||
import { OfficialWasmWorkerDialect } from "~/lib/kysely-official-wasm-worker";
|
import { WaSqliteWorkerDialect } from "~/lib/kysely-wasqlite-worker";
|
||||||
import wasmWorkerUrl from "~/lib/kysely-official-wasm-worker/worker?url";
|
import wasmWorkerUrl from "~/lib/kysely-wasqlite-worker/worker?url";
|
||||||
|
|
||||||
export const SELF_ID = 2;
|
export const SELF_ID = 2;
|
||||||
|
|
||||||
|
@ -13,7 +13,7 @@ export const worker = new Worker(wasmWorkerUrl, {
|
||||||
type: "module",
|
type: "module",
|
||||||
});
|
});
|
||||||
|
|
||||||
const dialect = new OfficialWasmWorkerDialect({
|
const dialect = new WaSqliteWorkerDialect({
|
||||||
fileName: DB_FILENAME,
|
fileName: DB_FILENAME,
|
||||||
preferOPFS: true,
|
preferOPFS: true,
|
||||||
worker,
|
worker,
|
||||||
|
|
|
@ -1,38 +0,0 @@
|
||||||
import type {
|
|
||||||
DatabaseIntrospector,
|
|
||||||
Dialect,
|
|
||||||
DialectAdapter,
|
|
||||||
Driver,
|
|
||||||
Kysely,
|
|
||||||
QueryCompiler,
|
|
||||||
} from "kysely";
|
|
||||||
import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely";
|
|
||||||
import { OfficialWasmWorkerDriver } from "./driver";
|
|
||||||
import type { OfficialWasmWorkerDialectConfig } from "./type";
|
|
||||||
|
|
||||||
export type {
|
|
||||||
Promisable,
|
|
||||||
OfficialWasmWorkerDialectConfig as WaSqliteWorkerDialectConfig,
|
|
||||||
} from "./type";
|
|
||||||
export { createOnMessageCallback } from "./worker/utils";
|
|
||||||
|
|
||||||
export class OfficialWasmWorkerDialect implements Dialect {
|
|
||||||
constructor(private config: OfficialWasmWorkerDialectConfig) {}
|
|
||||||
|
|
||||||
createDriver(): Driver {
|
|
||||||
return new OfficialWasmWorkerDriver(this.config);
|
|
||||||
}
|
|
||||||
|
|
||||||
createQueryCompiler(): QueryCompiler {
|
|
||||||
return new SqliteQueryCompiler();
|
|
||||||
}
|
|
||||||
|
|
||||||
createAdapter(): DialectAdapter {
|
|
||||||
return new SqliteAdapter();
|
|
||||||
}
|
|
||||||
|
|
||||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
|
||||||
createIntrospector(db: Kysely<any>): DatabaseIntrospector {
|
|
||||||
return new SqliteIntrospector(db);
|
|
||||||
}
|
|
||||||
}
|
|
|
@ -1,54 +0,0 @@
|
||||||
import type { SqlValue } from "@sqlite.org/sqlite-wasm";
|
|
||||||
import type { DatabaseConnection, QueryResult } from "kysely";
|
|
||||||
|
|
||||||
export type Promisable<T> = T | Promise<T>;
|
|
||||||
|
|
||||||
export interface OfficialWasmWorkerDialectConfig {
|
|
||||||
/**
|
|
||||||
* db file name
|
|
||||||
*/
|
|
||||||
fileName: string;
|
|
||||||
/**
|
|
||||||
* prefer to store data in OPFS
|
|
||||||
* @default true
|
|
||||||
*/
|
|
||||||
preferOPFS?: boolean;
|
|
||||||
/**
|
|
||||||
* official wasm worker
|
|
||||||
*/
|
|
||||||
worker?: Worker;
|
|
||||||
onCreateConnection?: (connection: DatabaseConnection) => Promisable<void>;
|
|
||||||
}
|
|
||||||
|
|
||||||
type InitMsg = [type: 0, fileName: string, useOPFS: boolean];
|
|
||||||
|
|
||||||
type RunMsg = [type: 1, isSelect: boolean, sql: string, parameters?: readonly unknown[]];
|
|
||||||
|
|
||||||
type CloseMsg = [2];
|
|
||||||
|
|
||||||
type StreamMsg = [type: 3, sql: string, parameters?: readonly unknown[]];
|
|
||||||
|
|
||||||
type LoadDbMsg = [type: 4, filename: string, useOPFS: boolean, statements: string[]];
|
|
||||||
|
|
||||||
export type MainToWorkerMsg = InitMsg | RunMsg | CloseMsg | StreamMsg | LoadDbMsg;
|
|
||||||
|
|
||||||
type Events = {
|
|
||||||
0: null;
|
|
||||||
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
|
||||||
1: QueryResult<any> | null;
|
|
||||||
2: null;
|
|
||||||
3: {
|
|
||||||
[columnName: string]: SqlValue;
|
|
||||||
}[];
|
|
||||||
4: null;
|
|
||||||
5: number;
|
|
||||||
6: null;
|
|
||||||
};
|
|
||||||
|
|
||||||
export type WorkerToMainMsg = {
|
|
||||||
[K in keyof Events]: [type: K, data: Events[K], err: unknown];
|
|
||||||
}[keyof Events];
|
|
||||||
|
|
||||||
export type EventWithError = {
|
|
||||||
[K in keyof Events]: [data: Events[K], err: unknown];
|
|
||||||
};
|
|
|
@ -1,45 +1,52 @@
|
||||||
import type { DatabaseConnection, Driver, QueryResult } from "kysely";
|
import type { DatabaseConnection, Driver, QueryResult } from "kysely";
|
||||||
import { CompiledQuery, SelectQueryNode } from "kysely";
|
|
||||||
import type { Emitter } from "zen-mitt";
|
import type { Emitter } from "zen-mitt";
|
||||||
|
import type { EventWithError, MainToWorkerMsg, WaSqliteWorkerDialectConfig, WorkerToMainMsg } from "./type";
|
||||||
|
import { isModuleWorkerSupport, isOpfsSupported } from "@subframe7536/sqlite-wasm";
|
||||||
|
import { CompiledQuery, SelectQueryNode } from "kysely";
|
||||||
import { mitt } from "zen-mitt";
|
import { mitt } from "zen-mitt";
|
||||||
import type { EventWithError, MainToWorkerMsg, OfficialWasmWorkerDialectConfig, WorkerToMainMsg } from "./type";
|
import { defaultWasmURL, defaultWorker, parseWorkerOrURL } from "./utils";
|
||||||
import workerUrl from "./worker?url";
|
|
||||||
|
|
||||||
export class OfficialWasmWorkerDriver implements Driver {
|
export class WaSqliteWorkerDriver implements Driver {
|
||||||
private worker?: Worker;
|
private worker?: Worker;
|
||||||
private connection?: DatabaseConnection;
|
private connection?: DatabaseConnection;
|
||||||
private connectionMutex = new ConnectionMutex();
|
private connectionMutex = new ConnectionMutex();
|
||||||
private mitt?: Emitter<EventWithError>;
|
private mitt?: Emitter<EventWithError>;
|
||||||
constructor(private config: OfficialWasmWorkerDialectConfig) {}
|
constructor(private config: WaSqliteWorkerDialectConfig) {}
|
||||||
|
|
||||||
async init(): Promise<void> {
|
async init(): Promise<void> {
|
||||||
// try to persist storage, https://web.dev/articles/persistent-storage#request_persistent_storage
|
// try to persist storage, https://web.dev/articles/persistent-storage#request_persistent_storage
|
||||||
try {
|
try {
|
||||||
if (navigator.storage?.persist && !(await navigator.storage.persisted())) {
|
if (!(await navigator.storage.persisted())) {
|
||||||
await navigator.storage.persist();
|
await navigator.storage.persist();
|
||||||
}
|
}
|
||||||
// biome-ignore lint/suspicious/noEmptyBlockStatements: <explanation>
|
// biome-ignore lint/suspicious/noEmptyBlockStatements: <explanation>
|
||||||
} catch {}
|
} catch {}
|
||||||
|
|
||||||
|
const useOPFS = (this.config.preferOPFS ?? true) ? await isOpfsSupported() : false;
|
||||||
|
|
||||||
this.mitt = mitt<EventWithError>();
|
this.mitt = mitt<EventWithError>();
|
||||||
|
|
||||||
this.worker =
|
this.worker = parseWorkerOrURL(this.config.worker || defaultWorker, useOPFS || isModuleWorkerSupport());
|
||||||
this.config.worker ??
|
|
||||||
new Worker(workerUrl, {
|
|
||||||
type: "module",
|
|
||||||
});
|
|
||||||
|
|
||||||
this.worker.onmessage = ({ data: [type, ...msg] }: MessageEvent<WorkerToMainMsg>) => {
|
// biome-ignore lint/style/noNonNullAssertion: <explanation>
|
||||||
|
this.worker!.onmessage = ({ data: [type, ...msg] }: MessageEvent<WorkerToMainMsg>) => {
|
||||||
this.mitt?.emit(type, ...msg);
|
this.mitt?.emit(type, ...msg);
|
||||||
};
|
};
|
||||||
|
|
||||||
this.worker.postMessage([0, this.config.fileName, this.config.preferOPFS ?? false] satisfies MainToWorkerMsg);
|
this.worker?.postMessage([
|
||||||
|
0,
|
||||||
|
this.config.fileName,
|
||||||
|
// if use OPFS, wasm should use sync version
|
||||||
|
parseWorkerOrURL(this.config.url ?? defaultWasmURL, !useOPFS) as string,
|
||||||
|
useOPFS,
|
||||||
|
] satisfies MainToWorkerMsg);
|
||||||
|
|
||||||
await new Promise<void>((resolve, reject) => {
|
await new Promise<void>((resolve, reject) => {
|
||||||
this.mitt?.once(0, (_, err) => (err ? reject(err) : resolve()));
|
this.mitt?.once(0, (_, err) => (err ? reject(err) : resolve()));
|
||||||
});
|
});
|
||||||
|
|
||||||
this.connection = new OfficialWasmWorkerConnection(this.worker, this.mitt);
|
// biome-ignore lint/style/noNonNullAssertion: <explanation>
|
||||||
|
this.connection = new WaSqliteWorkerConnection(this.worker!, this.mitt);
|
||||||
await this.config.onCreateConnection?.(this.connection);
|
await this.config.onCreateConnection?.(this.connection);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -47,7 +54,6 @@ export class OfficialWasmWorkerDriver implements Driver {
|
||||||
// SQLite only has one single connection. We use a mutex here to wait
|
// SQLite only has one single connection. We use a mutex here to wait
|
||||||
// until the single connection has been released.
|
// until the single connection has been released.
|
||||||
await this.connectionMutex.lock();
|
await this.connectionMutex.lock();
|
||||||
|
|
||||||
// biome-ignore lint/style/noNonNullAssertion: <explanation>
|
// biome-ignore lint/style/noNonNullAssertion: <explanation>
|
||||||
return this.connection!;
|
return this.connection!;
|
||||||
}
|
}
|
||||||
|
@ -64,12 +70,9 @@ export class OfficialWasmWorkerDriver implements Driver {
|
||||||
await connection.executeQuery(CompiledQuery.raw("rollback"));
|
await connection.executeQuery(CompiledQuery.raw("rollback"));
|
||||||
}
|
}
|
||||||
|
|
||||||
releaseConnection(): Promise<void> {
|
// biome-ignore lint/suspicious/useAwait: <explanation>
|
||||||
return new Promise((resolve) => {
|
async releaseConnection(): Promise<void> {
|
||||||
this.connectionMutex.unlock();
|
this.connectionMutex.unlock();
|
||||||
|
|
||||||
resolve();
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async destroy(): Promise<void> {
|
async destroy(): Promise<void> {
|
||||||
|
@ -116,7 +119,7 @@ class ConnectionMutex {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
class OfficialWasmWorkerConnection implements DatabaseConnection {
|
class WaSqliteWorkerConnection implements DatabaseConnection {
|
||||||
readonly worker: Worker;
|
readonly worker: Worker;
|
||||||
readonly mitt?: Emitter<EventWithError>;
|
readonly mitt?: Emitter<EventWithError>;
|
||||||
constructor(worker: Worker, mitt?: Emitter<EventWithError>) {
|
constructor(worker: Worker, mitt?: Emitter<EventWithError>) {
|
||||||
|
@ -127,18 +130,20 @@ class OfficialWasmWorkerConnection implements DatabaseConnection {
|
||||||
async *streamQuery<R>(compiledQuery: CompiledQuery): AsyncIterableIterator<QueryResult<R>> {
|
async *streamQuery<R>(compiledQuery: CompiledQuery): AsyncIterableIterator<QueryResult<R>> {
|
||||||
const { parameters, sql, query } = compiledQuery;
|
const { parameters, sql, query } = compiledQuery;
|
||||||
if (!SelectQueryNode.is(query)) {
|
if (!SelectQueryNode.is(query)) {
|
||||||
throw new Error("official wasm worker dialect only supports SELECT queries for streaming");
|
throw new Error("WaSqlite dialect only supported SELECT queries");
|
||||||
}
|
}
|
||||||
this.worker.postMessage([3, sql, parameters] satisfies MainToWorkerMsg);
|
this.worker.postMessage([3, sql, parameters] satisfies MainToWorkerMsg);
|
||||||
let done = false;
|
let done = false;
|
||||||
let resolveFn: (value: IteratorResult<QueryResult<R>>) => void;
|
let resolveFn: (value: IteratorResult<QueryResult<R>>) => void;
|
||||||
let rejectFn: (reason?: unknown) => void;
|
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||||
|
let rejectFn: (reason?: any) => void;
|
||||||
|
|
||||||
this.mitt?.on(3 /* data */, (data, err): void => {
|
this.mitt?.on(3 /* data */, (data, err): void => {
|
||||||
if (err) {
|
if (err) {
|
||||||
rejectFn(err);
|
rejectFn(err);
|
||||||
} else {
|
} else {
|
||||||
resolveFn({ value: { rows: data as R[] }, done: false });
|
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||||
|
resolveFn({ value: { rows: data as any }, done: false });
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
@ -167,10 +172,8 @@ class OfficialWasmWorkerConnection implements DatabaseConnection {
|
||||||
}
|
}
|
||||||
|
|
||||||
async executeQuery<R>(compiledQuery: CompiledQuery<unknown>): Promise<QueryResult<R>> {
|
async executeQuery<R>(compiledQuery: CompiledQuery<unknown>): Promise<QueryResult<R>> {
|
||||||
const { sql, parameters, query } = compiledQuery;
|
const { parameters, sql, query } = compiledQuery;
|
||||||
|
|
||||||
const isSelect = SelectQueryNode.is(query);
|
const isSelect = SelectQueryNode.is(query);
|
||||||
|
|
||||||
this.worker.postMessage([1, isSelect, sql, parameters] satisfies MainToWorkerMsg);
|
this.worker.postMessage([1, isSelect, sql, parameters] satisfies MainToWorkerMsg);
|
||||||
return new Promise((resolve, reject) => {
|
return new Promise((resolve, reject) => {
|
||||||
if (!this.mitt) {
|
if (!this.mitt) {
|
48
src/lib/kysely-wasqlite-worker/index.ts
Normal file
48
src/lib/kysely-wasqlite-worker/index.ts
Normal file
|
@ -0,0 +1,48 @@
|
||||||
|
import type { DatabaseIntrospector, Dialect, DialectAdapter, Driver, Kysely, QueryCompiler } from "kysely";
|
||||||
|
import type { WaSqliteWorkerDialectConfig } from "./type";
|
||||||
|
import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely";
|
||||||
|
import { WaSqliteWorkerDriver } from "./driver";
|
||||||
|
|
||||||
|
export type { Promisable, WaSqliteWorkerDialectConfig } from "./type";
|
||||||
|
export { createOnMessageCallback } from "./worker/utils";
|
||||||
|
|
||||||
|
export {
|
||||||
|
customFunction,
|
||||||
|
isIdbSupported,
|
||||||
|
isModuleWorkerSupport,
|
||||||
|
isOpfsSupported,
|
||||||
|
type SQLiteDB,
|
||||||
|
} from "@subframe7536/sqlite-wasm";
|
||||||
|
|
||||||
|
export class WaSqliteWorkerDialect implements Dialect {
|
||||||
|
/**
|
||||||
|
* dialect for [`wa-sqlite`](https://github.com/rhashimoto/wa-sqlite),
|
||||||
|
* execute sql in `Web Worker`,
|
||||||
|
* store data in [OPFS](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system) or IndexedDB
|
||||||
|
*
|
||||||
|
* @example
|
||||||
|
* import { WaSqliteWorkerDialect } from 'kysely-wasqlite-worker'
|
||||||
|
*
|
||||||
|
* const dialect = new WaSqliteWorkerDialect({
|
||||||
|
* fileName: 'test',
|
||||||
|
* })
|
||||||
|
*/
|
||||||
|
constructor(private config: WaSqliteWorkerDialectConfig) {}
|
||||||
|
|
||||||
|
createDriver(): Driver {
|
||||||
|
return new WaSqliteWorkerDriver(this.config);
|
||||||
|
}
|
||||||
|
|
||||||
|
createQueryCompiler(): QueryCompiler {
|
||||||
|
return new SqliteQueryCompiler();
|
||||||
|
}
|
||||||
|
|
||||||
|
createAdapter(): DialectAdapter {
|
||||||
|
return new SqliteAdapter();
|
||||||
|
}
|
||||||
|
|
||||||
|
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||||
|
createIntrospector(db: Kysely<any>): DatabaseIntrospector {
|
||||||
|
return new SqliteIntrospector(db);
|
||||||
|
}
|
||||||
|
}
|
81
src/lib/kysely-wasqlite-worker/type.ts
Normal file
81
src/lib/kysely-wasqlite-worker/type.ts
Normal file
|
@ -0,0 +1,81 @@
|
||||||
|
import type { SqlValue } from "@sqlite.org/sqlite-wasm";
|
||||||
|
import type { DatabaseConnection, QueryResult } from "kysely";
|
||||||
|
|
||||||
|
export type Promisable<T> = T | Promise<T>;
|
||||||
|
|
||||||
|
export interface WaSqliteWorkerDialectConfig {
|
||||||
|
/**
|
||||||
|
* db file name
|
||||||
|
*/
|
||||||
|
fileName: string;
|
||||||
|
/**
|
||||||
|
* prefer to store data in OPFS
|
||||||
|
* @default true
|
||||||
|
*/
|
||||||
|
preferOPFS?: boolean;
|
||||||
|
/**
|
||||||
|
* wasqlite worker
|
||||||
|
*
|
||||||
|
* built-in: {@link useDefaultWorker}
|
||||||
|
* @param supportModuleWorker if support `{ type: 'module' }` in worker options
|
||||||
|
* @example
|
||||||
|
* import { useDefaultWorker } from 'kysely-wasqlite-worker'
|
||||||
|
* @example
|
||||||
|
* (supportModuleWorker) => supportModuleWorker
|
||||||
|
* ? new Worker(
|
||||||
|
* new URL('kysely-wasqlite-worker/worker-module', import.meta.url),
|
||||||
|
* { type: 'module', credentials: 'same-origin' }
|
||||||
|
* )
|
||||||
|
* : new Worker(
|
||||||
|
* new URL('kysely-wasqlite-worker/worker-classic', import.meta.url),
|
||||||
|
* { type: 'classic', name: 'test' }
|
||||||
|
* )
|
||||||
|
*/
|
||||||
|
worker?: Worker | ((supportModuleWorker: boolean) => Worker);
|
||||||
|
/**
|
||||||
|
* wasm URL
|
||||||
|
*
|
||||||
|
* built-in: {@link useDefaultWasmURL}
|
||||||
|
* @param useAsyncWasm if need to use wa-sqlite-async.wasm
|
||||||
|
* @example
|
||||||
|
* import { useDefaultWasmURL } from 'kysely-wasqlite-worker'
|
||||||
|
* @example
|
||||||
|
* (useAsyncWasm) => useAsyncWasm
|
||||||
|
* ? 'https://cdn.jsdelivr.net/gh/rhashimoto/wa-sqlite@v1.0.0/dist/wa-sqlite-async.wasm'
|
||||||
|
* : new URL('kysely-wasqlite-worker/wasm-sync', import.meta.url).href
|
||||||
|
*/
|
||||||
|
url?: string | ((useAsyncWasm: boolean) => string);
|
||||||
|
onCreateConnection?: (connection: DatabaseConnection) => Promisable<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
type RunMsg = [type: 1, isSelect: boolean, sql: string, parameters?: readonly unknown[]];
|
||||||
|
type StreamMsg = [type: 3, sql: string, parameters?: readonly unknown[]];
|
||||||
|
|
||||||
|
type InitMsg = [type: 0, url: string, fileName: string, useOPFS: boolean];
|
||||||
|
|
||||||
|
type CloseMsg = [2];
|
||||||
|
|
||||||
|
type LoadDbMsg = [type: 4, filename: string, useOPFS: boolean, statements: string[]];
|
||||||
|
|
||||||
|
export type MainToWorkerMsg = InitMsg | RunMsg | CloseMsg | StreamMsg | LoadDbMsg;
|
||||||
|
|
||||||
|
export type WorkerToMainMsg = {
|
||||||
|
[K in keyof Events]: [type: K, data: Events[K], err: unknown];
|
||||||
|
}[keyof Events];
|
||||||
|
|
||||||
|
export type EventWithError = {
|
||||||
|
[K in keyof Events]: [data: Events[K], err: unknown];
|
||||||
|
};
|
||||||
|
|
||||||
|
type Events = {
|
||||||
|
0: null;
|
||||||
|
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||||
|
1: QueryResult<any> | null;
|
||||||
|
2: null;
|
||||||
|
3: {
|
||||||
|
[columnName: string]: SqlValue;
|
||||||
|
}[];
|
||||||
|
4: null;
|
||||||
|
5: number;
|
||||||
|
6: null;
|
||||||
|
};
|
24
src/lib/kysely-wasqlite-worker/utils.ts
Normal file
24
src/lib/kysely-wasqlite-worker/utils.ts
Normal file
|
@ -0,0 +1,24 @@
|
||||||
|
import asyncWasmUrl from "~/assets/wa-sqlite-async.wasm?url";
|
||||||
|
import syncWasmUrl from "~/assets/wa-sqlite.wasm?url";
|
||||||
|
|
||||||
|
export function parseWorkerOrURL<T, P extends T | ((is: boolean) => T)>(obj: P, is: boolean): T {
|
||||||
|
return typeof obj === "function" ? obj(is) : obj;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* auto load target worker
|
||||||
|
*
|
||||||
|
* **only basic worker options**
|
||||||
|
*/
|
||||||
|
export function defaultWorker(support: boolean): Worker {
|
||||||
|
return support
|
||||||
|
? new Worker(new URL("worker.mjs", import.meta.url), { type: "module" })
|
||||||
|
: new Worker(new URL("worker.js", import.meta.url));
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* auto load target wasm
|
||||||
|
*/
|
||||||
|
export function defaultWasmURL(useAsyncWasm: boolean): string {
|
||||||
|
return useAsyncWasm ? asyncWasmUrl : syncWasmUrl;
|
||||||
|
}
|
|
@ -1,70 +1,52 @@
|
||||||
import sqlite3InitModule, {
|
import type { SQLiteDB } from "@subframe7536/sqlite-wasm";
|
||||||
type BindingSpec,
|
|
||||||
type Database,
|
|
||||||
type OpfsDatabase,
|
|
||||||
type Sqlite3Static,
|
|
||||||
type SqlValue,
|
|
||||||
} from "@sqlite.org/sqlite-wasm";
|
|
||||||
import type { QueryResult } from "kysely";
|
import type { QueryResult } from "kysely";
|
||||||
import type { MainToWorkerMsg, WorkerToMainMsg } from "../type";
|
import type { MainToWorkerMsg, WorkerToMainMsg } from "../type";
|
||||||
|
import { initSQLite } from "@subframe7536/sqlite-wasm";
|
||||||
|
import { defaultWasmURL, parseWorkerOrURL } from "../utils";
|
||||||
|
|
||||||
let sqlite3: Sqlite3Static;
|
let db: SQLiteDB;
|
||||||
let db: Database | OpfsDatabase;
|
|
||||||
|
|
||||||
async function init(
|
async function init(
|
||||||
fileName: string,
|
fileName: string,
|
||||||
preferOpfs: boolean,
|
url: string,
|
||||||
afterInit?: (sqliteDB: Database | OpfsDatabase) => Promise<void>,
|
useOPFS: boolean,
|
||||||
|
afterInit?: (sqliteDB: SQLiteDB) => Promise<void>,
|
||||||
): Promise<void> {
|
): Promise<void> {
|
||||||
sqlite3 = await sqlite3InitModule();
|
db = await initSQLite(
|
||||||
|
(useOPFS
|
||||||
db = preferOpfs && "opfs" in sqlite3.oo1 ? new sqlite3.oo1.OpfsDb(fileName) : new sqlite3.oo1.DB(fileName);
|
? (await import("@subframe7536/sqlite-wasm/opfs")).useOpfsStorage
|
||||||
|
: (await import("@subframe7536/sqlite-wasm/idb")).useIdbStorage)(fileName, { url }),
|
||||||
|
);
|
||||||
await afterInit?.(db);
|
await afterInit?.(db);
|
||||||
}
|
}
|
||||||
|
|
||||||
function exec(isSelect: boolean, sql: string, parameters?: readonly unknown[]): QueryResult<SqlValue[]> {
|
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||||
const rows = db.exec(sql, {
|
async function exec(isSelect: boolean, sql: string, parameters?: readonly unknown[]): Promise<QueryResult<any>> {
|
||||||
bind: parameters as BindingSpec,
|
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||||
returnValue: "resultRows",
|
const rows = await db.run(sql, parameters as any[]);
|
||||||
});
|
|
||||||
|
|
||||||
return isSelect || rows.length
|
return isSelect || rows.length
|
||||||
? { rows }
|
? { rows }
|
||||||
: {
|
: {
|
||||||
rows,
|
rows,
|
||||||
insertId: BigInt(sqlite3.capi.sqlite3_last_insert_rowid(db)),
|
insertId: BigInt(db.lastInsertRowId()),
|
||||||
numAffectedRows: BigInt(db.changes()),
|
numAffectedRows: BigInt(db.changes()),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||||
function stream(
|
async function stream(onData: (data: any) => void, sql: string, parameters?: readonly unknown[]): Promise<void> {
|
||||||
onData: (data: { [columnName: string]: SqlValue }) => void,
|
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
|
||||||
sql: string,
|
await db.stream(onData, sql, parameters as any[]);
|
||||||
parameters?: readonly unknown[],
|
|
||||||
): void {
|
|
||||||
const stmt = db.prepare(sql);
|
|
||||||
|
|
||||||
if (parameters) {
|
|
||||||
stmt.bind(parameters as BindingSpec);
|
|
||||||
}
|
|
||||||
|
|
||||||
while (stmt.step()) {
|
|
||||||
onData(stmt.get({}));
|
|
||||||
}
|
|
||||||
|
|
||||||
stmt.finalize();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function loadDb(onData: (percentage: number) => void, fileName: string, useOPFS: boolean, statements: string[]) {
|
async function loadDb(onData: (percentage: number) => void, fileName: string, useOPFS: boolean, statements: string[]) {
|
||||||
if (!db) {
|
if (!db) {
|
||||||
await init(fileName, useOPFS);
|
await init(fileName, parseWorkerOrURL(defaultWasmURL, !useOPFS) as string, useOPFS);
|
||||||
}
|
}
|
||||||
|
|
||||||
const length = statements.length;
|
const length = statements.length;
|
||||||
let percentage = 0;
|
let percentage = 0;
|
||||||
|
|
||||||
for (let i = 0; i < length; i++) {
|
for (let i = 0; i < length; i += 1000) {
|
||||||
const newPercentage = Math.round((i / length) * 100);
|
const newPercentage = Math.round((i / length) * 100);
|
||||||
|
|
||||||
if (newPercentage !== percentage) {
|
if (newPercentage !== percentage) {
|
||||||
|
@ -75,8 +57,10 @@ async function loadDb(onData: (percentage: number) => void, fileName: string, us
|
||||||
|
|
||||||
console.log("executing statement");
|
console.log("executing statement");
|
||||||
|
|
||||||
db.exec(statements[i]);
|
await db.run(statements.slice(i, i + 1000).join(";"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// await db.run(statements.join(";"));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
@ -92,7 +76,7 @@ async function loadDb(onData: (percentage: number) => void, fileName: string, us
|
||||||
* )
|
* )
|
||||||
*/
|
*/
|
||||||
export function createOnMessageCallback(
|
export function createOnMessageCallback(
|
||||||
afterInit?: (sqliteDB: Database | OpfsDatabase) => Promise<void>,
|
afterInit?: (sqliteDB: SQLiteDB) => Promise<void>,
|
||||||
): (event: MessageEvent<MainToWorkerMsg>) => Promise<void> {
|
): (event: MessageEvent<MainToWorkerMsg>) => Promise<void> {
|
||||||
return async ({ data: [msg, data1, data2, data3] }: MessageEvent<MainToWorkerMsg>) => {
|
return async ({ data: [msg, data1, data2, data3] }: MessageEvent<MainToWorkerMsg>) => {
|
||||||
const ret: WorkerToMainMsg = [msg, null, null];
|
const ret: WorkerToMainMsg = [msg, null, null];
|
||||||
|
@ -100,16 +84,16 @@ export function createOnMessageCallback(
|
||||||
try {
|
try {
|
||||||
switch (msg) {
|
switch (msg) {
|
||||||
case 0:
|
case 0:
|
||||||
await init(data1, data2, afterInit);
|
await init(data1, data2, data3, afterInit);
|
||||||
break;
|
break;
|
||||||
case 1:
|
case 1:
|
||||||
ret[1] = exec(data1, data2, data3);
|
ret[1] = await exec(data1, data2, data3);
|
||||||
break;
|
break;
|
||||||
case 2:
|
case 2:
|
||||||
db.close();
|
await db.close();
|
||||||
break;
|
break;
|
||||||
case 3:
|
case 3:
|
||||||
stream((val) => postMessage([3, [val], null] satisfies WorkerToMainMsg), data1, data2);
|
await stream((val) => postMessage([3, [val], null] satisfies WorkerToMainMsg), data1, data2);
|
||||||
ret[0] = 4;
|
ret[0] = 4;
|
||||||
break;
|
break;
|
||||||
case 4:
|
case 4:
|
Loading…
Add table
Add a link
Reference in a new issue