duskflower/signalstats
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat: use official wasm in worker and cached data info / warning

Browse source
This commit is contained in:
Samuel 2025-01-21 20:18:39 +01:00
parent 36ff7afa4a
commit 851f19e82e
21 changed files with 492 additions and 398 deletions
Download patch file Download diff file Expand all files Collapse all files

BIN
src/assets/wa-sqlite-async.wasm
View file

Binary file not shown.

BIN
src/assets/wa-sqlite.wasm
View file

Binary file not shown.

8
src/components/ui/A.tsx Normal file
View file

@ -0,0 +1,8 @@
import { splitProps, type Component } from "solid-js";
import { type AnchorProps, A as BaseA } from "@solidjs/router";
import clsx from "clsx";
export const A: Component<AnchorProps> = (props) => {
const [local, other] = splitProps(props, ["class"]);
return <BaseA class={clsx("text-primary", local.class)} {...other} />;
};

40
src/components/ui/callout.tsx Normal file
View file

@ -0,0 +1,40 @@
import type { Component, ComponentProps } from "solid-js"
import { splitProps } from "solid-js"
import type { VariantProps } from "class-variance-authority"
import { cva } from "class-variance-authority"
import { cn } from "~/lib/utils"
const calloutVariants = cva("rounded-md border-l-4 p-2 pl-4", {
variants: {
variant: {
default: "border-info-foreground bg-info text-info-foreground",
success: "border-success-foreground bg-success text-success-foreground",
warning: "border-warning-foreground bg-warning text-warning-foreground",
error: "border-error-foreground bg-error text-error-foreground"
}
},
defaultVariants: {
variant: "default"
}
})
type CalloutProps = ComponentProps<"div"> & VariantProps<typeof calloutVariants>
const Callout: Component<CalloutProps> = (props) => {
const [local, others] = splitProps(props, ["class", "variant"])
return <div class={cn(calloutVariants({ variant: local.variant }), local.class)} {...others} />
}
const CalloutTitle: Component<ComponentProps<"h3">> = (props) => {
const [local, others] = splitProps(props, ["class"])
return <h3 class={cn("font-semibold", local.class)} {...others} />
}
const CalloutContent: Component<ComponentProps<"div">> = (props) => {
const [local, others] = splitProps(props, ["class"])
return <div class={cn("mt-2", local.class)} {...others} />
}
export { Callout, CalloutTitle, CalloutContent }

23
src/db/db-queries.ts
View file

@ -1,9 +1,19 @@
import { sql, type NotNull } from "kysely";
import { worker, kyselyDb, SELF_ID, DB_FILENAME } from "./db";
import { cached } from "../lib/db-cache";
import type { MainToWorkerMsg, WorkerToMainMsg } from "~/lib/kysely-wasqlite-worker/type";
import { worker, kyselyDb, SELF_ID, DB_FILENAME, setDbLoaded } from "./db";
import { cached, clearDbCache } from "../lib/db-cache";
import type { MainToWorkerMsg, WorkerToMainMsg } from "~/lib/kysely-official-wasm-worker/type";
export const loadDb = async (statements: string[], progressCallback?: (percentage: number) => void): Promise<void> => {
// try to persist storage, https://web.dev/articles/persistent-storage#request_persistent_storage
try {
if (navigator.storage?.persist && !(await navigator.storage.persisted())) {
await navigator.storage.persist();
}
// biome-ignore lint/suspicious/noEmptyBlockStatements: <explanation>
} catch {}
clearDbCache();
export const loadDb = (statements: string[], progressCallback?: (percentage: number) => void): Promise<void> => {
return new Promise((resolve, reject) => {
const progressListener = ({ data }: MessageEvent<WorkerToMainMsg>) => {
if (data[0] === 5) {
@ -19,12 +29,15 @@ export const loadDb = (statements: string[], progressCallback?: (percentage: num
worker.removeEventListener("message", progressListener);
worker.removeEventListener("message", endListener);
setDbLoaded(true);
resolve();
}
};
worker.addEventListener("message", progressListener);
worker.addEventListener("message", endListener);
worker.addEventListener("message", progressListener);
worker.postMessage([4, DB_FILENAME, true, statements] satisfies MainToWorkerMsg);
});

10
src/db/db.ts
View file

@ -1,9 +1,8 @@
import { makePersisted } from "@solid-primitives/storage";
import { Kysely } from "kysely";
import type { DB } from "./db-schema";
import { OfficialWasmWorkerDialect } from "~/lib/kysely-official-wasm-worker";
import wasmWorkerUrl from "~/lib/kysely-official-wasm-worker/worker?url";
import { createSignal } from "solid-js";
import { WaSqliteWorkerDialect } from "~/lib/kysely-wasqlite-worker";
import wasmWorkerUrl from "~/lib/kysely-wasqlite-worker/worker?url";
export const SELF_ID = 2;
@ -13,7 +12,7 @@ export const worker = new Worker(wasmWorkerUrl, {
type: "module",
});
const dialect = new WaSqliteWorkerDialect({
const dialect = new OfficialWasmWorkerDialect({
fileName: DB_FILENAME,
preferOPFS: true,
worker,
@ -23,4 +22,5 @@ export const kyselyDb = new Kysely<DB>({
dialect,
});
export const [dbHash, setDbHash] = makePersisted(createSignal<number>());
export const [dbLoaded, setDbLoaded] = createSignal(false);
// export const [dbHash, setDbHash] = makePersisted(createSignal<number>());

2
src/db/index.ts
View file

@ -1,2 +1,2 @@
export { kyselyDb, SELF_ID } from "./db";
export { kyselyDb, SELF_ID, dbLoaded, setDbLoaded } from "./db";
export * from "./db-queries";

53
src/index.tsx
View file

@ -1,8 +1,13 @@
/* @refresh reload */
import { MetaProvider } from "@solidjs/meta";
import { Router } from "@solidjs/router";
import { Router, useNavigate } from "@solidjs/router";
import { render } from "solid-js/web";
import App from "./App";
import { hasCashedData } from "./lib/db-cache";
import { createEffect, Show } from "solid-js";
import { dbLoaded } from "./db";
import { Callout, CalloutContent, CalloutTitle } from "./components/ui/callout";
import { A } from "./components/ui/A";
const root = document.getElementById("root");
@ -18,18 +23,44 @@ if (root) {
<div class="mx-auto max-w-screen-2xl">
<MetaProvider>
<Router
// root={(props) => {
// const navigate = useNavigate();
// const { pathname } = props.location;
root={(props) => {
const navigate = useNavigate();
// createEffect(() => {
// if (!db && pathname !== "/") {
// navigate("/");
// }
// });
createEffect(() => {
if (!dbLoaded() && !hasCashedData() && props.location.pathname !== "/") {
navigate("/");
}
});
// return props.children;
// }}
return (
<>
<Show
when={props.location.pathname !== "/" && !dbLoaded() && hasCashedData()}
fallback={
<Show when={!dbLoaded() && hasCashedData()}>
<Callout variant="default" class="my-4">
There is currently no backup database loaded, but you can watch statistics that have been
cached, meaning only chats you only watched or were preloaded.
<br />
<A href="/overview">Watch cached statistics</A>
</Callout>
</Show>
}
>
<Callout variant="warning" class="my-4">
<CalloutTitle>You are watching cached statistics</CalloutTitle>
<CalloutContent>
Currently there is no backup database loaded. You can only watch statistics that have been
cached, meaning only chats you only watched or were preloaded.
<br />
<A href="/">Load a backup</A>
</CalloutContent>
</Callout>
</Show>
{props.children}
</>
);
}}
>
<App />
</Router>

81
src/lib/db-cache.ts
View file

@ -1,47 +1,36 @@
import { deserialize, serialize } from "seroval";
import {} from "solid-js";
import { hashString } from "./hash";
export const DATABASE_HASH_PREFIX = "database";
export const hasCashedData = () => {
for (let i = 0, len = localStorage.length; i < len; i++) {
const key = localStorage.key(i);
if (key?.startsWith(DATABASE_HASH_PREFIX)) {
return true;
}
}
return false;
};
// clear the cache on new session so that selecting a different database does not result in wrong cache entries
// const clearDbCache = () => {
// for (let i = 0, len = localStorage.length; i < len; i++) {
// const key = localStorage.key(i);
export const clearDbCache = () => {
for (let i = 0, len = localStorage.length; i < len; i++) {
const key = localStorage.key(i);
// if (key?.startsWith(DATABASE_HASH_PREFIX)) {
// localStorage.removeItem(key);
// }
// }
// };
// let prevDbHash = dbHash();
// createRoot(() => {
// createEffect(() => {
// on(
// dbHash,
// (currentDbHash) => {
// if (currentDbHash && currentDbHash !== prevDbHash) {
// prevDbHash = currentDbHash;
// clearDbCache();
// }
// },
// {
// defer: true,
// },
// );
// });
// });
if (key?.startsWith(DATABASE_HASH_PREFIX)) {
localStorage.removeItem(key);
}
}
};
class LocalStorageCacheAdapter {
keys = new Set<string>(Object.keys(localStorage).filter((key) => key.startsWith(this.prefix)));
prefix = "database";
// TODO: real way of detecting if the db is loaded, on loading the db and opfs (if persisted db?)
// #dbLoaded = createMemo(() => !!dbHash());
keys = new Set<string>(Object.keys(localStorage).filter((key) => key.startsWith(DATABASE_HASH_PREFIX)));
#createKey(cacheName: string, key: string): string {
return `${this.prefix}-${cacheName}-${key}`;
return `${DATABASE_HASH_PREFIX}-${cacheName}-${key}`;
}
set(cacheName: string, key: string, value: unknown, isPromise = false) {
@ -60,13 +49,7 @@ class LocalStorageCacheAdapter {
}
has(cacheName: string, key: string): boolean {
// if (this.#dbLoaded()) {
return this.keys.has(this.#createKey(cacheName, key));
// }
// console.info("No database loaded");
// return false;
}
get<R>(
@ -78,7 +61,6 @@ class LocalStorageCacheAdapter {
value: R;
}
| undefined {
// if (this.#dbLoaded()) {
const item = localStorage.getItem(this.#createKey(cacheName, key));
if (item) {
@ -87,9 +69,6 @@ class LocalStorageCacheAdapter {
value: R;
};
}
// } else {
// console.info("No database loaded");
// }
}
}
@ -122,10 +101,14 @@ const createHashKey = (...args: unknown[]) => {
return hashString(stringToHash);
};
export const cached = <T extends unknown[], R, TT>(fn: (...args: T) => R, self?: ThisType<TT>): ((...args: T) => R) => {
type CachedFn<T extends unknown[], R> = ((...args: T) => R) & {
hasCacheFor: (...args: T) => boolean;
};
export const cached = <T extends unknown[], R, TT>(fn: (...args: T) => R, self?: ThisType<TT>): CachedFn<T, R> => {
const cacheName = hashString(fn.toString()).toString();
return (...args: T) => {
const cachedFn: CachedFn<T, R> = (...args: T) => {
const cacheKey = createHashKey(...args).toString();
const cachedValue = cache.get<R>(cacheName, cacheKey);
@ -154,4 +137,12 @@ export const cached = <T extends unknown[], R, TT>(fn: (...args: T) => R, self?:
return newValue;
};
cachedFn.hasCacheFor = (...args: T) => {
const cacheKey = createHashKey(...args).toString();
return cache.has(cacheName, cacheKey);
};
return cachedFn;
};

60
src/lib/kysely-wasqlite-worker/driver.ts → src/lib/kysely-official-wasm-worker/driver.ts
View file

@ -1,52 +1,45 @@
import type { DatabaseConnection, Driver, QueryResult } from "kysely";
import type { Emitter } from "zen-mitt";
import type { EventWithError, MainToWorkerMsg, WaSqliteWorkerDialectConfig, WorkerToMainMsg } from "./type";
import { isModuleWorkerSupport, isOpfsSupported } from "@subframe7536/sqlite-wasm";
import { CompiledQuery, SelectQueryNode } from "kysely";
import type { Emitter } from "zen-mitt";
import { mitt } from "zen-mitt";
import { defaultWasmURL, defaultWorker, parseWorkerOrURL } from "./utils";
import type { EventWithError, MainToWorkerMsg, OfficialWasmWorkerDialectConfig, WorkerToMainMsg } from "./type";
import workerUrl from "./worker?url";
export class WaSqliteWorkerDriver implements Driver {
export class OfficialWasmWorkerDriver implements Driver {
private worker?: Worker;
private connection?: DatabaseConnection;
private connectionMutex = new ConnectionMutex();
private mitt?: Emitter<EventWithError>;
constructor(private config: WaSqliteWorkerDialectConfig) {}
constructor(private config: OfficialWasmWorkerDialectConfig) {}
async init(): Promise<void> {
// try to persist storage, https://web.dev/articles/persistent-storage#request_persistent_storage
try {
if (!(await navigator.storage.persisted())) {
if (navigator.storage?.persist && !(await navigator.storage.persisted())) {
await navigator.storage.persist();
}
// biome-ignore lint/suspicious/noEmptyBlockStatements: <explanation>
} catch {}
const useOPFS = (this.config.preferOPFS ?? true) ? await isOpfsSupported() : false;
this.mitt = mitt<EventWithError>();
this.worker = parseWorkerOrURL(this.config.worker || defaultWorker, useOPFS || isModuleWorkerSupport());
this.worker =
this.config.worker ??
new Worker(workerUrl, {
type: "module",
});
// biome-ignore lint/style/noNonNullAssertion: <explanation>
this.worker!.onmessage = ({ data: [type, ...msg] }: MessageEvent<WorkerToMainMsg>) => {
this.worker.onmessage = ({ data: [type, ...msg] }: MessageEvent<WorkerToMainMsg>) => {
this.mitt?.emit(type, ...msg);
};
this.worker?.postMessage([
0,
this.config.fileName,
// if use OPFS, wasm should use sync version
parseWorkerOrURL(this.config.url ?? defaultWasmURL, !useOPFS) as string,
useOPFS,
] satisfies MainToWorkerMsg);
this.worker.postMessage([0, this.config.fileName, this.config.preferOPFS ?? false] satisfies MainToWorkerMsg);
await new Promise<void>((resolve, reject) => {
this.mitt?.once(0, (_, err) => (err ? reject(err) : resolve()));
});
// biome-ignore lint/style/noNonNullAssertion: <explanation>
this.connection = new WaSqliteWorkerConnection(this.worker!, this.mitt);
this.connection = new OfficialWasmWorkerConnection(this.worker, this.mitt);
await this.config.onCreateConnection?.(this.connection);
}
@ -54,6 +47,7 @@ export class WaSqliteWorkerDriver implements Driver {
// SQLite only has one single connection. We use a mutex here to wait
// until the single connection has been released.
await this.connectionMutex.lock();
// biome-ignore lint/style/noNonNullAssertion: <explanation>
return this.connection!;
}
@ -70,9 +64,12 @@ export class WaSqliteWorkerDriver implements Driver {
await connection.executeQuery(CompiledQuery.raw("rollback"));
}
// biome-ignore lint/suspicious/useAwait: <explanation>
async releaseConnection(): Promise<void> {
this.connectionMutex.unlock();
releaseConnection(): Promise<void> {
return new Promise((resolve) => {
this.connectionMutex.unlock();
resolve();
});
}
async destroy(): Promise<void> {
@ -119,7 +116,7 @@ class ConnectionMutex {
}
}
class WaSqliteWorkerConnection implements DatabaseConnection {
class OfficialWasmWorkerConnection implements DatabaseConnection {
readonly worker: Worker;
readonly mitt?: Emitter<EventWithError>;
constructor(worker: Worker, mitt?: Emitter<EventWithError>) {
@ -130,20 +127,18 @@ class WaSqliteWorkerConnection implements DatabaseConnection {
async *streamQuery<R>(compiledQuery: CompiledQuery): AsyncIterableIterator<QueryResult<R>> {
const { parameters, sql, query } = compiledQuery;
if (!SelectQueryNode.is(query)) {
throw new Error("WaSqlite dialect only supported SELECT queries");
throw new Error("official wasm worker dialect only supports SELECT queries for streaming");
}
this.worker.postMessage([3, sql, parameters] satisfies MainToWorkerMsg);
let done = false;
let resolveFn: (value: IteratorResult<QueryResult<R>>) => void;
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
let rejectFn: (reason?: any) => void;
let rejectFn: (reason?: unknown) => void;
this.mitt?.on(3 /* data */, (data, err): void => {
if (err) {
rejectFn(err);
} else {
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
resolveFn({ value: { rows: data as any }, done: false });
resolveFn({ value: { rows: data as R[] }, done: false });
}
});
@ -172,9 +167,12 @@ class WaSqliteWorkerConnection implements DatabaseConnection {
}
async executeQuery<R>(compiledQuery: CompiledQuery<unknown>): Promise<QueryResult<R>> {
const { parameters, sql, query } = compiledQuery;
const { sql, parameters, query } = compiledQuery;
const isSelect = SelectQueryNode.is(query);
this.worker.postMessage([1, isSelect, sql, parameters] satisfies MainToWorkerMsg);
return new Promise((resolve, reject) => {
if (!this.mitt) {
reject(new Error("kysely instance has been destroyed"));

38
src/lib/kysely-official-wasm-worker/index.ts Normal file
View file

@ -0,0 +1,38 @@
import type {
DatabaseIntrospector,
Dialect,
DialectAdapter,
Driver,
Kysely,
QueryCompiler,
} from "kysely";
import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely";
import { OfficialWasmWorkerDriver } from "./driver";
import type { OfficialWasmWorkerDialectConfig } from "./type";
export type {
Promisable,
OfficialWasmWorkerDialectConfig as WaSqliteWorkerDialectConfig,
} from "./type";
export { createOnMessageCallback } from "./worker/utils";
export class OfficialWasmWorkerDialect implements Dialect {
constructor(private config: OfficialWasmWorkerDialectConfig) {}
createDriver(): Driver {
return new OfficialWasmWorkerDriver(this.config);
}
createQueryCompiler(): QueryCompiler {
return new SqliteQueryCompiler();
}
createAdapter(): DialectAdapter {
return new SqliteAdapter();
}
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
createIntrospector(db: Kysely<any>): DatabaseIntrospector {
return new SqliteIntrospector(db);
}
}

56
src/lib/kysely-official-wasm-worker/type.ts Normal file
View file

@ -0,0 +1,56 @@
import type { SqlValue } from "@sqlite.org/sqlite-wasm";
import type { DatabaseConnection, QueryResult } from "kysely";
export type Promisable<T> = T | Promise<T>;
export interface OfficialWasmWorkerDialectConfig {
/**
* db file name
*/
fileName: string;
/**
* prefer to store data in OPFS
* @default true
*/
preferOPFS?: boolean;
/**
* official wasm worker
*/
worker?: Worker;
onCreateConnection?: (connection: DatabaseConnection) => Promisable<void>;
}
type InitMsg = [type: 0, fileName: string, useOPFS: boolean];
type RunMsg = [type: 1, isSelect: boolean, sql: string, parameters?: readonly unknown[]];
type CloseMsg = [2];
type StreamMsg = [type: 3, sql: string, parameters?: readonly unknown[]];
type LoadDbMsg = [type: 4, filename: string, useOPFS: boolean, statements: string[]];
type IsInitMsg = [type: 5];
export type MainToWorkerMsg = InitMsg | RunMsg | CloseMsg | StreamMsg | LoadDbMsg | IsInitMsg;
type Events = {
0: null;
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
1: QueryResult<any> | null;
2: null;
3: {
[columnName: string]: SqlValue;
}[];
4: null;
5: number;
6: null;
};
export type WorkerToMainMsg = {
[K in keyof Events]: [type: K, data: Events[K], err: unknown];
}[keyof Events];
export type EventWithError = {
[K in keyof Events]: [data: Events[K], err: unknown];
};

0
src/lib/kysely-wasqlite-worker/worker/index.ts → src/lib/kysely-official-wasm-worker/worker/index.ts
View file

147
src/lib/kysely-official-wasm-worker/worker/utils.ts Normal file
View file

@ -0,0 +1,147 @@
import sqlite3InitModule, {
type BindingSpec,
type Database,
type OpfsDatabase,
type Sqlite3Static,
type SqlValue,
} from "@sqlite.org/sqlite-wasm";
import type { QueryResult } from "kysely";
import type { MainToWorkerMsg, WorkerToMainMsg } from "../type";
let sqlite3: Sqlite3Static;
let currentDbName: string;
let db: Database | OpfsDatabase;
async function init(
fileName: string,
preferOpfs: boolean,
afterInit?: (sqliteDB: Database | OpfsDatabase) => Promise<void>,
): Promise<void> {
if (db && currentDbName === fileName) {
return;
}
// only open new Db if there is no db opened or we want to open a different db
currentDbName = fileName;
sqlite3 = await sqlite3InitModule();
db = preferOpfs && "opfs" in sqlite3.oo1 ? new sqlite3.oo1.OpfsDb(fileName) : new sqlite3.oo1.DB(fileName);
await afterInit?.(db);
}
function exec(
isSelect: boolean,
sql: string,
parameters?: readonly unknown[],
): QueryResult<{
[columnName: string]: SqlValue;
}> {
if (isSelect) {
const rows = db.selectObjects(sql, parameters as BindingSpec);
return { rows };
} else {
db.exec(sql, {
bind: parameters as BindingSpec,
returnValue: "resultRows",
});
return {
rows: [],
insertId: BigInt(sqlite3.capi.sqlite3_last_insert_rowid(db)),
numAffectedRows: BigInt(db.changes()),
};
}
}
function stream(
onData: (data: { [columnName: string]: SqlValue }) => void,
sql: string,
parameters?: readonly unknown[],
): void {
const stmt = db.prepare(sql);
if (parameters) {
stmt.bind(parameters as BindingSpec);
}
while (stmt.step()) {
onData(stmt.get({}));
}
stmt.finalize();
}
async function loadDb(onData: (percentage: number) => void, fileName: string, useOPFS: boolean, statements: string[]) {
if (!db) {
await init(fileName, useOPFS);
}
const length = statements.length;
let percentage = 0;
for (let i = 0; i < length; i++) {
const newPercentage = Math.round((i / length) * 100);
if (newPercentage !== percentage) {
onData(newPercentage);
percentage = newPercentage;
}
db.exec(statements[i]);
}
}
/**
* Handle worker message, support custom callback on initialization
* @example
* // worker.ts
* import { createOnMessageCallback, customFunction } from 'kysely-wasqlite-worker'
*
* onmessage = createOnMessageCallback(
* async (sqliteDB: SQLiteDB) => {
* customFunction(sqliteDB.sqlite, sqliteDB.db, 'customFunction', (a, b) => a + b)
* }
* )
*/
export function createOnMessageCallback(
afterInit?: (sqliteDB: Database | OpfsDatabase) => Promise<void>,
): (event: MessageEvent<MainToWorkerMsg>) => Promise<void> {
return async ({ data: [msg, data1, data2, data3] }: MessageEvent<MainToWorkerMsg>) => {
const ret: WorkerToMainMsg = [msg, null, null];
try {
switch (msg) {
case 0:
await init(data1, data2, afterInit);
break;
case 1:
ret[1] = exec(data1, data2, data3);
break;
case 2:
db.close();
break;
case 3:
stream((val) => postMessage([3, [val], null] satisfies WorkerToMainMsg), data1, data2);
ret[0] = 4;
break;
case 4:
await loadDb(
(percentage) => postMessage([5, percentage, null] satisfies WorkerToMainMsg),
data1,
data2,
data3,
);
ret[0] = 6;
break;
}
} catch (error) {
ret[2] = error;
}
postMessage(ret);
};
}

48
src/lib/kysely-wasqlite-worker/index.ts
View file

@ -1,48 +0,0 @@
import type { DatabaseIntrospector, Dialect, DialectAdapter, Driver, Kysely, QueryCompiler } from "kysely";
import type { WaSqliteWorkerDialectConfig } from "./type";
import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely";
import { WaSqliteWorkerDriver } from "./driver";
export type { Promisable, WaSqliteWorkerDialectConfig } from "./type";
export { createOnMessageCallback } from "./worker/utils";
export {
customFunction,
isIdbSupported,
isModuleWorkerSupport,
isOpfsSupported,
type SQLiteDB,
} from "@subframe7536/sqlite-wasm";
export class WaSqliteWorkerDialect implements Dialect {
/**
* dialect for [`wa-sqlite`](https://github.com/rhashimoto/wa-sqlite),
* execute sql in `Web Worker`,
* store data in [OPFS](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system) or IndexedDB
*
* @example
* import { WaSqliteWorkerDialect } from 'kysely-wasqlite-worker'
*
* const dialect = new WaSqliteWorkerDialect({
* fileName: 'test',
* })
*/
constructor(private config: WaSqliteWorkerDialectConfig) {}
createDriver(): Driver {
return new WaSqliteWorkerDriver(this.config);
}
createQueryCompiler(): QueryCompiler {
return new SqliteQueryCompiler();
}
createAdapter(): DialectAdapter {
return new SqliteAdapter();
}
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
createIntrospector(db: Kysely<any>): DatabaseIntrospector {
return new SqliteIntrospector(db);
}
}

81
src/lib/kysely-wasqlite-worker/type.ts
View file

@ -1,81 +0,0 @@
import type { SqlValue } from "@sqlite.org/sqlite-wasm";
import type { DatabaseConnection, QueryResult } from "kysely";
export type Promisable<T> = T | Promise<T>;
export interface WaSqliteWorkerDialectConfig {
/**
* db file name
*/
fileName: string;
/**
* prefer to store data in OPFS
* @default true
*/
preferOPFS?: boolean;
/**
* wasqlite worker
*
* built-in: {@link useDefaultWorker}
* @param supportModuleWorker if support `{ type: 'module' }` in worker options
* @example
* import { useDefaultWorker } from 'kysely-wasqlite-worker'
* @example
* (supportModuleWorker) => supportModuleWorker
* ? new Worker(
* new URL('kysely-wasqlite-worker/worker-module', import.meta.url),
* { type: 'module', credentials: 'same-origin' }
* )
* : new Worker(
* new URL('kysely-wasqlite-worker/worker-classic', import.meta.url),
* { type: 'classic', name: 'test' }
* )
*/
worker?: Worker | ((supportModuleWorker: boolean) => Worker);
/**
* wasm URL
*
* built-in: {@link useDefaultWasmURL}
* @param useAsyncWasm if need to use wa-sqlite-async.wasm
* @example
* import { useDefaultWasmURL } from 'kysely-wasqlite-worker'
* @example
* (useAsyncWasm) => useAsyncWasm
* ? 'https://cdn.jsdelivr.net/gh/rhashimoto/wa-sqlite@v1.0.0/dist/wa-sqlite-async.wasm'
* : new URL('kysely-wasqlite-worker/wasm-sync', import.meta.url).href
*/
url?: string | ((useAsyncWasm: boolean) => string);
onCreateConnection?: (connection: DatabaseConnection) => Promisable<void>;
}
type RunMsg = [type: 1, isSelect: boolean, sql: string, parameters?: readonly unknown[]];
type StreamMsg = [type: 3, sql: string, parameters?: readonly unknown[]];
type InitMsg = [type: 0, url: string, fileName: string, useOPFS: boolean];
type CloseMsg = [2];
type LoadDbMsg = [type: 4, filename: string, useOPFS: boolean, statements: string[]];
export type MainToWorkerMsg = InitMsg | RunMsg | CloseMsg | StreamMsg | LoadDbMsg;
export type WorkerToMainMsg = {
[K in keyof Events]: [type: K, data: Events[K], err: unknown];
}[keyof Events];
export type EventWithError = {
[K in keyof Events]: [data: Events[K], err: unknown];
};
type Events = {
0: null;
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
1: QueryResult<any> | null;
2: null;
3: {
[columnName: string]: SqlValue;
}[];
4: null;
5: number;
6: null;
};

24
src/lib/kysely-wasqlite-worker/utils.ts
View file

@ -1,24 +0,0 @@
import asyncWasmUrl from "~/assets/wa-sqlite-async.wasm?url";
import syncWasmUrl from "~/assets/wa-sqlite.wasm?url";
export function parseWorkerOrURL<T, P extends T | ((is: boolean) => T)>(obj: P, is: boolean): T {
return typeof obj === "function" ? obj(is) : obj;
}
/**
* auto load target worker
*
* **only basic worker options**
*/
export function defaultWorker(support: boolean): Worker {
return support
? new Worker(new URL("worker.mjs", import.meta.url), { type: "module" })
: new Worker(new URL("worker.js", import.meta.url));
}
/**
* auto load target wasm
*/
export function defaultWasmURL(useAsyncWasm: boolean): string {
return useAsyncWasm ? asyncWasmUrl : syncWasmUrl;
}

110
src/lib/kysely-wasqlite-worker/worker/utils.ts
View file

@ -1,110 +0,0 @@
import type { SQLiteDB } from "@subframe7536/sqlite-wasm";
import type { QueryResult } from "kysely";
import type { MainToWorkerMsg, WorkerToMainMsg } from "../type";
import { initSQLite } from "@subframe7536/sqlite-wasm";
import { defaultWasmURL, parseWorkerOrURL } from "../utils";
let db: SQLiteDB;
async function init(
fileName: string,
url: string,
useOPFS: boolean,
afterInit?: (sqliteDB: SQLiteDB) => Promise<void>,
): Promise<void> {
db = await initSQLite(
(useOPFS
? (await import("@subframe7536/sqlite-wasm/opfs")).useOpfsStorage
: (await import("@subframe7536/sqlite-wasm/idb")).useIdbStorage)(fileName, { url }),
);
await afterInit?.(db);
}
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
async function exec(isSelect: boolean, sql: string, parameters?: readonly unknown[]): Promise<QueryResult<any>> {
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
const rows = await db.run(sql, parameters as any[]);
return isSelect || rows.length
? { rows }
: {
rows,
insertId: BigInt(db.lastInsertRowId()),
numAffectedRows: BigInt(db.changes()),
};
}
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
async function stream(onData: (data: any) => void, sql: string, parameters?: readonly unknown[]): Promise<void> {
// biome-ignore lint/suspicious/noExplicitAny: <explanation>
await db.stream(onData, sql, parameters as any[]);
}
async function loadDb(onData: (percentage: number) => void, fileName: string, useOPFS: boolean, statements: string[]) {
if (!db) {
await init(fileName, parseWorkerOrURL(defaultWasmURL, !useOPFS) as string, useOPFS);
}
const length = statements.length;
let percentage = 0;
for (let i = 0; i < length; i += 1000) {
const newPercentage = Math.round((i / length) * 100);
if (newPercentage !== percentage) {
onData(newPercentage);
percentage = newPercentage;
}
console.log("executing statement");
await db.run(statements.slice(i, i + 1000).join(";"));
}
// await db.run(statements.join(";"));
}
/**
* Handle worker message, support custom callback on initialization
* @example
* // worker.ts
* import { createOnMessageCallback, customFunction } from 'kysely-wasqlite-worker'
*
* onmessage = createOnMessageCallback(
* async (sqliteDB: SQLiteDB) => {
* customFunction(sqliteDB.sqlite, sqliteDB.db, 'customFunction', (a, b) => a + b)
* }
* )
*/
export function createOnMessageCallback(
afterInit?: (sqliteDB: SQLiteDB) => Promise<void>,
): (event: MessageEvent<MainToWorkerMsg>) => Promise<void> {
return async ({ data: [msg, data1, data2, data3] }: MessageEvent<MainToWorkerMsg>) => {
const ret: WorkerToMainMsg = [msg, null, null];
try {
switch (msg) {
case 0:
await init(data1, data2, data3, afterInit);
break;
case 1:
ret[1] = await exec(data1, data2, data3);
break;
case 2:
await db.close();
break;
case 3:
await stream((val) => postMessage([3, [val], null] satisfies WorkerToMainMsg), data1, data2);
ret[0] = 4;
break;
case 4:
loadDb((percentage) => postMessage([5, percentage, null] satisfies WorkerToMainMsg), data1, data2, data3);
ret[0] = 6;
break;
}
} catch (error) {
console.error(error);
ret[2] = error;
}
postMessage(ret);
};
}

39
src/pages/home.tsx
View file

@ -6,7 +6,7 @@ import { Portal } from "solid-js/web";
import { Flex } from "~/components/ui/flex";
import { Progress, ProgressLabel, ProgressValueLabel } from "~/components/ui/progress";
import { loadDb } from "~/db/db-queries";
import { loadDb } from "~/db";
import { decryptBackup } from "~/lib/decryptor";
import { createDropzone, createFileUploader } from "@solid-primitives/upload";
import { Button } from "~/components/ui/button";
@ -37,29 +37,38 @@ export const Home: Component<RouteSectionProps> = () => {
const [loadingProgress, setLoadingProgress] = createSignal<number>();
// const [isLoadingDatabase, setIsLoadingDatabase] = createSignal(false);
const onSubmit: JSX.EventHandler<HTMLFormElement, SubmitEvent> = (event) => {
const onSubmit: JSX.EventHandler<HTMLFormElement, SubmitEvent> = async (event) => {
event.preventDefault();
const currentBackupFile = backupFile();
const currentPassphrase = passphrase();
if (currentBackupFile && currentPassphrase) {
decryptBackup(currentBackupFile, currentPassphrase, setDecryptionProgress)
.then(async (result) => {
setDecryptionProgress(undefined);
// setIsLoadingDatabase(true);
setLoadingProgress(0);
// const hashChunk = await currentBackupFile.slice(-1000).text();
// const hash = hashString(hashChunk);
await loadDb(result.database_statements, (newValue) => (console.log(newValue), setLoadingProgress(newValue)));
// if (hash === dbHash()) {
// return;
// }
// setIsLoadingDatabase(false);
setLoadingProgress(undefined);
// setDbHash(hash);
navigate("/overview");
})
.catch((error) => {
console.error("Decryption failed:", error);
});
try {
const decrypted = await decryptBackup(currentBackupFile, currentPassphrase, setDecryptionProgress);
setDecryptionProgress(undefined);
// setIsLoadingDatabase(true);
setLoadingProgress(0);
await loadDb(decrypted.database_statements, setLoadingProgress);
// setIsLoadingDatabase(false);
setLoadingProgress(undefined);
navigate("/overview");
} catch (error) {
console.error("Decryption failed:", error);
}
}
};

8
src/pages/overview/index.tsx
View file

@ -11,7 +11,9 @@ export const Overview: Component<RouteSectionProps> = () => {
const [allSelfSentMessagesCount] = createResource(() => overallSentMessagesQuery(SELF_ID));
const [roomOverview] = createResource<RoomOverview[] | undefined>(async () => {
return (await allThreadsOverviewQuery())?.map((row) => {
const overview = await allThreadsOverviewQuery();
return overview.map((row) => {
const isGroup = row.title !== null;
let name = "";
@ -41,9 +43,7 @@ export const Overview: Component<RouteSectionProps> = () => {
<div>
<p>All messages: {allSelfSentMessagesCount()?.messageCount as number}</p>
<Show when={!roomOverview.loading && roomOverview()} fallback="Loading...">
{(currentRoomOverview) => (
console.log(currentRoomOverview()), (<OverviewTable data={currentRoomOverview()} />)
)}
{(currentRoomOverview) => <OverviewTable data={currentRoomOverview()} />}
</Show>
</div>
</>

62
src/pages/overview/overview-table.tsx
View file

@ -26,6 +26,7 @@ import { TextField, TextFieldInput } from "~/components/ui/text-field";
import { cn } from "~/lib/utils";
import { Flex } from "~/components/ui/flex";
import { dbLoaded, threadSentMessagesOverviewQuery } from "~/db";
export interface RoomOverview {
threadId: number;
@ -55,6 +56,18 @@ const isGroupFilterFn: FilterFn<RoomOverview> = (row, _columnId, filterValue) =>
return !row.original.isGroup;
};
const rowIsAvailable = (threadId: number): boolean => {
if (dbLoaded()) {
return true;
}
if (threadSentMessagesOverviewQuery.hasCacheFor(threadId)) {
return true;
}
return false;
};
const SortingDisplay: Component<{ sorting: false | SortDirection; class?: string; activeClass?: string }> = (props) => {
return (
<Switch>
@ -92,13 +105,14 @@ export const columns = [
cell: (props) => {
const isArchived = props.row.getValue("archived");
const isGroup = props.row.getValue("isGroup");
const isCached = !dbLoaded() && rowIsAvailable(props.row.original.threadId);
return (
<Flex class="w-full" flexDirection="row">
<span class="max-w-2xl overflow-hidden text-ellipsis whitespace-nowrap font-bold">
{props.cell.getValue()}
</span>
<Show when={isArchived || isGroup}>
<Show when={isArchived || isGroup || !isCached}>
<Flex flexDirection="row" class="ml-auto gap-2">
<Show when={isArchived}>
<Badge variant="outline" class="ml-auto">
@ -110,6 +124,11 @@ export const columns = [
Group
</Badge>
</Show>
<Show when={!isCached}>
<Badge variant="outline" class="ml-auto">
Not available
</Badge>
</Show>
</Flex>
</Show>
</Flex>
@ -267,12 +286,12 @@ export const OverviewTable = (props: OverviewTableProps) => {
</div>
<div class="flex items-start space-x-2">
<Checkbox
id="show-archived"
id="show-groups"
checked={(table.getColumn("isGroup")?.getFilterValue() as boolean | undefined) ?? false}
onChange={(value) => table.getColumn("isGroup")?.setFilterValue(value)}
/>
<div class="grid gap-1.5 leading-none">
<Label for="show-archived">Show group chats (detailed analysis not implemented)</Label>
<Label for="show-groups">Show group chats (detailed analysis not implemented)</Label>
</div>
</div>
</div>
@ -315,32 +334,39 @@ export const OverviewTable = (props: OverviewTableProps) => {
{(row) => (
<TableRow
class="cursor-pointer [&:last-of-type>td:first-of-type]:rounded-bl-md [&:last-of-type>td:last-of-type]:rounded-br-md"
classList={{
"text-muted-foreground": !rowIsAvailable(row.original.threadId),
}}
data-state={row.getIsSelected() && "selected"}
onPointerEnter={(event) => {
const threadId = row.original.threadId;
const isGroup = row.original.isGroup;
const preloadTimeout = setTimeout(() => {
preload(`/${isGroup ? "group" : "dm"}/${threadId.toString()}`, {
preloadData: true,
});
}, 20);
if (rowIsAvailable(threadId)) {
const preloadTimeout = setTimeout(() => {
preload(`/${isGroup ? "group" : "dm"}/${threadId.toString()}`, {
preloadData: true,
});
}, 20);
event.currentTarget.addEventListener(
"pointerout",
() => {
clearTimeout(preloadTimeout);
},
{
once: true,
},
);
event.currentTarget.addEventListener(
"pointerout",
() => {
clearTimeout(preloadTimeout);
},
{
once: true,
},
);
}
}}
onClick={() => {
const threadId = row.original.threadId;
const isGroup = row.original.isGroup;
navigate(`/${isGroup ? "group" : "dm"}/${threadId.toString()}`);
if (rowIsAvailable(threadId)) {
navigate(`/${isGroup ? "group" : "dm"}/${threadId.toString()}`);
}
}}
>
<For each={row.getVisibleCells()}>