diff --git a/src/assets/wa-sqlite-async.wasm b/src/assets/wa-sqlite-async.wasm deleted file mode 100755 index cbf0c2b..0000000 Binary files a/src/assets/wa-sqlite-async.wasm and /dev/null differ diff --git a/src/assets/wa-sqlite.wasm b/src/assets/wa-sqlite.wasm deleted file mode 100755 index e735879..0000000 Binary files a/src/assets/wa-sqlite.wasm and /dev/null differ diff --git a/src/components/ui/A.tsx b/src/components/ui/A.tsx new file mode 100644 index 0000000..6ab6450 --- /dev/null +++ b/src/components/ui/A.tsx @@ -0,0 +1,8 @@ +import { splitProps, type Component } from "solid-js"; +import { type AnchorProps, A as BaseA } from "@solidjs/router"; +import clsx from "clsx"; + +export const A: Component = (props) => { + const [local, other] = splitProps(props, ["class"]); + return ; +}; diff --git a/src/components/ui/callout.tsx b/src/components/ui/callout.tsx new file mode 100644 index 0000000..f0efd66 --- /dev/null +++ b/src/components/ui/callout.tsx @@ -0,0 +1,40 @@ +import type { Component, ComponentProps } from "solid-js" +import { splitProps } from "solid-js" + +import type { VariantProps } from "class-variance-authority" +import { cva } from "class-variance-authority" + +import { cn } from "~/lib/utils" + +const calloutVariants = cva("rounded-md border-l-4 p-2 pl-4", { + variants: { + variant: { + default: "border-info-foreground bg-info text-info-foreground", + success: "border-success-foreground bg-success text-success-foreground", + warning: "border-warning-foreground bg-warning text-warning-foreground", + error: "border-error-foreground bg-error text-error-foreground" + } + }, + defaultVariants: { + variant: "default" + } +}) + +type CalloutProps = ComponentProps<"div"> & VariantProps + +const Callout: Component = (props) => { + const [local, others] = splitProps(props, ["class", "variant"]) + return
+} + +const CalloutTitle: Component> = (props) => { + const [local, others] = splitProps(props, ["class"]) + return

+} + +const CalloutContent: Component> = (props) => { + const [local, others] = splitProps(props, ["class"]) + return
+} + +export { Callout, CalloutTitle, CalloutContent } diff --git a/src/db/db-queries.ts b/src/db/db-queries.ts index 4d063fe..4d1a87a 100644 --- a/src/db/db-queries.ts +++ b/src/db/db-queries.ts @@ -1,9 +1,19 @@ import { sql, type NotNull } from "kysely"; -import { worker, kyselyDb, SELF_ID, DB_FILENAME } from "./db"; -import { cached } from "../lib/db-cache"; -import type { MainToWorkerMsg, WorkerToMainMsg } from "~/lib/kysely-wasqlite-worker/type"; +import { worker, kyselyDb, SELF_ID, DB_FILENAME, setDbLoaded } from "./db"; +import { cached, clearDbCache } from "../lib/db-cache"; +import type { MainToWorkerMsg, WorkerToMainMsg } from "~/lib/kysely-official-wasm-worker/type"; + +export const loadDb = async (statements: string[], progressCallback?: (percentage: number) => void): Promise => { + // try to persist storage, https://web.dev/articles/persistent-storage#request_persistent_storage + try { + if (navigator.storage?.persist && !(await navigator.storage.persisted())) { + await navigator.storage.persist(); + } + // biome-ignore lint/suspicious/noEmptyBlockStatements: + } catch {} + + clearDbCache(); -export const loadDb = (statements: string[], progressCallback?: (percentage: number) => void): Promise => { return new Promise((resolve, reject) => { const progressListener = ({ data }: MessageEvent) => { if (data[0] === 5) { @@ -19,12 +29,15 @@ export const loadDb = (statements: string[], progressCallback?: (percentage: num worker.removeEventListener("message", progressListener); worker.removeEventListener("message", endListener); + + setDbLoaded(true); + resolve(); } }; - worker.addEventListener("message", progressListener); worker.addEventListener("message", endListener); + worker.addEventListener("message", progressListener); worker.postMessage([4, DB_FILENAME, true, statements] satisfies MainToWorkerMsg); }); diff --git a/src/db/db.ts b/src/db/db.ts index 3fc4aa4..2cd2082 100644 --- a/src/db/db.ts +++ b/src/db/db.ts @@ -1,9 +1,8 @@ -import { makePersisted } from "@solid-primitives/storage"; import { Kysely } from "kysely"; import type { DB } from "./db-schema"; +import { OfficialWasmWorkerDialect } from "~/lib/kysely-official-wasm-worker"; +import wasmWorkerUrl from "~/lib/kysely-official-wasm-worker/worker?url"; import { createSignal } from "solid-js"; -import { WaSqliteWorkerDialect } from "~/lib/kysely-wasqlite-worker"; -import wasmWorkerUrl from "~/lib/kysely-wasqlite-worker/worker?url"; export const SELF_ID = 2; @@ -13,7 +12,7 @@ export const worker = new Worker(wasmWorkerUrl, { type: "module", }); -const dialect = new WaSqliteWorkerDialect({ +const dialect = new OfficialWasmWorkerDialect({ fileName: DB_FILENAME, preferOPFS: true, worker, @@ -23,4 +22,5 @@ export const kyselyDb = new Kysely({ dialect, }); -export const [dbHash, setDbHash] = makePersisted(createSignal()); +export const [dbLoaded, setDbLoaded] = createSignal(false); +// export const [dbHash, setDbHash] = makePersisted(createSignal()); diff --git a/src/db/index.ts b/src/db/index.ts index 5a4f08d..319e19d 100644 --- a/src/db/index.ts +++ b/src/db/index.ts @@ -1,2 +1,2 @@ -export { kyselyDb, SELF_ID } from "./db"; +export { kyselyDb, SELF_ID, dbLoaded, setDbLoaded } from "./db"; export * from "./db-queries"; diff --git a/src/index.tsx b/src/index.tsx index 6bddfe1..e326885 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -1,8 +1,13 @@ /* @refresh reload */ import { MetaProvider } from "@solidjs/meta"; -import { Router } from "@solidjs/router"; +import { Router, useNavigate } from "@solidjs/router"; import { render } from "solid-js/web"; import App from "./App"; +import { hasCashedData } from "./lib/db-cache"; +import { createEffect, Show } from "solid-js"; +import { dbLoaded } from "./db"; +import { Callout, CalloutContent, CalloutTitle } from "./components/ui/callout"; +import { A } from "./components/ui/A"; const root = document.getElementById("root"); @@ -18,18 +23,44 @@ if (root) {
{ - // const navigate = useNavigate(); - // const { pathname } = props.location; + root={(props) => { + const navigate = useNavigate(); - // createEffect(() => { - // if (!db && pathname !== "/") { - // navigate("/"); - // } - // }); + createEffect(() => { + if (!dbLoaded() && !hasCashedData() && props.location.pathname !== "/") { + navigate("/"); + } + }); - // return props.children; - // }} + return ( + <> + + + There is currently no backup database loaded, but you can watch statistics that have been + cached, meaning only chats you only watched or were preloaded. +
+ Watch cached statistics + + + } + > + + You are watching cached statistics + + Currently there is no backup database loaded. You can only watch statistics that have been + cached, meaning only chats you only watched or were preloaded. +
+ Load a backup + + + + {props.children} + + ); + }} > diff --git a/src/lib/db-cache.ts b/src/lib/db-cache.ts index 1a075b6..2ea2e55 100644 --- a/src/lib/db-cache.ts +++ b/src/lib/db-cache.ts @@ -1,47 +1,36 @@ import { deserialize, serialize } from "seroval"; -import {} from "solid-js"; import { hashString } from "./hash"; export const DATABASE_HASH_PREFIX = "database"; +export const hasCashedData = () => { + for (let i = 0, len = localStorage.length; i < len; i++) { + const key = localStorage.key(i); + + if (key?.startsWith(DATABASE_HASH_PREFIX)) { + return true; + } + } + + return false; +}; + // clear the cache on new session so that selecting a different database does not result in wrong cache entries -// const clearDbCache = () => { -// for (let i = 0, len = localStorage.length; i < len; i++) { -// const key = localStorage.key(i); +export const clearDbCache = () => { + for (let i = 0, len = localStorage.length; i < len; i++) { + const key = localStorage.key(i); -// if (key?.startsWith(DATABASE_HASH_PREFIX)) { -// localStorage.removeItem(key); -// } -// } -// }; - -// let prevDbHash = dbHash(); - -// createRoot(() => { -// createEffect(() => { -// on( -// dbHash, -// (currentDbHash) => { -// if (currentDbHash && currentDbHash !== prevDbHash) { -// prevDbHash = currentDbHash; -// clearDbCache(); -// } -// }, -// { -// defer: true, -// }, -// ); -// }); -// }); + if (key?.startsWith(DATABASE_HASH_PREFIX)) { + localStorage.removeItem(key); + } + } +}; class LocalStorageCacheAdapter { - keys = new Set(Object.keys(localStorage).filter((key) => key.startsWith(this.prefix))); - prefix = "database"; - // TODO: real way of detecting if the db is loaded, on loading the db and opfs (if persisted db?) - // #dbLoaded = createMemo(() => !!dbHash()); + keys = new Set(Object.keys(localStorage).filter((key) => key.startsWith(DATABASE_HASH_PREFIX))); #createKey(cacheName: string, key: string): string { - return `${this.prefix}-${cacheName}-${key}`; + return `${DATABASE_HASH_PREFIX}-${cacheName}-${key}`; } set(cacheName: string, key: string, value: unknown, isPromise = false) { @@ -60,13 +49,7 @@ class LocalStorageCacheAdapter { } has(cacheName: string, key: string): boolean { - // if (this.#dbLoaded()) { return this.keys.has(this.#createKey(cacheName, key)); - // } - - // console.info("No database loaded"); - - // return false; } get( @@ -78,7 +61,6 @@ class LocalStorageCacheAdapter { value: R; } | undefined { - // if (this.#dbLoaded()) { const item = localStorage.getItem(this.#createKey(cacheName, key)); if (item) { @@ -87,9 +69,6 @@ class LocalStorageCacheAdapter { value: R; }; } - // } else { - // console.info("No database loaded"); - // } } } @@ -122,10 +101,14 @@ const createHashKey = (...args: unknown[]) => { return hashString(stringToHash); }; -export const cached = (fn: (...args: T) => R, self?: ThisType): ((...args: T) => R) => { +type CachedFn = ((...args: T) => R) & { + hasCacheFor: (...args: T) => boolean; +}; + +export const cached = (fn: (...args: T) => R, self?: ThisType): CachedFn => { const cacheName = hashString(fn.toString()).toString(); - return (...args: T) => { + const cachedFn: CachedFn = (...args: T) => { const cacheKey = createHashKey(...args).toString(); const cachedValue = cache.get(cacheName, cacheKey); @@ -154,4 +137,12 @@ export const cached = (fn: (...args: T) => R, self?: return newValue; }; + + cachedFn.hasCacheFor = (...args: T) => { + const cacheKey = createHashKey(...args).toString(); + + return cache.has(cacheName, cacheKey); + }; + + return cachedFn; }; diff --git a/src/lib/kysely-wasqlite-worker/driver.ts b/src/lib/kysely-official-wasm-worker/driver.ts similarity index 70% rename from src/lib/kysely-wasqlite-worker/driver.ts rename to src/lib/kysely-official-wasm-worker/driver.ts index a43c2e7..f77e4e3 100644 --- a/src/lib/kysely-wasqlite-worker/driver.ts +++ b/src/lib/kysely-official-wasm-worker/driver.ts @@ -1,52 +1,45 @@ import type { DatabaseConnection, Driver, QueryResult } from "kysely"; -import type { Emitter } from "zen-mitt"; -import type { EventWithError, MainToWorkerMsg, WaSqliteWorkerDialectConfig, WorkerToMainMsg } from "./type"; -import { isModuleWorkerSupport, isOpfsSupported } from "@subframe7536/sqlite-wasm"; import { CompiledQuery, SelectQueryNode } from "kysely"; +import type { Emitter } from "zen-mitt"; import { mitt } from "zen-mitt"; -import { defaultWasmURL, defaultWorker, parseWorkerOrURL } from "./utils"; +import type { EventWithError, MainToWorkerMsg, OfficialWasmWorkerDialectConfig, WorkerToMainMsg } from "./type"; +import workerUrl from "./worker?url"; -export class WaSqliteWorkerDriver implements Driver { +export class OfficialWasmWorkerDriver implements Driver { private worker?: Worker; private connection?: DatabaseConnection; private connectionMutex = new ConnectionMutex(); private mitt?: Emitter; - constructor(private config: WaSqliteWorkerDialectConfig) {} + constructor(private config: OfficialWasmWorkerDialectConfig) {} async init(): Promise { // try to persist storage, https://web.dev/articles/persistent-storage#request_persistent_storage try { - if (!(await navigator.storage.persisted())) { + if (navigator.storage?.persist && !(await navigator.storage.persisted())) { await navigator.storage.persist(); } // biome-ignore lint/suspicious/noEmptyBlockStatements: } catch {} - const useOPFS = (this.config.preferOPFS ?? true) ? await isOpfsSupported() : false; - this.mitt = mitt(); - this.worker = parseWorkerOrURL(this.config.worker || defaultWorker, useOPFS || isModuleWorkerSupport()); + this.worker = + this.config.worker ?? + new Worker(workerUrl, { + type: "module", + }); - // biome-ignore lint/style/noNonNullAssertion: - this.worker!.onmessage = ({ data: [type, ...msg] }: MessageEvent) => { + this.worker.onmessage = ({ data: [type, ...msg] }: MessageEvent) => { this.mitt?.emit(type, ...msg); }; - this.worker?.postMessage([ - 0, - this.config.fileName, - // if use OPFS, wasm should use sync version - parseWorkerOrURL(this.config.url ?? defaultWasmURL, !useOPFS) as string, - useOPFS, - ] satisfies MainToWorkerMsg); + this.worker.postMessage([0, this.config.fileName, this.config.preferOPFS ?? false] satisfies MainToWorkerMsg); await new Promise((resolve, reject) => { this.mitt?.once(0, (_, err) => (err ? reject(err) : resolve())); }); - // biome-ignore lint/style/noNonNullAssertion: - this.connection = new WaSqliteWorkerConnection(this.worker!, this.mitt); + this.connection = new OfficialWasmWorkerConnection(this.worker, this.mitt); await this.config.onCreateConnection?.(this.connection); } @@ -54,6 +47,7 @@ export class WaSqliteWorkerDriver implements Driver { // SQLite only has one single connection. We use a mutex here to wait // until the single connection has been released. await this.connectionMutex.lock(); + // biome-ignore lint/style/noNonNullAssertion: return this.connection!; } @@ -70,9 +64,12 @@ export class WaSqliteWorkerDriver implements Driver { await connection.executeQuery(CompiledQuery.raw("rollback")); } - // biome-ignore lint/suspicious/useAwait: - async releaseConnection(): Promise { - this.connectionMutex.unlock(); + releaseConnection(): Promise { + return new Promise((resolve) => { + this.connectionMutex.unlock(); + + resolve(); + }); } async destroy(): Promise { @@ -119,7 +116,7 @@ class ConnectionMutex { } } -class WaSqliteWorkerConnection implements DatabaseConnection { +class OfficialWasmWorkerConnection implements DatabaseConnection { readonly worker: Worker; readonly mitt?: Emitter; constructor(worker: Worker, mitt?: Emitter) { @@ -130,20 +127,18 @@ class WaSqliteWorkerConnection implements DatabaseConnection { async *streamQuery(compiledQuery: CompiledQuery): AsyncIterableIterator> { const { parameters, sql, query } = compiledQuery; if (!SelectQueryNode.is(query)) { - throw new Error("WaSqlite dialect only supported SELECT queries"); + throw new Error("official wasm worker dialect only supports SELECT queries for streaming"); } this.worker.postMessage([3, sql, parameters] satisfies MainToWorkerMsg); let done = false; let resolveFn: (value: IteratorResult>) => void; - // biome-ignore lint/suspicious/noExplicitAny: - let rejectFn: (reason?: any) => void; + let rejectFn: (reason?: unknown) => void; this.mitt?.on(3 /* data */, (data, err): void => { if (err) { rejectFn(err); } else { - // biome-ignore lint/suspicious/noExplicitAny: - resolveFn({ value: { rows: data as any }, done: false }); + resolveFn({ value: { rows: data as R[] }, done: false }); } }); @@ -172,9 +167,12 @@ class WaSqliteWorkerConnection implements DatabaseConnection { } async executeQuery(compiledQuery: CompiledQuery): Promise> { - const { parameters, sql, query } = compiledQuery; + const { sql, parameters, query } = compiledQuery; + const isSelect = SelectQueryNode.is(query); + this.worker.postMessage([1, isSelect, sql, parameters] satisfies MainToWorkerMsg); + return new Promise((resolve, reject) => { if (!this.mitt) { reject(new Error("kysely instance has been destroyed")); diff --git a/src/lib/kysely-official-wasm-worker/index.ts b/src/lib/kysely-official-wasm-worker/index.ts new file mode 100644 index 0000000..1886c71 --- /dev/null +++ b/src/lib/kysely-official-wasm-worker/index.ts @@ -0,0 +1,38 @@ +import type { + DatabaseIntrospector, + Dialect, + DialectAdapter, + Driver, + Kysely, + QueryCompiler, +} from "kysely"; +import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely"; +import { OfficialWasmWorkerDriver } from "./driver"; +import type { OfficialWasmWorkerDialectConfig } from "./type"; + +export type { + Promisable, + OfficialWasmWorkerDialectConfig as WaSqliteWorkerDialectConfig, +} from "./type"; +export { createOnMessageCallback } from "./worker/utils"; + +export class OfficialWasmWorkerDialect implements Dialect { + constructor(private config: OfficialWasmWorkerDialectConfig) {} + + createDriver(): Driver { + return new OfficialWasmWorkerDriver(this.config); + } + + createQueryCompiler(): QueryCompiler { + return new SqliteQueryCompiler(); + } + + createAdapter(): DialectAdapter { + return new SqliteAdapter(); + } + + // biome-ignore lint/suspicious/noExplicitAny: + createIntrospector(db: Kysely): DatabaseIntrospector { + return new SqliteIntrospector(db); + } +} diff --git a/src/lib/kysely-official-wasm-worker/type.ts b/src/lib/kysely-official-wasm-worker/type.ts new file mode 100644 index 0000000..6ddd4b0 --- /dev/null +++ b/src/lib/kysely-official-wasm-worker/type.ts @@ -0,0 +1,56 @@ +import type { SqlValue } from "@sqlite.org/sqlite-wasm"; +import type { DatabaseConnection, QueryResult } from "kysely"; + +export type Promisable = T | Promise; + +export interface OfficialWasmWorkerDialectConfig { + /** + * db file name + */ + fileName: string; + /** + * prefer to store data in OPFS + * @default true + */ + preferOPFS?: boolean; + /** + * official wasm worker + */ + worker?: Worker; + onCreateConnection?: (connection: DatabaseConnection) => Promisable; +} + +type InitMsg = [type: 0, fileName: string, useOPFS: boolean]; + +type RunMsg = [type: 1, isSelect: boolean, sql: string, parameters?: readonly unknown[]]; + +type CloseMsg = [2]; + +type StreamMsg = [type: 3, sql: string, parameters?: readonly unknown[]]; + +type LoadDbMsg = [type: 4, filename: string, useOPFS: boolean, statements: string[]]; + +type IsInitMsg = [type: 5]; + +export type MainToWorkerMsg = InitMsg | RunMsg | CloseMsg | StreamMsg | LoadDbMsg | IsInitMsg; + +type Events = { + 0: null; + // biome-ignore lint/suspicious/noExplicitAny: + 1: QueryResult | null; + 2: null; + 3: { + [columnName: string]: SqlValue; + }[]; + 4: null; + 5: number; + 6: null; +}; + +export type WorkerToMainMsg = { + [K in keyof Events]: [type: K, data: Events[K], err: unknown]; +}[keyof Events]; + +export type EventWithError = { + [K in keyof Events]: [data: Events[K], err: unknown]; +}; diff --git a/src/lib/kysely-wasqlite-worker/worker/index.ts b/src/lib/kysely-official-wasm-worker/worker/index.ts similarity index 100% rename from src/lib/kysely-wasqlite-worker/worker/index.ts rename to src/lib/kysely-official-wasm-worker/worker/index.ts diff --git a/src/lib/kysely-official-wasm-worker/worker/utils.ts b/src/lib/kysely-official-wasm-worker/worker/utils.ts new file mode 100644 index 0000000..827ff85 --- /dev/null +++ b/src/lib/kysely-official-wasm-worker/worker/utils.ts @@ -0,0 +1,147 @@ +import sqlite3InitModule, { + type BindingSpec, + type Database, + type OpfsDatabase, + type Sqlite3Static, + type SqlValue, +} from "@sqlite.org/sqlite-wasm"; +import type { QueryResult } from "kysely"; +import type { MainToWorkerMsg, WorkerToMainMsg } from "../type"; + +let sqlite3: Sqlite3Static; +let currentDbName: string; +let db: Database | OpfsDatabase; + +async function init( + fileName: string, + preferOpfs: boolean, + afterInit?: (sqliteDB: Database | OpfsDatabase) => Promise, +): Promise { + if (db && currentDbName === fileName) { + return; + } + + // only open new Db if there is no db opened or we want to open a different db + currentDbName = fileName; + + sqlite3 = await sqlite3InitModule(); + + db = preferOpfs && "opfs" in sqlite3.oo1 ? new sqlite3.oo1.OpfsDb(fileName) : new sqlite3.oo1.DB(fileName); + + await afterInit?.(db); +} + +function exec( + isSelect: boolean, + sql: string, + parameters?: readonly unknown[], +): QueryResult<{ + [columnName: string]: SqlValue; +}> { + if (isSelect) { + const rows = db.selectObjects(sql, parameters as BindingSpec); + + return { rows }; + } else { + db.exec(sql, { + bind: parameters as BindingSpec, + returnValue: "resultRows", + }); + + return { + rows: [], + insertId: BigInt(sqlite3.capi.sqlite3_last_insert_rowid(db)), + numAffectedRows: BigInt(db.changes()), + }; + } +} + +function stream( + onData: (data: { [columnName: string]: SqlValue }) => void, + sql: string, + parameters?: readonly unknown[], +): void { + const stmt = db.prepare(sql); + + if (parameters) { + stmt.bind(parameters as BindingSpec); + } + + while (stmt.step()) { + onData(stmt.get({})); + } + + stmt.finalize(); +} + +async function loadDb(onData: (percentage: number) => void, fileName: string, useOPFS: boolean, statements: string[]) { + if (!db) { + await init(fileName, useOPFS); + } + + const length = statements.length; + let percentage = 0; + + for (let i = 0; i < length; i++) { + const newPercentage = Math.round((i / length) * 100); + + if (newPercentage !== percentage) { + onData(newPercentage); + + percentage = newPercentage; + } + + db.exec(statements[i]); + } +} + +/** + * Handle worker message, support custom callback on initialization + * @example + * // worker.ts + * import { createOnMessageCallback, customFunction } from 'kysely-wasqlite-worker' + * + * onmessage = createOnMessageCallback( + * async (sqliteDB: SQLiteDB) => { + * customFunction(sqliteDB.sqlite, sqliteDB.db, 'customFunction', (a, b) => a + b) + * } + * ) + */ +export function createOnMessageCallback( + afterInit?: (sqliteDB: Database | OpfsDatabase) => Promise, +): (event: MessageEvent) => Promise { + return async ({ data: [msg, data1, data2, data3] }: MessageEvent) => { + const ret: WorkerToMainMsg = [msg, null, null]; + + try { + switch (msg) { + case 0: + await init(data1, data2, afterInit); + break; + case 1: + ret[1] = exec(data1, data2, data3); + break; + case 2: + db.close(); + break; + case 3: + stream((val) => postMessage([3, [val], null] satisfies WorkerToMainMsg), data1, data2); + ret[0] = 4; + break; + case 4: + await loadDb( + (percentage) => postMessage([5, percentage, null] satisfies WorkerToMainMsg), + data1, + data2, + data3, + ); + ret[0] = 6; + break; + } + } catch (error) { + ret[2] = error; + } + + postMessage(ret); + }; +} diff --git a/src/lib/kysely-wasqlite-worker/index.ts b/src/lib/kysely-wasqlite-worker/index.ts deleted file mode 100644 index c89dc2f..0000000 --- a/src/lib/kysely-wasqlite-worker/index.ts +++ /dev/null @@ -1,48 +0,0 @@ -import type { DatabaseIntrospector, Dialect, DialectAdapter, Driver, Kysely, QueryCompiler } from "kysely"; -import type { WaSqliteWorkerDialectConfig } from "./type"; -import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely"; -import { WaSqliteWorkerDriver } from "./driver"; - -export type { Promisable, WaSqliteWorkerDialectConfig } from "./type"; -export { createOnMessageCallback } from "./worker/utils"; - -export { - customFunction, - isIdbSupported, - isModuleWorkerSupport, - isOpfsSupported, - type SQLiteDB, -} from "@subframe7536/sqlite-wasm"; - -export class WaSqliteWorkerDialect implements Dialect { - /** - * dialect for [`wa-sqlite`](https://github.com/rhashimoto/wa-sqlite), - * execute sql in `Web Worker`, - * store data in [OPFS](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system) or IndexedDB - * - * @example - * import { WaSqliteWorkerDialect } from 'kysely-wasqlite-worker' - * - * const dialect = new WaSqliteWorkerDialect({ - * fileName: 'test', - * }) - */ - constructor(private config: WaSqliteWorkerDialectConfig) {} - - createDriver(): Driver { - return new WaSqliteWorkerDriver(this.config); - } - - createQueryCompiler(): QueryCompiler { - return new SqliteQueryCompiler(); - } - - createAdapter(): DialectAdapter { - return new SqliteAdapter(); - } - - // biome-ignore lint/suspicious/noExplicitAny: - createIntrospector(db: Kysely): DatabaseIntrospector { - return new SqliteIntrospector(db); - } -} diff --git a/src/lib/kysely-wasqlite-worker/type.ts b/src/lib/kysely-wasqlite-worker/type.ts deleted file mode 100644 index 28095cc..0000000 --- a/src/lib/kysely-wasqlite-worker/type.ts +++ /dev/null @@ -1,81 +0,0 @@ -import type { SqlValue } from "@sqlite.org/sqlite-wasm"; -import type { DatabaseConnection, QueryResult } from "kysely"; - -export type Promisable = T | Promise; - -export interface WaSqliteWorkerDialectConfig { - /** - * db file name - */ - fileName: string; - /** - * prefer to store data in OPFS - * @default true - */ - preferOPFS?: boolean; - /** - * wasqlite worker - * - * built-in: {@link useDefaultWorker} - * @param supportModuleWorker if support `{ type: 'module' }` in worker options - * @example - * import { useDefaultWorker } from 'kysely-wasqlite-worker' - * @example - * (supportModuleWorker) => supportModuleWorker - * ? new Worker( - * new URL('kysely-wasqlite-worker/worker-module', import.meta.url), - * { type: 'module', credentials: 'same-origin' } - * ) - * : new Worker( - * new URL('kysely-wasqlite-worker/worker-classic', import.meta.url), - * { type: 'classic', name: 'test' } - * ) - */ - worker?: Worker | ((supportModuleWorker: boolean) => Worker); - /** - * wasm URL - * - * built-in: {@link useDefaultWasmURL} - * @param useAsyncWasm if need to use wa-sqlite-async.wasm - * @example - * import { useDefaultWasmURL } from 'kysely-wasqlite-worker' - * @example - * (useAsyncWasm) => useAsyncWasm - * ? 'https://cdn.jsdelivr.net/gh/rhashimoto/wa-sqlite@v1.0.0/dist/wa-sqlite-async.wasm' - * : new URL('kysely-wasqlite-worker/wasm-sync', import.meta.url).href - */ - url?: string | ((useAsyncWasm: boolean) => string); - onCreateConnection?: (connection: DatabaseConnection) => Promisable; -} - -type RunMsg = [type: 1, isSelect: boolean, sql: string, parameters?: readonly unknown[]]; -type StreamMsg = [type: 3, sql: string, parameters?: readonly unknown[]]; - -type InitMsg = [type: 0, url: string, fileName: string, useOPFS: boolean]; - -type CloseMsg = [2]; - -type LoadDbMsg = [type: 4, filename: string, useOPFS: boolean, statements: string[]]; - -export type MainToWorkerMsg = InitMsg | RunMsg | CloseMsg | StreamMsg | LoadDbMsg; - -export type WorkerToMainMsg = { - [K in keyof Events]: [type: K, data: Events[K], err: unknown]; -}[keyof Events]; - -export type EventWithError = { - [K in keyof Events]: [data: Events[K], err: unknown]; -}; - -type Events = { - 0: null; - // biome-ignore lint/suspicious/noExplicitAny: - 1: QueryResult | null; - 2: null; - 3: { - [columnName: string]: SqlValue; - }[]; - 4: null; - 5: number; - 6: null; -}; diff --git a/src/lib/kysely-wasqlite-worker/utils.ts b/src/lib/kysely-wasqlite-worker/utils.ts deleted file mode 100644 index 9724a7e..0000000 --- a/src/lib/kysely-wasqlite-worker/utils.ts +++ /dev/null @@ -1,24 +0,0 @@ -import asyncWasmUrl from "~/assets/wa-sqlite-async.wasm?url"; -import syncWasmUrl from "~/assets/wa-sqlite.wasm?url"; - -export function parseWorkerOrURL T)>(obj: P, is: boolean): T { - return typeof obj === "function" ? obj(is) : obj; -} - -/** - * auto load target worker - * - * **only basic worker options** - */ -export function defaultWorker(support: boolean): Worker { - return support - ? new Worker(new URL("worker.mjs", import.meta.url), { type: "module" }) - : new Worker(new URL("worker.js", import.meta.url)); -} - -/** - * auto load target wasm - */ -export function defaultWasmURL(useAsyncWasm: boolean): string { - return useAsyncWasm ? asyncWasmUrl : syncWasmUrl; -} diff --git a/src/lib/kysely-wasqlite-worker/worker/utils.ts b/src/lib/kysely-wasqlite-worker/worker/utils.ts deleted file mode 100644 index 26b746c..0000000 --- a/src/lib/kysely-wasqlite-worker/worker/utils.ts +++ /dev/null @@ -1,110 +0,0 @@ -import type { SQLiteDB } from "@subframe7536/sqlite-wasm"; -import type { QueryResult } from "kysely"; -import type { MainToWorkerMsg, WorkerToMainMsg } from "../type"; -import { initSQLite } from "@subframe7536/sqlite-wasm"; -import { defaultWasmURL, parseWorkerOrURL } from "../utils"; - -let db: SQLiteDB; - -async function init( - fileName: string, - url: string, - useOPFS: boolean, - afterInit?: (sqliteDB: SQLiteDB) => Promise, -): Promise { - db = await initSQLite( - (useOPFS - ? (await import("@subframe7536/sqlite-wasm/opfs")).useOpfsStorage - : (await import("@subframe7536/sqlite-wasm/idb")).useIdbStorage)(fileName, { url }), - ); - await afterInit?.(db); -} - -// biome-ignore lint/suspicious/noExplicitAny: -async function exec(isSelect: boolean, sql: string, parameters?: readonly unknown[]): Promise> { - // biome-ignore lint/suspicious/noExplicitAny: - const rows = await db.run(sql, parameters as any[]); - return isSelect || rows.length - ? { rows } - : { - rows, - insertId: BigInt(db.lastInsertRowId()), - numAffectedRows: BigInt(db.changes()), - }; -} -// biome-ignore lint/suspicious/noExplicitAny: -async function stream(onData: (data: any) => void, sql: string, parameters?: readonly unknown[]): Promise { - // biome-ignore lint/suspicious/noExplicitAny: - await db.stream(onData, sql, parameters as any[]); -} - -async function loadDb(onData: (percentage: number) => void, fileName: string, useOPFS: boolean, statements: string[]) { - if (!db) { - await init(fileName, parseWorkerOrURL(defaultWasmURL, !useOPFS) as string, useOPFS); - } - - const length = statements.length; - let percentage = 0; - - for (let i = 0; i < length; i += 1000) { - const newPercentage = Math.round((i / length) * 100); - - if (newPercentage !== percentage) { - onData(newPercentage); - - percentage = newPercentage; - } - - console.log("executing statement"); - - await db.run(statements.slice(i, i + 1000).join(";")); - } - - // await db.run(statements.join(";")); -} - -/** - * Handle worker message, support custom callback on initialization - * @example - * // worker.ts - * import { createOnMessageCallback, customFunction } from 'kysely-wasqlite-worker' - * - * onmessage = createOnMessageCallback( - * async (sqliteDB: SQLiteDB) => { - * customFunction(sqliteDB.sqlite, sqliteDB.db, 'customFunction', (a, b) => a + b) - * } - * ) - */ -export function createOnMessageCallback( - afterInit?: (sqliteDB: SQLiteDB) => Promise, -): (event: MessageEvent) => Promise { - return async ({ data: [msg, data1, data2, data3] }: MessageEvent) => { - const ret: WorkerToMainMsg = [msg, null, null]; - - try { - switch (msg) { - case 0: - await init(data1, data2, data3, afterInit); - break; - case 1: - ret[1] = await exec(data1, data2, data3); - break; - case 2: - await db.close(); - break; - case 3: - await stream((val) => postMessage([3, [val], null] satisfies WorkerToMainMsg), data1, data2); - ret[0] = 4; - break; - case 4: - loadDb((percentage) => postMessage([5, percentage, null] satisfies WorkerToMainMsg), data1, data2, data3); - ret[0] = 6; - break; - } - } catch (error) { - console.error(error); - ret[2] = error; - } - postMessage(ret); - }; -} diff --git a/src/pages/home.tsx b/src/pages/home.tsx index d2e2405..5ca7fa3 100644 --- a/src/pages/home.tsx +++ b/src/pages/home.tsx @@ -6,7 +6,7 @@ import { Portal } from "solid-js/web"; import { Flex } from "~/components/ui/flex"; import { Progress, ProgressLabel, ProgressValueLabel } from "~/components/ui/progress"; -import { loadDb } from "~/db/db-queries"; +import { loadDb } from "~/db"; import { decryptBackup } from "~/lib/decryptor"; import { createDropzone, createFileUploader } from "@solid-primitives/upload"; import { Button } from "~/components/ui/button"; @@ -37,29 +37,38 @@ export const Home: Component = () => { const [loadingProgress, setLoadingProgress] = createSignal(); // const [isLoadingDatabase, setIsLoadingDatabase] = createSignal(false); - const onSubmit: JSX.EventHandler = (event) => { + const onSubmit: JSX.EventHandler = async (event) => { event.preventDefault(); const currentBackupFile = backupFile(); const currentPassphrase = passphrase(); if (currentBackupFile && currentPassphrase) { - decryptBackup(currentBackupFile, currentPassphrase, setDecryptionProgress) - .then(async (result) => { - setDecryptionProgress(undefined); - // setIsLoadingDatabase(true); - setLoadingProgress(0); + // const hashChunk = await currentBackupFile.slice(-1000).text(); + // const hash = hashString(hashChunk); - await loadDb(result.database_statements, (newValue) => (console.log(newValue), setLoadingProgress(newValue))); + // if (hash === dbHash()) { + // return; + // } - // setIsLoadingDatabase(false); - setLoadingProgress(undefined); + // setDbHash(hash); - navigate("/overview"); - }) - .catch((error) => { - console.error("Decryption failed:", error); - }); + try { + const decrypted = await decryptBackup(currentBackupFile, currentPassphrase, setDecryptionProgress); + + setDecryptionProgress(undefined); + // setIsLoadingDatabase(true); + setLoadingProgress(0); + + await loadDb(decrypted.database_statements, setLoadingProgress); + + // setIsLoadingDatabase(false); + setLoadingProgress(undefined); + + navigate("/overview"); + } catch (error) { + console.error("Decryption failed:", error); + } } }; diff --git a/src/pages/overview/index.tsx b/src/pages/overview/index.tsx index 424b24a..61e52b6 100644 --- a/src/pages/overview/index.tsx +++ b/src/pages/overview/index.tsx @@ -11,7 +11,9 @@ export const Overview: Component = () => { const [allSelfSentMessagesCount] = createResource(() => overallSentMessagesQuery(SELF_ID)); const [roomOverview] = createResource(async () => { - return (await allThreadsOverviewQuery())?.map((row) => { + const overview = await allThreadsOverviewQuery(); + + return overview.map((row) => { const isGroup = row.title !== null; let name = ""; @@ -41,9 +43,7 @@ export const Overview: Component = () => {

All messages: {allSelfSentMessagesCount()?.messageCount as number}

- {(currentRoomOverview) => ( - console.log(currentRoomOverview()), () - )} + {(currentRoomOverview) => }
diff --git a/src/pages/overview/overview-table.tsx b/src/pages/overview/overview-table.tsx index 2d72261..0defa76 100644 --- a/src/pages/overview/overview-table.tsx +++ b/src/pages/overview/overview-table.tsx @@ -26,6 +26,7 @@ import { TextField, TextFieldInput } from "~/components/ui/text-field"; import { cn } from "~/lib/utils"; import { Flex } from "~/components/ui/flex"; +import { dbLoaded, threadSentMessagesOverviewQuery } from "~/db"; export interface RoomOverview { threadId: number; @@ -55,6 +56,18 @@ const isGroupFilterFn: FilterFn = (row, _columnId, filterValue) => return !row.original.isGroup; }; +const rowIsAvailable = (threadId: number): boolean => { + if (dbLoaded()) { + return true; + } + + if (threadSentMessagesOverviewQuery.hasCacheFor(threadId)) { + return true; + } + + return false; +}; + const SortingDisplay: Component<{ sorting: false | SortDirection; class?: string; activeClass?: string }> = (props) => { return ( @@ -92,13 +105,14 @@ export const columns = [ cell: (props) => { const isArchived = props.row.getValue("archived"); const isGroup = props.row.getValue("isGroup"); + const isCached = !dbLoaded() && rowIsAvailable(props.row.original.threadId); return ( {props.cell.getValue()} - + @@ -110,6 +124,11 @@ export const columns = [ Group + + + Not available + + @@ -267,12 +286,12 @@ export const OverviewTable = (props: OverviewTableProps) => {
table.getColumn("isGroup")?.setFilterValue(value)} />
- +
@@ -315,32 +334,39 @@ export const OverviewTable = (props: OverviewTableProps) => { {(row) => ( { const threadId = row.original.threadId; const isGroup = row.original.isGroup; - const preloadTimeout = setTimeout(() => { - preload(`/${isGroup ? "group" : "dm"}/${threadId.toString()}`, { - preloadData: true, - }); - }, 20); + if (rowIsAvailable(threadId)) { + const preloadTimeout = setTimeout(() => { + preload(`/${isGroup ? "group" : "dm"}/${threadId.toString()}`, { + preloadData: true, + }); + }, 20); - event.currentTarget.addEventListener( - "pointerout", - () => { - clearTimeout(preloadTimeout); - }, - { - once: true, - }, - ); + event.currentTarget.addEventListener( + "pointerout", + () => { + clearTimeout(preloadTimeout); + }, + { + once: true, + }, + ); + } }} onClick={() => { const threadId = row.original.threadId; const isGroup = row.original.isGroup; - navigate(`/${isGroup ? "group" : "dm"}/${threadId.toString()}`); + if (rowIsAvailable(threadId)) { + navigate(`/${isGroup ? "group" : "dm"}/${threadId.toString()}`); + } }} >