feat: use official wasm in worker and cached data info / warning

src/components/ui/A.tsx

@@ -0,0 +1,8 @@
+import { splitProps, type Component } from "solid-js";
+import { type AnchorProps, A as BaseA } from "@solidjs/router";
+import clsx from "clsx";
+
+export const A: Component<AnchorProps> = (props) => {
+  const [local, other] = splitProps(props, ["class"]);
+  return <BaseA class={clsx("text-primary", local.class)} {...other} />;
+};

src/components/ui/callout.tsx

@@ -0,0 +1,40 @@
+import type { Component, ComponentProps } from "solid-js"
+import { splitProps } from "solid-js"
+
+import type { VariantProps } from "class-variance-authority"
+import { cva } from "class-variance-authority"
+
+import { cn } from "~/lib/utils"
+
+const calloutVariants = cva("rounded-md border-l-4 p-2 pl-4", {
+  variants: {
+    variant: {
+      default: "border-info-foreground bg-info text-info-foreground",
+      success: "border-success-foreground bg-success text-success-foreground",
+      warning: "border-warning-foreground bg-warning text-warning-foreground",
+      error: "border-error-foreground bg-error text-error-foreground"
+    }
+  },
+  defaultVariants: {
+    variant: "default"
+  }
+})
+
+type CalloutProps = ComponentProps<"div"> & VariantProps<typeof calloutVariants>
+
+const Callout: Component<CalloutProps> = (props) => {
+  const [local, others] = splitProps(props, ["class", "variant"])
+  return <div class={cn(calloutVariants({ variant: local.variant }), local.class)} {...others} />
+}
+
+const CalloutTitle: Component<ComponentProps<"h3">> = (props) => {
+  const [local, others] = splitProps(props, ["class"])
+  return <h3 class={cn("font-semibold", local.class)} {...others} />
+}
+
+const CalloutContent: Component<ComponentProps<"div">> = (props) => {
+  const [local, others] = splitProps(props, ["class"])
+  return <div class={cn("mt-2", local.class)} {...others} />
+}
+
+export { Callout, CalloutTitle, CalloutContent }

src/db/db-queries.ts

@@ -1,9 +1,19 @@
 import { sql, type NotNull } from "kysely";
-import { worker, kyselyDb, SELF_ID, DB_FILENAME } from "./db";
+import { worker, kyselyDb, SELF_ID, DB_FILENAME, setDbLoaded } from "./db";
-import { cached } from "../lib/db-cache";
+import { cached, clearDbCache } from "../lib/db-cache";
-import type { MainToWorkerMsg, WorkerToMainMsg } from "~/lib/kysely-wasqlite-worker/type";
+import type { MainToWorkerMsg, WorkerToMainMsg } from "~/lib/kysely-official-wasm-worker/type";
+
+export const loadDb = async (statements: string[], progressCallback?: (percentage: number) => void): Promise<void> => {
+  // try to persist storage, https://web.dev/articles/persistent-storage#request_persistent_storage
+  try {
+    if (navigator.storage?.persist && !(await navigator.storage.persisted())) {
+      await navigator.storage.persist();
+    }
+    // biome-ignore lint/suspicious/noEmptyBlockStatements: <explanation>
+  } catch {}
+
+  clearDbCache();
 
-export const loadDb = (statements: string[], progressCallback?: (percentage: number) => void): Promise<void> => {
   return new Promise((resolve, reject) => {
     const progressListener = ({ data }: MessageEvent<WorkerToMainMsg>) => {
       if (data[0] === 5) {
@@ -19,12 +29,15 @@ export const loadDb = (statements: string[], progressCallback?: (percentage: num
 
         worker.removeEventListener("message", progressListener);
         worker.removeEventListener("message", endListener);
+
+        setDbLoaded(true);
+
         resolve();
       }
     };
 
-    worker.addEventListener("message", progressListener);
     worker.addEventListener("message", endListener);
+    worker.addEventListener("message", progressListener);
 
     worker.postMessage([4, DB_FILENAME, true, statements] satisfies MainToWorkerMsg);
   });

src/db/db.ts

@@ -1,9 +1,8 @@
-import { makePersisted } from "@solid-primitives/storage";
 import { Kysely } from "kysely";
 import type { DB } from "./db-schema";
+import { OfficialWasmWorkerDialect } from "~/lib/kysely-official-wasm-worker";
+import wasmWorkerUrl from "~/lib/kysely-official-wasm-worker/worker?url";
 import { createSignal } from "solid-js";
-import { WaSqliteWorkerDialect } from "~/lib/kysely-wasqlite-worker";
-import wasmWorkerUrl from "~/lib/kysely-wasqlite-worker/worker?url";
 
 export const SELF_ID = 2;
 
@@ -13,7 +12,7 @@ export const worker = new Worker(wasmWorkerUrl, {
   type: "module",
 });
 
-const dialect = new WaSqliteWorkerDialect({
+const dialect = new OfficialWasmWorkerDialect({
   fileName: DB_FILENAME,
   preferOPFS: true,
   worker,
@@ -23,4 +22,5 @@ export const kyselyDb = new Kysely<DB>({
   dialect,
 });
 
-export const [dbHash, setDbHash] = makePersisted(createSignal<number>());
+export const [dbLoaded, setDbLoaded] = createSignal(false);
+// export const [dbHash, setDbHash] = makePersisted(createSignal<number>());

src/db/index.ts

@@ -1,2 +1,2 @@
-export { kyselyDb, SELF_ID } from "./db";
+export { kyselyDb, SELF_ID, dbLoaded, setDbLoaded } from "./db";
 export * from "./db-queries";

src/index.tsx

@@ -1,8 +1,13 @@
 /* @refresh reload */
 import { MetaProvider } from "@solidjs/meta";
-import { Router } from "@solidjs/router";
+import { Router, useNavigate } from "@solidjs/router";
 import { render } from "solid-js/web";
 import App from "./App";
+import { hasCashedData } from "./lib/db-cache";
+import { createEffect, Show } from "solid-js";
+import { dbLoaded } from "./db";
+import { Callout, CalloutContent, CalloutTitle } from "./components/ui/callout";
+import { A } from "./components/ui/A";
 
 const root = document.getElementById("root");
 
@@ -18,18 +23,44 @@ if (root) {
       <div class="mx-auto max-w-screen-2xl">
         <MetaProvider>
           <Router
-          // root={(props) => {
+            root={(props) => {
-          //   const navigate = useNavigate();
+              const navigate = useNavigate();
-          //   const { pathname } = props.location;
 
-          //   createEffect(() => {
+              createEffect(() => {
-          //     if (!db && pathname !== "/") {
+                if (!dbLoaded() && !hasCashedData() && props.location.pathname !== "/") {
-          //       navigate("/");
+                  navigate("/");
-          //     }
+                }
-          //   });
+              });
 
-          //   return props.children;
+              return (
-          // }}
+                <>
+                  <Show
+                    when={props.location.pathname !== "/" && !dbLoaded() && hasCashedData()}
+                    fallback={
+                      <Show when={!dbLoaded() && hasCashedData()}>
+                        <Callout variant="default" class="my-4">
+                          There is currently no backup database loaded, but you can watch statistics that have been
+                          cached, meaning only chats you only watched or were preloaded.
+                          <br />
+                          <A href="/overview">Watch cached statistics</A>
+                        </Callout>
+                      </Show>
+                    }
+                  >
+                    <Callout variant="warning" class="my-4">
+                      <CalloutTitle>You are watching cached statistics</CalloutTitle>
+                      <CalloutContent>
+                        Currently there is no backup database loaded. You can only watch statistics that have been
+                        cached, meaning only chats you only watched or were preloaded.
+                        <br />
+                        <A href="/">Load a backup</A>
+                      </CalloutContent>
+                    </Callout>
+                  </Show>
+                  {props.children}
+                </>
+              );
+            }}
           >
             <App />
           </Router>

src/lib/db-cache.ts

@@ -1,47 +1,36 @@
 import { deserialize, serialize } from "seroval";
-import {} from "solid-js";
 import { hashString } from "./hash";
 
 export const DATABASE_HASH_PREFIX = "database";
 
+export const hasCashedData = () => {
+  for (let i = 0, len = localStorage.length; i < len; i++) {
+    const key = localStorage.key(i);
+
+    if (key?.startsWith(DATABASE_HASH_PREFIX)) {
+      return true;
+    }
+  }
+
+  return false;
+};
+
 // clear the cache on new session so that selecting a different database does not result in wrong cache entries
-// const clearDbCache = () => {
+export const clearDbCache = () => {
-//   for (let i = 0, len = localStorage.length; i < len; i++) {
+  for (let i = 0, len = localStorage.length; i < len; i++) {
-//     const key = localStorage.key(i);
+    const key = localStorage.key(i);
 
-//     if (key?.startsWith(DATABASE_HASH_PREFIX)) {
+    if (key?.startsWith(DATABASE_HASH_PREFIX)) {
-//       localStorage.removeItem(key);
+      localStorage.removeItem(key);
-//     }
+    }
-//   }
+  }
-// };
+};
-
-// let prevDbHash = dbHash();
-
-// createRoot(() => {
-//   createEffect(() => {
-//     on(
-//       dbHash,
-//       (currentDbHash) => {
-//         if (currentDbHash && currentDbHash !== prevDbHash) {
-//           prevDbHash = currentDbHash;
-//           clearDbCache();
-//         }
-//       },
-//       {
-//         defer: true,
-//       },
-//     );
-//   });
-// });
 
 class LocalStorageCacheAdapter {
-  keys = new Set<string>(Object.keys(localStorage).filter((key) => key.startsWith(this.prefix)));
+  keys = new Set<string>(Object.keys(localStorage).filter((key) => key.startsWith(DATABASE_HASH_PREFIX)));
-  prefix = "database";
-  // TODO: real way of detecting if the db is loaded, on loading the db and opfs (if persisted db?)
-  // #dbLoaded = createMemo(() => !!dbHash());
 
   #createKey(cacheName: string, key: string): string {
-    return `${this.prefix}-${cacheName}-${key}`;
+    return `${DATABASE_HASH_PREFIX}-${cacheName}-${key}`;
   }
 
   set(cacheName: string, key: string, value: unknown, isPromise = false) {
@@ -60,13 +49,7 @@ class LocalStorageCacheAdapter {
   }
 
   has(cacheName: string, key: string): boolean {
-    // if (this.#dbLoaded()) {
     return this.keys.has(this.#createKey(cacheName, key));
-    // }
-
-    // console.info("No database loaded");
-
-    // return false;
   }
 
   get<R>(
@@ -78,7 +61,6 @@ class LocalStorageCacheAdapter {
         value: R;
       }
     | undefined {
-    // if (this.#dbLoaded()) {
     const item = localStorage.getItem(this.#createKey(cacheName, key));
 
     if (item) {
@@ -87,9 +69,6 @@ class LocalStorageCacheAdapter {
         value: R;
       };
     }
-    // } else {
-    //   console.info("No database loaded");
-    // }
   }
 }
 
@@ -122,10 +101,14 @@ const createHashKey = (...args: unknown[]) => {
   return hashString(stringToHash);
 };
 
-export const cached = <T extends unknown[], R, TT>(fn: (...args: T) => R, self?: ThisType<TT>): ((...args: T) => R) => {
+type CachedFn<T extends unknown[], R> = ((...args: T) => R) & {
+  hasCacheFor: (...args: T) => boolean;
+};
+
+export const cached = <T extends unknown[], R, TT>(fn: (...args: T) => R, self?: ThisType<TT>): CachedFn<T, R> => {
   const cacheName = hashString(fn.toString()).toString();
 
-  return (...args: T) => {
+  const cachedFn: CachedFn<T, R> = (...args: T) => {
     const cacheKey = createHashKey(...args).toString();
 
     const cachedValue = cache.get<R>(cacheName, cacheKey);
@@ -154,4 +137,12 @@ export const cached = <T extends unknown[], R, TT>(fn: (...args: T) => R, self?:
 
     return newValue;
   };
+
+  cachedFn.hasCacheFor = (...args: T) => {
+    const cacheKey = createHashKey(...args).toString();
+
+    return cache.has(cacheName, cacheKey);
+  };
+
+  return cachedFn;
 };

src/lib/kysely-official-wasm-worker/driver.ts

@@ -1,52 +1,45 @@
 import type { DatabaseConnection, Driver, QueryResult } from "kysely";
-import type { Emitter } from "zen-mitt";
-import type { EventWithError, MainToWorkerMsg, WaSqliteWorkerDialectConfig, WorkerToMainMsg } from "./type";
-import { isModuleWorkerSupport, isOpfsSupported } from "@subframe7536/sqlite-wasm";
 import { CompiledQuery, SelectQueryNode } from "kysely";
+import type { Emitter } from "zen-mitt";
 import { mitt } from "zen-mitt";
-import { defaultWasmURL, defaultWorker, parseWorkerOrURL } from "./utils";
+import type { EventWithError, MainToWorkerMsg, OfficialWasmWorkerDialectConfig, WorkerToMainMsg } from "./type";
+import workerUrl from "./worker?url";
 
-export class WaSqliteWorkerDriver implements Driver {
+export class OfficialWasmWorkerDriver implements Driver {
   private worker?: Worker;
   private connection?: DatabaseConnection;
   private connectionMutex = new ConnectionMutex();
   private mitt?: Emitter<EventWithError>;
-  constructor(private config: WaSqliteWorkerDialectConfig) {}
+  constructor(private config: OfficialWasmWorkerDialectConfig) {}
 
   async init(): Promise<void> {
     // try to persist storage, https://web.dev/articles/persistent-storage#request_persistent_storage
     try {
-      if (!(await navigator.storage.persisted())) {
+      if (navigator.storage?.persist && !(await navigator.storage.persisted())) {
         await navigator.storage.persist();
       }
       // biome-ignore lint/suspicious/noEmptyBlockStatements: <explanation>
     } catch {}
 
-    const useOPFS = (this.config.preferOPFS ?? true) ? await isOpfsSupported() : false;
-
     this.mitt = mitt<EventWithError>();
 
-    this.worker = parseWorkerOrURL(this.config.worker || defaultWorker, useOPFS || isModuleWorkerSupport());
+    this.worker =
+      this.config.worker ??
+      new Worker(workerUrl, {
+        type: "module",
+      });
 
-    // biome-ignore lint/style/noNonNullAssertion: <explanation>
+    this.worker.onmessage = ({ data: [type, ...msg] }: MessageEvent<WorkerToMainMsg>) => {
-    this.worker!.onmessage = ({ data: [type, ...msg] }: MessageEvent<WorkerToMainMsg>) => {
       this.mitt?.emit(type, ...msg);
     };
 
-    this.worker?.postMessage([
+    this.worker.postMessage([0, this.config.fileName, this.config.preferOPFS ?? false] satisfies MainToWorkerMsg);
-      0,
-      this.config.fileName,
-      // if use OPFS, wasm should use sync version
-      parseWorkerOrURL(this.config.url ?? defaultWasmURL, !useOPFS) as string,
-      useOPFS,
-    ] satisfies MainToWorkerMsg);
 
     await new Promise<void>((resolve, reject) => {
       this.mitt?.once(0, (_, err) => (err ? reject(err) : resolve()));
     });
 
-    // biome-ignore lint/style/noNonNullAssertion: <explanation>
+    this.connection = new OfficialWasmWorkerConnection(this.worker, this.mitt);
-    this.connection = new WaSqliteWorkerConnection(this.worker!, this.mitt);
     await this.config.onCreateConnection?.(this.connection);
   }
 
@@ -54,6 +47,7 @@ export class WaSqliteWorkerDriver implements Driver {
     // SQLite only has one single connection. We use a mutex here to wait
     // until the single connection has been released.
     await this.connectionMutex.lock();
+
     // biome-ignore lint/style/noNonNullAssertion: <explanation>
     return this.connection!;
   }
@@ -70,9 +64,12 @@ export class WaSqliteWorkerDriver implements Driver {
     await connection.executeQuery(CompiledQuery.raw("rollback"));
   }
 
-  // biome-ignore lint/suspicious/useAwait: <explanation>
+  releaseConnection(): Promise<void> {
-  async releaseConnection(): Promise<void> {
+    return new Promise((resolve) => {
-    this.connectionMutex.unlock();
+      this.connectionMutex.unlock();
+
+      resolve();
+    });
   }
 
   async destroy(): Promise<void> {
@@ -119,7 +116,7 @@ class ConnectionMutex {
   }
 }
 
-class WaSqliteWorkerConnection implements DatabaseConnection {
+class OfficialWasmWorkerConnection implements DatabaseConnection {
   readonly worker: Worker;
   readonly mitt?: Emitter<EventWithError>;
   constructor(worker: Worker, mitt?: Emitter<EventWithError>) {
@@ -130,20 +127,18 @@ class WaSqliteWorkerConnection implements DatabaseConnection {
   async *streamQuery<R>(compiledQuery: CompiledQuery): AsyncIterableIterator<QueryResult<R>> {
     const { parameters, sql, query } = compiledQuery;
     if (!SelectQueryNode.is(query)) {
-      throw new Error("WaSqlite dialect only supported SELECT queries");
+      throw new Error("official wasm worker dialect only supports SELECT queries for streaming");
     }
     this.worker.postMessage([3, sql, parameters] satisfies MainToWorkerMsg);
     let done = false;
     let resolveFn: (value: IteratorResult<QueryResult<R>>) => void;
-    // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+    let rejectFn: (reason?: unknown) => void;
-    let rejectFn: (reason?: any) => void;
 
     this.mitt?.on(3 /* data */, (data, err): void => {
       if (err) {
         rejectFn(err);
       } else {
-        // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+        resolveFn({ value: { rows: data as R[] }, done: false });
-        resolveFn({ value: { rows: data as any }, done: false });
       }
     });
 
@@ -172,9 +167,12 @@ class WaSqliteWorkerConnection implements DatabaseConnection {
   }
 
   async executeQuery<R>(compiledQuery: CompiledQuery<unknown>): Promise<QueryResult<R>> {
-    const { parameters, sql, query } = compiledQuery;
+    const { sql, parameters, query } = compiledQuery;
+
     const isSelect = SelectQueryNode.is(query);
+
     this.worker.postMessage([1, isSelect, sql, parameters] satisfies MainToWorkerMsg);
+
     return new Promise((resolve, reject) => {
       if (!this.mitt) {
         reject(new Error("kysely instance has been destroyed"));

src/lib/kysely-official-wasm-worker/index.ts

@@ -0,0 +1,38 @@
+import type {
+  DatabaseIntrospector,
+  Dialect,
+  DialectAdapter,
+  Driver,
+  Kysely,
+  QueryCompiler,
+} from "kysely";
+import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely";
+import { OfficialWasmWorkerDriver } from "./driver";
+import type { OfficialWasmWorkerDialectConfig } from "./type";
+
+export type {
+  Promisable,
+  OfficialWasmWorkerDialectConfig as WaSqliteWorkerDialectConfig,
+} from "./type";
+export { createOnMessageCallback } from "./worker/utils";
+
+export class OfficialWasmWorkerDialect implements Dialect {
+  constructor(private config: OfficialWasmWorkerDialectConfig) {}
+
+  createDriver(): Driver {
+    return new OfficialWasmWorkerDriver(this.config);
+  }
+
+  createQueryCompiler(): QueryCompiler {
+    return new SqliteQueryCompiler();
+  }
+
+  createAdapter(): DialectAdapter {
+    return new SqliteAdapter();
+  }
+
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  createIntrospector(db: Kysely<any>): DatabaseIntrospector {
+    return new SqliteIntrospector(db);
+  }
+}

src/lib/kysely-official-wasm-worker/type.ts

@@ -0,0 +1,56 @@
+import type { SqlValue } from "@sqlite.org/sqlite-wasm";
+import type { DatabaseConnection, QueryResult } from "kysely";
+
+export type Promisable<T> = T | Promise<T>;
+
+export interface OfficialWasmWorkerDialectConfig {
+  /**
+   * db file name
+   */
+  fileName: string;
+  /**
+   * prefer to store data in OPFS
+   * @default true
+   */
+  preferOPFS?: boolean;
+  /**
+   * official wasm worker
+   */
+  worker?: Worker;
+  onCreateConnection?: (connection: DatabaseConnection) => Promisable<void>;
+}
+
+type InitMsg = [type: 0, fileName: string, useOPFS: boolean];
+
+type RunMsg = [type: 1, isSelect: boolean, sql: string, parameters?: readonly unknown[]];
+
+type CloseMsg = [2];
+
+type StreamMsg = [type: 3, sql: string, parameters?: readonly unknown[]];
+
+type LoadDbMsg = [type: 4, filename: string, useOPFS: boolean, statements: string[]];
+
+type IsInitMsg = [type: 5];
+
+export type MainToWorkerMsg = InitMsg | RunMsg | CloseMsg | StreamMsg | LoadDbMsg | IsInitMsg;
+
+type Events = {
+  0: null;
+  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
+  1: QueryResult<any> | null;
+  2: null;
+  3: {
+    [columnName: string]: SqlValue;
+  }[];
+  4: null;
+  5: number;
+  6: null;
+};
+
+export type WorkerToMainMsg = {
+  [K in keyof Events]: [type: K, data: Events[K], err: unknown];
+}[keyof Events];
+
+export type EventWithError = {
+  [K in keyof Events]: [data: Events[K], err: unknown];
+};

src/lib/kysely-official-wasm-worker/worker/utils.ts

@@ -0,0 +1,147 @@
+import sqlite3InitModule, {
+  type BindingSpec,
+  type Database,
+  type OpfsDatabase,
+  type Sqlite3Static,
+  type SqlValue,
+} from "@sqlite.org/sqlite-wasm";
+import type { QueryResult } from "kysely";
+import type { MainToWorkerMsg, WorkerToMainMsg } from "../type";
+
+let sqlite3: Sqlite3Static;
+let currentDbName: string;
+let db: Database | OpfsDatabase;
+
+async function init(
+  fileName: string,
+  preferOpfs: boolean,
+  afterInit?: (sqliteDB: Database | OpfsDatabase) => Promise<void>,
+): Promise<void> {
+  if (db && currentDbName === fileName) {
+    return;
+  }
+
+  // only open new Db if there is no db opened or we want to open a different db
+  currentDbName = fileName;
+
+  sqlite3 = await sqlite3InitModule();
+
+  db = preferOpfs && "opfs" in sqlite3.oo1 ? new sqlite3.oo1.OpfsDb(fileName) : new sqlite3.oo1.DB(fileName);
+
+  await afterInit?.(db);
+}
+
+function exec(
+  isSelect: boolean,
+  sql: string,
+  parameters?: readonly unknown[],
+): QueryResult<{
+  [columnName: string]: SqlValue;
+}> {
+  if (isSelect) {
+    const rows = db.selectObjects(sql, parameters as BindingSpec);
+
+    return { rows };
+  } else {
+    db.exec(sql, {
+      bind: parameters as BindingSpec,
+      returnValue: "resultRows",
+    });
+
+    return {
+      rows: [],
+      insertId: BigInt(sqlite3.capi.sqlite3_last_insert_rowid(db)),
+      numAffectedRows: BigInt(db.changes()),
+    };
+  }
+}
+
+function stream(
+  onData: (data: { [columnName: string]: SqlValue }) => void,
+  sql: string,
+  parameters?: readonly unknown[],
+): void {
+  const stmt = db.prepare(sql);
+
+  if (parameters) {
+    stmt.bind(parameters as BindingSpec);
+  }
+
+  while (stmt.step()) {
+    onData(stmt.get({}));
+  }
+
+  stmt.finalize();
+}
+
+async function loadDb(onData: (percentage: number) => void, fileName: string, useOPFS: boolean, statements: string[]) {
+  if (!db) {
+    await init(fileName, useOPFS);
+  }
+
+  const length = statements.length;
+  let percentage = 0;
+
+  for (let i = 0; i < length; i++) {
+    const newPercentage = Math.round((i / length) * 100);
+
+    if (newPercentage !== percentage) {
+      onData(newPercentage);
+
+      percentage = newPercentage;
+    }
+
+    db.exec(statements[i]);
+  }
+}
+
+/**
+ * Handle worker message, support custom callback on initialization
+ * @example
+ * // worker.ts
+ * import { createOnMessageCallback, customFunction } from 'kysely-wasqlite-worker'
+ *
+ * onmessage = createOnMessageCallback(
+ *   async (sqliteDB: SQLiteDB) => {
+ *     customFunction(sqliteDB.sqlite, sqliteDB.db, 'customFunction', (a, b) => a + b)
+ *   }
+ * )
+ */
+export function createOnMessageCallback(
+  afterInit?: (sqliteDB: Database | OpfsDatabase) => Promise<void>,
+): (event: MessageEvent<MainToWorkerMsg>) => Promise<void> {
+  return async ({ data: [msg, data1, data2, data3] }: MessageEvent<MainToWorkerMsg>) => {
+    const ret: WorkerToMainMsg = [msg, null, null];
+
+    try {
+      switch (msg) {
+        case 0:
+          await init(data1, data2, afterInit);
+          break;
+        case 1:
+          ret[1] = exec(data1, data2, data3);
+          break;
+        case 2:
+          db.close();
+          break;
+        case 3:
+          stream((val) => postMessage([3, [val], null] satisfies WorkerToMainMsg), data1, data2);
+          ret[0] = 4;
+          break;
+        case 4:
+          await loadDb(
+            (percentage) => postMessage([5, percentage, null] satisfies WorkerToMainMsg),
+            data1,
+            data2,
+            data3,
+          );
+          ret[0] = 6;
+          break;
+      }
+    } catch (error) {
+      ret[2] = error;
+    }
+
+    postMessage(ret);
+  };
+}

src/lib/kysely-wasqlite-worker/index.ts

@@ -1,48 +0,0 @@
-import type { DatabaseIntrospector, Dialect, DialectAdapter, Driver, Kysely, QueryCompiler } from "kysely";
-import type { WaSqliteWorkerDialectConfig } from "./type";
-import { SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from "kysely";
-import { WaSqliteWorkerDriver } from "./driver";
-
-export type { Promisable, WaSqliteWorkerDialectConfig } from "./type";
-export { createOnMessageCallback } from "./worker/utils";
-
-export {
-  customFunction,
-  isIdbSupported,
-  isModuleWorkerSupport,
-  isOpfsSupported,
-  type SQLiteDB,
-} from "@subframe7536/sqlite-wasm";
-
-export class WaSqliteWorkerDialect implements Dialect {
-  /**
-   * dialect for [`wa-sqlite`](https://github.com/rhashimoto/wa-sqlite),
-   * execute sql in `Web Worker`,
-   * store data in [OPFS](https://developer.mozilla.org/en-US/docs/Web/API/File_System_API/Origin_private_file_system) or IndexedDB
-   *
-   * @example
-   * import { WaSqliteWorkerDialect } from 'kysely-wasqlite-worker'
-   *
-   * const dialect = new WaSqliteWorkerDialect({
-   *   fileName: 'test',
-   * })
-   */
-  constructor(private config: WaSqliteWorkerDialectConfig) {}
-
-  createDriver(): Driver {
-    return new WaSqliteWorkerDriver(this.config);
-  }
-
-  createQueryCompiler(): QueryCompiler {
-    return new SqliteQueryCompiler();
-  }
-
-  createAdapter(): DialectAdapter {
-    return new SqliteAdapter();
-  }
-
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-  createIntrospector(db: Kysely<any>): DatabaseIntrospector {
-    return new SqliteIntrospector(db);
-  }
-}

src/lib/kysely-wasqlite-worker/type.ts

@@ -1,81 +0,0 @@
-import type { SqlValue } from "@sqlite.org/sqlite-wasm";
-import type { DatabaseConnection, QueryResult } from "kysely";
-
-export type Promisable<T> = T | Promise<T>;
-
-export interface WaSqliteWorkerDialectConfig {
-  /**
-   * db file name
-   */
-  fileName: string;
-  /**
-   * prefer to store data in OPFS
-   * @default true
-   */
-  preferOPFS?: boolean;
-  /**
-   * wasqlite worker
-   *
-   * built-in: {@link useDefaultWorker}
-   * @param supportModuleWorker if support `{ type: 'module' }` in worker options
-   * @example
-   * import { useDefaultWorker } from 'kysely-wasqlite-worker'
-   * @example
-   * (supportModuleWorker) => supportModuleWorker
-   *   ? new Worker(
-   *       new URL('kysely-wasqlite-worker/worker-module', import.meta.url),
-   *       { type: 'module', credentials: 'same-origin' }
-   *     )
-   *   : new Worker(
-   *       new URL('kysely-wasqlite-worker/worker-classic', import.meta.url),
-   *       { type: 'classic', name: 'test' }
-   *     )
-   */
-  worker?: Worker | ((supportModuleWorker: boolean) => Worker);
-  /**
-   * wasm URL
-   *
-   * built-in: {@link useDefaultWasmURL}
-   * @param useAsyncWasm if need to use wa-sqlite-async.wasm
-   * @example
-   * import { useDefaultWasmURL } from 'kysely-wasqlite-worker'
-   * @example
-   * (useAsyncWasm) => useAsyncWasm
-   *   ? 'https://cdn.jsdelivr.net/gh/rhashimoto/wa-sqlite@v1.0.0/dist/wa-sqlite-async.wasm'
-   *   : new URL('kysely-wasqlite-worker/wasm-sync', import.meta.url).href
-   */
-  url?: string | ((useAsyncWasm: boolean) => string);
-  onCreateConnection?: (connection: DatabaseConnection) => Promisable<void>;
-}
-
-type RunMsg = [type: 1, isSelect: boolean, sql: string, parameters?: readonly unknown[]];
-type StreamMsg = [type: 3, sql: string, parameters?: readonly unknown[]];
-
-type InitMsg = [type: 0, url: string, fileName: string, useOPFS: boolean];
-
-type CloseMsg = [2];
-
-type LoadDbMsg = [type: 4, filename: string, useOPFS: boolean, statements: string[]];
-
-export type MainToWorkerMsg = InitMsg | RunMsg | CloseMsg | StreamMsg | LoadDbMsg;
-
-export type WorkerToMainMsg = {
-  [K in keyof Events]: [type: K, data: Events[K], err: unknown];
-}[keyof Events];
-
-export type EventWithError = {
-  [K in keyof Events]: [data: Events[K], err: unknown];
-};
-
-type Events = {
-  0: null;
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-  1: QueryResult<any> | null;
-  2: null;
-  3: {
-    [columnName: string]: SqlValue;
-  }[];
-  4: null;
-  5: number;
-  6: null;
-};

src/lib/kysely-wasqlite-worker/utils.ts

@@ -1,24 +0,0 @@
-import asyncWasmUrl from "~/assets/wa-sqlite-async.wasm?url";
-import syncWasmUrl from "~/assets/wa-sqlite.wasm?url";
-
-export function parseWorkerOrURL<T, P extends T | ((is: boolean) => T)>(obj: P, is: boolean): T {
-  return typeof obj === "function" ? obj(is) : obj;
-}
-
-/**
- * auto load target worker
- *
- * **only basic worker options**
- */
-export function defaultWorker(support: boolean): Worker {
-  return support
-    ? new Worker(new URL("worker.mjs", import.meta.url), { type: "module" })
-    : new Worker(new URL("worker.js", import.meta.url));
-}
-
-/**
- * auto load target wasm
- */
-export function defaultWasmURL(useAsyncWasm: boolean): string {
-  return useAsyncWasm ? asyncWasmUrl : syncWasmUrl;
-}

src/lib/kysely-wasqlite-worker/worker/utils.ts

@@ -1,110 +0,0 @@
-import type { SQLiteDB } from "@subframe7536/sqlite-wasm";
-import type { QueryResult } from "kysely";
-import type { MainToWorkerMsg, WorkerToMainMsg } from "../type";
-import { initSQLite } from "@subframe7536/sqlite-wasm";
-import { defaultWasmURL, parseWorkerOrURL } from "../utils";
-
-let db: SQLiteDB;
-
-async function init(
-  fileName: string,
-  url: string,
-  useOPFS: boolean,
-  afterInit?: (sqliteDB: SQLiteDB) => Promise<void>,
-): Promise<void> {
-  db = await initSQLite(
-    (useOPFS
-      ? (await import("@subframe7536/sqlite-wasm/opfs")).useOpfsStorage
-      : (await import("@subframe7536/sqlite-wasm/idb")).useIdbStorage)(fileName, { url }),
-  );
-  await afterInit?.(db);
-}
-
-// biome-ignore lint/suspicious/noExplicitAny: <explanation>
-async function exec(isSelect: boolean, sql: string, parameters?: readonly unknown[]): Promise<QueryResult<any>> {
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-  const rows = await db.run(sql, parameters as any[]);
-  return isSelect || rows.length
-    ? { rows }
-    : {
-        rows,
-        insertId: BigInt(db.lastInsertRowId()),
-        numAffectedRows: BigInt(db.changes()),
-      };
-}
-// biome-ignore lint/suspicious/noExplicitAny: <explanation>
-async function stream(onData: (data: any) => void, sql: string, parameters?: readonly unknown[]): Promise<void> {
-  // biome-ignore lint/suspicious/noExplicitAny: <explanation>
-  await db.stream(onData, sql, parameters as any[]);
-}
-
-async function loadDb(onData: (percentage: number) => void, fileName: string, useOPFS: boolean, statements: string[]) {
-  if (!db) {
-    await init(fileName, parseWorkerOrURL(defaultWasmURL, !useOPFS) as string, useOPFS);
-  }
-
-  const length = statements.length;
-  let percentage = 0;
-
-  for (let i = 0; i < length; i += 1000) {
-    const newPercentage = Math.round((i / length) * 100);
-
-    if (newPercentage !== percentage) {
-      onData(newPercentage);
-
-      percentage = newPercentage;
-    }
-
-    console.log("executing statement");
-
-    await db.run(statements.slice(i, i + 1000).join(";"));
-  }
-
-  // await db.run(statements.join(";"));
-}
-
-/**
- * Handle worker message, support custom callback on initialization
- * @example
- * // worker.ts
- * import { createOnMessageCallback, customFunction } from 'kysely-wasqlite-worker'
- *
- * onmessage = createOnMessageCallback(
- *   async (sqliteDB: SQLiteDB) => {
- *     customFunction(sqliteDB.sqlite, sqliteDB.db, 'customFunction', (a, b) => a + b)
- *   }
- * )
- */
-export function createOnMessageCallback(
-  afterInit?: (sqliteDB: SQLiteDB) => Promise<void>,
-): (event: MessageEvent<MainToWorkerMsg>) => Promise<void> {
-  return async ({ data: [msg, data1, data2, data3] }: MessageEvent<MainToWorkerMsg>) => {
-    const ret: WorkerToMainMsg = [msg, null, null];
-
-    try {
-      switch (msg) {
-        case 0:
-          await init(data1, data2, data3, afterInit);
-          break;
-        case 1:
-          ret[1] = await exec(data1, data2, data3);
-          break;
-        case 2:
-          await db.close();
-          break;
-        case 3:
-          await stream((val) => postMessage([3, [val], null] satisfies WorkerToMainMsg), data1, data2);
-          ret[0] = 4;
-          break;
-        case 4:
-          loadDb((percentage) => postMessage([5, percentage, null] satisfies WorkerToMainMsg), data1, data2, data3);
-          ret[0] = 6;
-          break;
-      }
-    } catch (error) {
-      console.error(error);
-      ret[2] = error;
-    }
-    postMessage(ret);
-  };
-}

src/pages/home.tsx

@@ -6,7 +6,7 @@ import { Portal } from "solid-js/web";
 import { Flex } from "~/components/ui/flex";
 
 import { Progress, ProgressLabel, ProgressValueLabel } from "~/components/ui/progress";
-import { loadDb } from "~/db/db-queries";
+import { loadDb } from "~/db";
 import { decryptBackup } from "~/lib/decryptor";
 import { createDropzone, createFileUploader } from "@solid-primitives/upload";
 import { Button } from "~/components/ui/button";
@@ -37,29 +37,38 @@ export const Home: Component<RouteSectionProps> = () => {
   const [loadingProgress, setLoadingProgress] = createSignal<number>();
   // const [isLoadingDatabase, setIsLoadingDatabase] = createSignal(false);
 
-  const onSubmit: JSX.EventHandler<HTMLFormElement, SubmitEvent> = (event) => {
+  const onSubmit: JSX.EventHandler<HTMLFormElement, SubmitEvent> = async (event) => {
     event.preventDefault();
 
     const currentBackupFile = backupFile();
     const currentPassphrase = passphrase();
 
     if (currentBackupFile && currentPassphrase) {
-      decryptBackup(currentBackupFile, currentPassphrase, setDecryptionProgress)
+      // const hashChunk = await currentBackupFile.slice(-1000).text();
-        .then(async (result) => {
+      // const hash = hashString(hashChunk);
-          setDecryptionProgress(undefined);
-          // setIsLoadingDatabase(true);
-          setLoadingProgress(0);
 
-          await loadDb(result.database_statements, (newValue) => (console.log(newValue), setLoadingProgress(newValue)));
+      // if (hash === dbHash()) {
+      //   return;
+      // }
 
-          // setIsLoadingDatabase(false);
+      // setDbHash(hash);
-          setLoadingProgress(undefined);
 
-          navigate("/overview");
+      try {
-        })
+        const decrypted = await decryptBackup(currentBackupFile, currentPassphrase, setDecryptionProgress);
-        .catch((error) => {
+
-          console.error("Decryption failed:", error);
+        setDecryptionProgress(undefined);
-        });
+        // setIsLoadingDatabase(true);
+        setLoadingProgress(0);
+
+        await loadDb(decrypted.database_statements, setLoadingProgress);
+
+        // setIsLoadingDatabase(false);
+        setLoadingProgress(undefined);
+
+        navigate("/overview");
+      } catch (error) {
+        console.error("Decryption failed:", error);
+      }
     }
   };
 

src/pages/overview/index.tsx

@@ -11,7 +11,9 @@ export const Overview: Component<RouteSectionProps> = () => {
   const [allSelfSentMessagesCount] = createResource(() => overallSentMessagesQuery(SELF_ID));
 
   const [roomOverview] = createResource<RoomOverview[] | undefined>(async () => {
-    return (await allThreadsOverviewQuery())?.map((row) => {
+    const overview = await allThreadsOverviewQuery();
+
+    return overview.map((row) => {
       const isGroup = row.title !== null;
 
       let name = "";
@@ -41,9 +43,7 @@ export const Overview: Component<RouteSectionProps> = () => {
       <div>
         <p>All messages: {allSelfSentMessagesCount()?.messageCount as number}</p>
         <Show when={!roomOverview.loading && roomOverview()} fallback="Loading...">
-          {(currentRoomOverview) => (
+          {(currentRoomOverview) => <OverviewTable data={currentRoomOverview()} />}
-            console.log(currentRoomOverview()), (<OverviewTable data={currentRoomOverview()} />)
-          )}
         </Show>
       </div>
     </>

src/pages/overview/overview-table.tsx

@@ -26,6 +26,7 @@ import { TextField, TextFieldInput } from "~/components/ui/text-field";
 
 import { cn } from "~/lib/utils";
 import { Flex } from "~/components/ui/flex";
+import { dbLoaded, threadSentMessagesOverviewQuery } from "~/db";
 
 export interface RoomOverview {
   threadId: number;
@@ -55,6 +56,18 @@ const isGroupFilterFn: FilterFn<RoomOverview> = (row, _columnId, filterValue) =>
   return !row.original.isGroup;
 };
 
+const rowIsAvailable = (threadId: number): boolean => {
+  if (dbLoaded()) {
+    return true;
+  }
+
+  if (threadSentMessagesOverviewQuery.hasCacheFor(threadId)) {
+    return true;
+  }
+
+  return false;
+};
+
 const SortingDisplay: Component<{ sorting: false | SortDirection; class?: string; activeClass?: string }> = (props) => {
   return (
     <Switch>
@@ -92,13 +105,14 @@ export const columns = [
     cell: (props) => {
       const isArchived = props.row.getValue("archived");
       const isGroup = props.row.getValue("isGroup");
+      const isCached = !dbLoaded() && rowIsAvailable(props.row.original.threadId);
 
       return (
         <Flex class="w-full" flexDirection="row">
           <span class="max-w-2xl overflow-hidden text-ellipsis whitespace-nowrap font-bold">
             {props.cell.getValue()}
           </span>
-          <Show when={isArchived || isGroup}>
+          <Show when={isArchived || isGroup || !isCached}>
             <Flex flexDirection="row" class="ml-auto gap-2">
               <Show when={isArchived}>
                 <Badge variant="outline" class="ml-auto">
@@ -110,6 +124,11 @@ export const columns = [
                   Group
                 </Badge>
               </Show>
+              <Show when={!isCached}>
+                <Badge variant="outline" class="ml-auto">
+                  Not available
+                </Badge>
+              </Show>
             </Flex>
           </Show>
         </Flex>
@@ -267,12 +286,12 @@ export const OverviewTable = (props: OverviewTableProps) => {
         </div>
         <div class="flex items-start space-x-2">
           <Checkbox
-            id="show-archived"
+            id="show-groups"
             checked={(table.getColumn("isGroup")?.getFilterValue() as boolean | undefined) ?? false}
             onChange={(value) => table.getColumn("isGroup")?.setFilterValue(value)}
           />
           <div class="grid gap-1.5 leading-none">
-            <Label for="show-archived">Show group chats (detailed analysis not implemented)</Label>
+            <Label for="show-groups">Show group chats (detailed analysis not implemented)</Label>
           </div>
         </div>
       </div>
@@ -315,32 +334,39 @@ export const OverviewTable = (props: OverviewTableProps) => {
               {(row) => (
                 <TableRow
                   class="cursor-pointer [&:last-of-type>td:first-of-type]:rounded-bl-md [&:last-of-type>td:last-of-type]:rounded-br-md"
+                  classList={{
+                    "text-muted-foreground": !rowIsAvailable(row.original.threadId),
+                  }}
                   data-state={row.getIsSelected() && "selected"}
                   onPointerEnter={(event) => {
                     const threadId = row.original.threadId;
                     const isGroup = row.original.isGroup;
 
-                    const preloadTimeout = setTimeout(() => {
+                    if (rowIsAvailable(threadId)) {
-                      preload(`/${isGroup ? "group" : "dm"}/${threadId.toString()}`, {
+                      const preloadTimeout = setTimeout(() => {
-                        preloadData: true,
+                        preload(`/${isGroup ? "group" : "dm"}/${threadId.toString()}`, {
-                      });
+                          preloadData: true,
-                    }, 20);
+                        });
+                      }, 20);
 
-                    event.currentTarget.addEventListener(
+                      event.currentTarget.addEventListener(
-                      "pointerout",
+                        "pointerout",
-                      () => {
+                        () => {
-                        clearTimeout(preloadTimeout);
+                          clearTimeout(preloadTimeout);
-                      },
+                        },
-                      {
+                        {
-                        once: true,
+                          once: true,
-                      },
+                        },
-                    );
+                      );
+                    }
                   }}
                   onClick={() => {
                     const threadId = row.original.threadId;
                     const isGroup = row.original.isGroup;
 
-                    navigate(`/${isGroup ? "group" : "dm"}/${threadId.toString()}`);
+                    if (rowIsAvailable(threadId)) {
+                      navigate(`/${isGroup ? "group" : "dm"}/${threadId.toString()}`);
+                    }
                   }}
                 >
                   <For each={row.getVisibleCells()}>