feat(dm-stats): add most used words stat

package.json

@@ -42,6 +42,7 @@
     "@solidjs/router": "^0.15.1",
     "@tanstack/solid-table": "^8.20.5",
     "chart.js": "^4.4.7",
+    "chartjs-chart-wordcloud": "^4.4.4",
     "chartjs-plugin-deferred": "^2.0.0",
     "chartjs-plugin-zoom": "^2.2.0",
     "class-variance-authority": "^0.7.1",

pnpm-lock.yaml

@@ -26,6 +26,9 @@ importers:
       chart.js:
         specifier: ^4.4.7
         version: 4.4.7
+      chartjs-chart-wordcloud:
+        specifier: ^4.4.4
+        version: 4.4.4(chart.js@4.4.7)
       chartjs-plugin-deferred:
         specifier: ^2.0.0
         version: 2.0.0(chart.js@4.4.7)
@@ -672,6 +675,12 @@ packages:
   '@types/babel__traverse@7.20.6':
     resolution: {integrity: sha512-r1bzfrm0tomOI8g1SzvCaQHo6Lcv6zu0EA+W2kHrt8dyrHQxGzBBL4kdkzIS+jBMV+EYcMAEAqXqYaLJq5rOZg==}
 
+  '@types/d3-cloud@1.2.9':
+    resolution: {integrity: sha512-5EWJvnlCrqTThGp8lYHx+DL00sOjx2HTlXH1WRe93k5pfOIhPQaL63NttaKYIbT7bTXp/USiunjNS/N4ipttIQ==}
+
+  '@types/d3@3.5.53':
+    resolution: {integrity: sha512-8yKQA9cAS6+wGsJpBysmnhlaaxlN42Qizqkw+h2nILSlS+MAG2z4JdO6p+PJrJ+ACvimkmLJL281h157e52psQ==}
+
   '@types/emscripten@1.39.13':
     resolution: {integrity: sha512-cFq+fO/isvhvmuP/+Sl4K4jtU6E23DoivtbO4r50e3odaxAiVdbfSYRDdJ4gCdxx+3aRjhphS5ZMwIH4hFy/Cw==}
 
@@ -875,6 +884,11 @@ packages:
     resolution: {integrity: sha512-pwkcKfdzTMAU/+jNosKhNL2bHtJc/sSmYgVbuGTEDhzkrhmyihmP7vUc/5ZK9WopidMDHNe3Wm7jOd/WhuHWuw==}
     engines: {pnpm: '>=8'}
 
+  chartjs-chart-wordcloud@4.4.4:
+    resolution: {integrity: sha512-9ruQX5CjfBwpO31mYqniD3BQ9YTCHxWe52eoUl9Z3ecNQWF3OCHeDVv7lHe8ca1NBihfCgoRa3Y0eBGwBmgwcw==}
+    peerDependencies:
+      chart.js: ^4.1.0
+
   chartjs-plugin-deferred@2.0.0:
     resolution: {integrity: sha512-jq6b8Wt23WS6zxiX8oVB1MXq4uaJX2KGTyiqnq6xo4ctZPgFkT/FuIEKpJjsF1WkYv7ZQrqrrRg1fLw6O5ZEfQ==}
     peerDependencies:
@@ -934,6 +948,12 @@ packages:
   csstype@3.1.3:
     resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==}
 
+  d3-cloud@1.2.7:
+    resolution: {integrity: sha512-8TrgcgwRIpoZYQp7s3fGB7tATWfhckRb8KcVd1bOgqkNdkJRDGWfdSf4HkHHzZxSczwQJdSxvfPudwir5IAJ3w==}
+
+  d3-dispatch@1.0.6:
+    resolution: {integrity: sha512-fVjoElzjhCEy+Hbn8KygnmMS7Or0a9sI2UzGwoB7cCtvI1XpVN9GpoYlnb3xt2YV66oXYb1fLJ8GMvP4hdU1RA==}
+
   date-fns@4.1.0:
     resolution: {integrity: sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==}
 
@@ -2489,6 +2509,12 @@ snapshots:
     dependencies:
       '@babel/types': 7.26.3
 
+  '@types/d3-cloud@1.2.9':
+    dependencies:
+      '@types/d3': 3.5.53
+
+  '@types/d3@3.5.53': {}
+
   '@types/emscripten@1.39.13': {}
 
   '@types/estree@1.0.6': {}
@@ -2718,6 +2744,12 @@ snapshots:
     dependencies:
       '@kurkle/color': 0.3.4
 
+  chartjs-chart-wordcloud@4.4.4(chart.js@4.4.7):
+    dependencies:
+      '@types/d3-cloud': 1.2.9
+      chart.js: 4.4.7
+      d3-cloud: 1.2.7
+
   chartjs-plugin-deferred@2.0.0(chart.js@4.4.7):
     dependencies:
       chart.js: 4.4.7
@@ -2776,6 +2808,12 @@ snapshots:
 
   csstype@3.1.3: {}
 
+  d3-cloud@1.2.7:
+    dependencies:
+      d3-dispatch: 1.0.6
+
+  d3-dispatch@1.0.6: {}
+
   date-fns@4.1.0: {}
 
   debug@4.4.0:

src/components/ui/charts.tsx

@@ -36,6 +36,7 @@ import {
   ScatterController,
   Tooltip,
 } from "chart.js";
+import { WordCloudController, WordElement } from "chartjs-chart-wordcloud";
 import ChartDeferred from "chartjs-plugin-deferred";
 import ChartZoom from "chartjs-plugin-zoom";
 
@@ -276,6 +277,7 @@ const RadarChart = /* #__PURE__ */ createTypedChart("radar", [
   RadialLinearScale,
 ]);
 const ScatterChart = /* #__PURE__ */ createTypedChart("scatter", [ScatterController, PointElement, LinearScale]);
+const WordCloudChart = /* #__PURE__ */ createTypedChart("wordCloud", [WordCloudController, WordElement]);
 
 export {
   BarChart,
@@ -287,4 +289,5 @@ export {
   PolarAreaChart,
   RadarChart,
   ScatterChart,
+  WordCloudChart,
 };

src/db.ts

@@ -1,4 +1,4 @@
-import { type Accessor, createMemo, createSignal, DEV, type Setter } from "solid-js";
+import { type Accessor, createEffect, createMemo, createSignal, DEV, type Setter } from "solid-js";
 
 import { Kysely, type NotNull, sql } from "kysely";
 import type { DB } from "kysely-codegen";
@@ -41,6 +41,16 @@ const sqlJsDialect = () => {
   }
 };
 
+createEffect(() => {
+  const db = rawDb();
+
+  if (db) {
+    db.create_function("is_not_empty", (str: string | null) => {
+      return str !== null && str !== "";
+    });
+  }
+});
+
 const kyselyDb = createMemo(() => {
   const currentSqlJsDialect = sqlJsDialect();
 
@@ -131,7 +141,7 @@ const dmOverviewQueryRaw = (dmId: number) =>
 
 export const dmOverviewQuery = cached(dmOverviewQueryRaw);
 
-const threadSentMessagesPerPersonOverviewQueryRaw = (dmId: number) =>
+const threadSentMessagesPerPersonOverviewQueryRaw = (threadId: number) =>
   kyselyDb()
     .selectFrom("message")
     .select((eb) => [
@@ -141,10 +151,43 @@ const threadSentMessagesPerPersonOverviewQueryRaw = (dmId: number) =>
     ])
     .groupBy(["from_recipient_id", "message_date"])
     .orderBy(["message_date"])
-    .where((eb) => eb.and([eb("body", "is not", null), eb("body", "!=", ""), eb("thread_id", "=", dmId)]))
+    .where((eb) => eb.and([eb("body", "is not", null), eb("body", "!=", ""), eb("thread_id", "=", threadId)]))
     .$narrowType<{
       message_count: number;
     }>()
     .execute();
 
 export const dmSentMessagesPerPersonOverviewQuery = cached(threadSentMessagesPerPersonOverviewQueryRaw);
+
+const threadMostUsedWordsQueryRaw = (threadId: number, limit = 10) =>
+  kyselyDb()
+    .withRecursive("words", (eb) => {
+      return eb
+        .selectFrom("message")
+        .select([
+          sql`LOWER(substr(body, 1, instr(body || " ", " ") - 1))`.as("word"),
+          sql`(substr(body, instr(body || " ", " ") + 1))`.as("rest"),
+        ])
+        .where((eb) => eb.and([eb("body", "is not", null), eb("thread_id", "=", threadId)]))
+        .unionAll((ebInner) => {
+          return ebInner
+            .selectFrom("words")
+            .select([
+              sql`LOWER(substr(rest, 1, instr(rest || " ", " ") - 1))`.as("word"),
+              sql`(substr(rest, instr(rest || " ", " ") + 1))`.as("rest"),
+            ])
+            .where("rest", "<>", "");
+        });
+    })
+    .selectFrom("words")
+    .select((eb) => ["word", eb.fn.countAll().as("count")])
+    .where("word", "<>", "")
+    .groupBy("word")
+    .orderBy("count desc")
+    .limit(limit)
+    .$narrowType<{
+      count: number;
+    }>()
+    .execute();
+
+export const threadMostUsedWordsQuery = cached(threadMostUsedWordsQueryRaw);

src/lib/db-cache.ts

@@ -82,13 +82,13 @@ class LocalStorageCacheAdapter {
 
 const cache = new LocalStorageCacheAdapter();
 
-export const cached = <T, R, TT>(fn: (...args: T[]) => R, self?: ThisType<TT>): ((...args: T[]) => R) => {
+export const cached = <T extends unknown[], R, TT>(fn: (...args: T) => R, self?: ThisType<TT>): ((...args: T) => R) => {
   const cacheName = hashString(fn.toString()).toString();
 
   // important to return a promise on follow-up calls even if the data is immediately available
   let isPromise: boolean;
 
-  return (...args: T[]) => {
+  return (...args: T) => {
     const cacheKey = JSON.stringify(args);
 
     const cachedValue = cache.get<R>(cacheName, cacheKey);

src/pages/dm/dm-id.tsx

@@ -1,15 +1,16 @@
-import { type Accessor, type Component, createResource, Show } from "solid-js";
+import { type Accessor, type Component, createEffect, createResource, Show } from "solid-js";
 import type { RouteSectionProps } from "@solidjs/router";
 
 import { type ChartData } from "chart.js";
 
-import { LineChart } from "~/components/ui/charts";
+import { LineChart, WordCloudChart } from "~/components/ui/charts";
 
-import { dmPartnerRecipientQuery, dmSentMessagesPerPersonOverviewQuery, SELF_ID } from "~/db";
+import { dmPartnerRecipientQuery, dmSentMessagesPerPersonOverviewQuery, SELF_ID, threadMostUsedWordsQuery } from "~/db";
 
 export const DmId: Component<RouteSectionProps> = (props) => {
   const dmId = () => Number(props.params.dmid);
 
+  // the other person in the chat with name and id
   const [dmPartner] = createResource(async () => {
     const dmPartner = await dmPartnerRecipientQuery(dmId());
 
@@ -26,15 +27,17 @@ export const DmId: Component<RouteSectionProps> = (props) => {
 
   const [dmMessagesPerPerson] = createResource(() => dmSentMessagesPerPersonOverviewQuery(dmId()));
 
+  // maps all the message counts to dates
   const dmMessages = () => {
     return dmMessagesPerPerson()?.reduce<
       {
         date: Date;
         totalMessages: number;
-        [key: number]: number;
+        [recipientId: number]: number;
       }[]
     >((prev, curr) => {
       const existingDate = prev.find(({ date }) => date === curr.message_date);
+
       if (existingDate) {
         existingDate[curr.from_recipient_id] = curr.message_count;
 
@@ -51,6 +54,8 @@ export const DmId: Component<RouteSectionProps> = (props) => {
     }, []);
   };
 
+  const [mostUsedWordCounts] = createResource(async () => threadMostUsedWordsQuery(dmId(), 300));
+
   const recipients = () => {
     const currentDmPartner = dmPartner();
 
@@ -72,6 +77,31 @@ export const DmId: Component<RouteSectionProps> = (props) => {
     ];
   };
 
+  const maxWordSize = 100;
+
+  const mostUsedWordChartData: Accessor<ChartData<"wordCloud"> | undefined> = () => {
+    const currentMostUsedWordCounts = mostUsedWordCounts();
+
+    if (currentMostUsedWordCounts) {
+      // ordered descending in db query
+      const highestWordCount = currentMostUsedWordCounts[0].count;
+
+      const calcWordSizeInPixels = (count: number) => {
+        return 10 + Math.round((maxWordSize / highestWordCount) * count);
+      };
+
+      return {
+        labels: currentMostUsedWordCounts.map(({ word }) => word),
+        datasets: [
+          {
+            label: "Used",
+            data: currentMostUsedWordCounts.map(({ count }) => calcWordSizeInPixels(count)),
+          },
+        ],
+      };
+    }
+  };
+
   const dateChartData: Accessor<ChartData<"line"> | undefined> = () => {
     const currentDmMessages = dmMessages();
     const currentRecipients = recipients();
@@ -108,36 +138,50 @@ export const DmId: Component<RouteSectionProps> = (props) => {
   };
 
   return (
-    <Show when={dateChartData()}>
+    <div>
-      {(currentDateChartData) => (
+      <Show when={dateChartData()}>
-        <div class="max-h-96">
+        {(currentDateChartData) => (
-          <LineChart
+          <div class="max-h-96">
-            options={{
+            <LineChart
-              normalized: true,
+              options={{
-              aspectRatio: 2,
+                normalized: true,
-              plugins: {
+                aspectRatio: 2,
-                zoom: {
+                plugins: {
-                  pan: {
-                    enabled: true,
-                    mode: "xy",
-                  },
                   zoom: {
-                    wheel: {
+                    pan: {
                       enabled: true,
+                      mode: "xy",
                     },
-                    pinch: {
+                    zoom: {
-                      enabled: true,
+                      wheel: {
+                        enabled: true,
+                      },
+                      pinch: {
+                        enabled: true,
+                      },
+                      mode: "xy",
                     },
-                    mode: "xy",
                   },
                 },
-              },
+              }}
-            }}
+              data={currentDateChartData()}
-            data={currentDateChartData()}
+            />
-          />
+          </div>
-        </div>
+        )}
-      )}
+      </Show>
-    </Show>
+      <Show when={mostUsedWordChartData()}>
+        {(currentMostUsedWordChartData) => (
+          <div class="max-w-3xl">
+            <WordCloudChart
+              options={{
+                normalized: true,
+              }}
+              data={currentMostUsedWordChartData()}
+            />
+          </div>
+        )}
+      </Show>
+    </div>
   );
 };
 

src/pages/overview/index.tsx

@@ -1,4 +1,4 @@
-import { type Component, createEffect, createResource, Show } from "solid-js";
+import { type Component, createResource, Show } from "solid-js";
 import type { RouteSectionProps } from "@solidjs/router";
 
 import { allThreadsOverviewQuery, overallSentMessagesQuery, SELF_ID } from "~/db";
@@ -12,8 +12,6 @@ export const Overview: Component<RouteSectionProps> = () => {
     return (await allThreadsOverviewQuery()).rows.map((row) => {
       const isGroup = row.title !== null;
 
-      console.log(row);
-
       // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
       const name = (
         isGroup
@@ -35,10 +33,6 @@ export const Overview: Component<RouteSectionProps> = () => {
     });
   });
 
-  createEffect(() => {
-    console.log(roomOverview());
-  });
-
   return (
     <div>
       <p>All messages: {allSelfSentMessagesCount()?.messageCount as number}</p>