From cd4c5095e5559b453cf0d6c9f73e67152ede238c Mon Sep 17 00:00:00 2001 From: Samuel Date: Fri, 13 Dec 2024 17:10:44 +0100 Subject: [PATCH] feat(dm-stats): add most used words stat --- package.json | 1 + pnpm-lock.yaml | 38 +++++++++++++ src/components/ui/charts.tsx | 3 ++ src/db.ts | 49 +++++++++++++++-- src/lib/db-cache.ts | 4 +- src/pages/dm/dm-id.tsx | 100 +++++++++++++++++++++++++---------- src/pages/overview/index.tsx | 8 +-- 7 files changed, 163 insertions(+), 40 deletions(-) diff --git a/package.json b/package.json index b6dd09a..a7a8ef5 100644 --- a/package.json +++ b/package.json @@ -42,6 +42,7 @@ "@solidjs/router": "^0.15.1", "@tanstack/solid-table": "^8.20.5", "chart.js": "^4.4.7", + "chartjs-chart-wordcloud": "^4.4.4", "chartjs-plugin-deferred": "^2.0.0", "chartjs-plugin-zoom": "^2.2.0", "class-variance-authority": "^0.7.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 67d797e..f346487 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,9 @@ importers: chart.js: specifier: ^4.4.7 version: 4.4.7 + chartjs-chart-wordcloud: + specifier: ^4.4.4 + version: 4.4.4(chart.js@4.4.7) chartjs-plugin-deferred: specifier: ^2.0.0 version: 2.0.0(chart.js@4.4.7) @@ -672,6 +675,12 @@ packages: '@types/babel__traverse@7.20.6': resolution: {integrity: sha512-r1bzfrm0tomOI8g1SzvCaQHo6Lcv6zu0EA+W2kHrt8dyrHQxGzBBL4kdkzIS+jBMV+EYcMAEAqXqYaLJq5rOZg==} + '@types/d3-cloud@1.2.9': + resolution: {integrity: sha512-5EWJvnlCrqTThGp8lYHx+DL00sOjx2HTlXH1WRe93k5pfOIhPQaL63NttaKYIbT7bTXp/USiunjNS/N4ipttIQ==} + + '@types/d3@3.5.53': + resolution: {integrity: sha512-8yKQA9cAS6+wGsJpBysmnhlaaxlN42Qizqkw+h2nILSlS+MAG2z4JdO6p+PJrJ+ACvimkmLJL281h157e52psQ==} + '@types/emscripten@1.39.13': resolution: {integrity: sha512-cFq+fO/isvhvmuP/+Sl4K4jtU6E23DoivtbO4r50e3odaxAiVdbfSYRDdJ4gCdxx+3aRjhphS5ZMwIH4hFy/Cw==} @@ -875,6 +884,11 @@ packages: resolution: {integrity: sha512-pwkcKfdzTMAU/+jNosKhNL2bHtJc/sSmYgVbuGTEDhzkrhmyihmP7vUc/5ZK9WopidMDHNe3Wm7jOd/WhuHWuw==} engines: {pnpm: '>=8'} + chartjs-chart-wordcloud@4.4.4: + resolution: {integrity: sha512-9ruQX5CjfBwpO31mYqniD3BQ9YTCHxWe52eoUl9Z3ecNQWF3OCHeDVv7lHe8ca1NBihfCgoRa3Y0eBGwBmgwcw==} + peerDependencies: + chart.js: ^4.1.0 + chartjs-plugin-deferred@2.0.0: resolution: {integrity: sha512-jq6b8Wt23WS6zxiX8oVB1MXq4uaJX2KGTyiqnq6xo4ctZPgFkT/FuIEKpJjsF1WkYv7ZQrqrrRg1fLw6O5ZEfQ==} peerDependencies: @@ -934,6 +948,12 @@ packages: csstype@3.1.3: resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==} + d3-cloud@1.2.7: + resolution: {integrity: sha512-8TrgcgwRIpoZYQp7s3fGB7tATWfhckRb8KcVd1bOgqkNdkJRDGWfdSf4HkHHzZxSczwQJdSxvfPudwir5IAJ3w==} + + d3-dispatch@1.0.6: + resolution: {integrity: sha512-fVjoElzjhCEy+Hbn8KygnmMS7Or0a9sI2UzGwoB7cCtvI1XpVN9GpoYlnb3xt2YV66oXYb1fLJ8GMvP4hdU1RA==} + date-fns@4.1.0: resolution: {integrity: sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==} @@ -2489,6 +2509,12 @@ snapshots: dependencies: '@babel/types': 7.26.3 + '@types/d3-cloud@1.2.9': + dependencies: + '@types/d3': 3.5.53 + + '@types/d3@3.5.53': {} + '@types/emscripten@1.39.13': {} '@types/estree@1.0.6': {} @@ -2718,6 +2744,12 @@ snapshots: dependencies: '@kurkle/color': 0.3.4 + chartjs-chart-wordcloud@4.4.4(chart.js@4.4.7): + dependencies: + '@types/d3-cloud': 1.2.9 + chart.js: 4.4.7 + d3-cloud: 1.2.7 + chartjs-plugin-deferred@2.0.0(chart.js@4.4.7): dependencies: chart.js: 4.4.7 @@ -2776,6 +2808,12 @@ snapshots: csstype@3.1.3: {} + d3-cloud@1.2.7: + dependencies: + d3-dispatch: 1.0.6 + + d3-dispatch@1.0.6: {} + date-fns@4.1.0: {} debug@4.4.0: diff --git a/src/components/ui/charts.tsx b/src/components/ui/charts.tsx index 760a30a..1f2ec54 100644 --- a/src/components/ui/charts.tsx +++ b/src/components/ui/charts.tsx @@ -36,6 +36,7 @@ import { ScatterController, Tooltip, } from "chart.js"; +import { WordCloudController, WordElement } from "chartjs-chart-wordcloud"; import ChartDeferred from "chartjs-plugin-deferred"; import ChartZoom from "chartjs-plugin-zoom"; @@ -276,6 +277,7 @@ const RadarChart = /* #__PURE__ */ createTypedChart("radar", [ RadialLinearScale, ]); const ScatterChart = /* #__PURE__ */ createTypedChart("scatter", [ScatterController, PointElement, LinearScale]); +const WordCloudChart = /* #__PURE__ */ createTypedChart("wordCloud", [WordCloudController, WordElement]); export { BarChart, @@ -287,4 +289,5 @@ export { PolarAreaChart, RadarChart, ScatterChart, + WordCloudChart, }; diff --git a/src/db.ts b/src/db.ts index dff344b..e5faa31 100644 --- a/src/db.ts +++ b/src/db.ts @@ -1,4 +1,4 @@ -import { type Accessor, createMemo, createSignal, DEV, type Setter } from "solid-js"; +import { type Accessor, createEffect, createMemo, createSignal, DEV, type Setter } from "solid-js"; import { Kysely, type NotNull, sql } from "kysely"; import type { DB } from "kysely-codegen"; @@ -41,6 +41,16 @@ const sqlJsDialect = () => { } }; +createEffect(() => { + const db = rawDb(); + + if (db) { + db.create_function("is_not_empty", (str: string | null) => { + return str !== null && str !== ""; + }); + } +}); + const kyselyDb = createMemo(() => { const currentSqlJsDialect = sqlJsDialect(); @@ -131,7 +141,7 @@ const dmOverviewQueryRaw = (dmId: number) => export const dmOverviewQuery = cached(dmOverviewQueryRaw); -const threadSentMessagesPerPersonOverviewQueryRaw = (dmId: number) => +const threadSentMessagesPerPersonOverviewQueryRaw = (threadId: number) => kyselyDb() .selectFrom("message") .select((eb) => [ @@ -141,10 +151,43 @@ const threadSentMessagesPerPersonOverviewQueryRaw = (dmId: number) => ]) .groupBy(["from_recipient_id", "message_date"]) .orderBy(["message_date"]) - .where((eb) => eb.and([eb("body", "is not", null), eb("body", "!=", ""), eb("thread_id", "=", dmId)])) + .where((eb) => eb.and([eb("body", "is not", null), eb("body", "!=", ""), eb("thread_id", "=", threadId)])) .$narrowType<{ message_count: number; }>() .execute(); export const dmSentMessagesPerPersonOverviewQuery = cached(threadSentMessagesPerPersonOverviewQueryRaw); + +const threadMostUsedWordsQueryRaw = (threadId: number, limit = 10) => + kyselyDb() + .withRecursive("words", (eb) => { + return eb + .selectFrom("message") + .select([ + sql`LOWER(substr(body, 1, instr(body || " ", " ") - 1))`.as("word"), + sql`(substr(body, instr(body || " ", " ") + 1))`.as("rest"), + ]) + .where((eb) => eb.and([eb("body", "is not", null), eb("thread_id", "=", threadId)])) + .unionAll((ebInner) => { + return ebInner + .selectFrom("words") + .select([ + sql`LOWER(substr(rest, 1, instr(rest || " ", " ") - 1))`.as("word"), + sql`(substr(rest, instr(rest || " ", " ") + 1))`.as("rest"), + ]) + .where("rest", "<>", ""); + }); + }) + .selectFrom("words") + .select((eb) => ["word", eb.fn.countAll().as("count")]) + .where("word", "<>", "") + .groupBy("word") + .orderBy("count desc") + .limit(limit) + .$narrowType<{ + count: number; + }>() + .execute(); + +export const threadMostUsedWordsQuery = cached(threadMostUsedWordsQueryRaw); diff --git a/src/lib/db-cache.ts b/src/lib/db-cache.ts index f327ec7..5c92833 100644 --- a/src/lib/db-cache.ts +++ b/src/lib/db-cache.ts @@ -82,13 +82,13 @@ class LocalStorageCacheAdapter { const cache = new LocalStorageCacheAdapter(); -export const cached = (fn: (...args: T[]) => R, self?: ThisType): ((...args: T[]) => R) => { +export const cached = (fn: (...args: T) => R, self?: ThisType): ((...args: T) => R) => { const cacheName = hashString(fn.toString()).toString(); // important to return a promise on follow-up calls even if the data is immediately available let isPromise: boolean; - return (...args: T[]) => { + return (...args: T) => { const cacheKey = JSON.stringify(args); const cachedValue = cache.get(cacheName, cacheKey); diff --git a/src/pages/dm/dm-id.tsx b/src/pages/dm/dm-id.tsx index cba3744..c27dde8 100644 --- a/src/pages/dm/dm-id.tsx +++ b/src/pages/dm/dm-id.tsx @@ -1,15 +1,16 @@ -import { type Accessor, type Component, createResource, Show } from "solid-js"; +import { type Accessor, type Component, createEffect, createResource, Show } from "solid-js"; import type { RouteSectionProps } from "@solidjs/router"; import { type ChartData } from "chart.js"; -import { LineChart } from "~/components/ui/charts"; +import { LineChart, WordCloudChart } from "~/components/ui/charts"; -import { dmPartnerRecipientQuery, dmSentMessagesPerPersonOverviewQuery, SELF_ID } from "~/db"; +import { dmPartnerRecipientQuery, dmSentMessagesPerPersonOverviewQuery, SELF_ID, threadMostUsedWordsQuery } from "~/db"; export const DmId: Component = (props) => { const dmId = () => Number(props.params.dmid); + // the other person in the chat with name and id const [dmPartner] = createResource(async () => { const dmPartner = await dmPartnerRecipientQuery(dmId()); @@ -26,15 +27,17 @@ export const DmId: Component = (props) => { const [dmMessagesPerPerson] = createResource(() => dmSentMessagesPerPersonOverviewQuery(dmId())); + // maps all the message counts to dates const dmMessages = () => { return dmMessagesPerPerson()?.reduce< { date: Date; totalMessages: number; - [key: number]: number; + [recipientId: number]: number; }[] >((prev, curr) => { const existingDate = prev.find(({ date }) => date === curr.message_date); + if (existingDate) { existingDate[curr.from_recipient_id] = curr.message_count; @@ -51,6 +54,8 @@ export const DmId: Component = (props) => { }, []); }; + const [mostUsedWordCounts] = createResource(async () => threadMostUsedWordsQuery(dmId(), 300)); + const recipients = () => { const currentDmPartner = dmPartner(); @@ -72,6 +77,31 @@ export const DmId: Component = (props) => { ]; }; + const maxWordSize = 100; + + const mostUsedWordChartData: Accessor | undefined> = () => { + const currentMostUsedWordCounts = mostUsedWordCounts(); + + if (currentMostUsedWordCounts) { + // ordered descending in db query + const highestWordCount = currentMostUsedWordCounts[0].count; + + const calcWordSizeInPixels = (count: number) => { + return 10 + Math.round((maxWordSize / highestWordCount) * count); + }; + + return { + labels: currentMostUsedWordCounts.map(({ word }) => word), + datasets: [ + { + label: "Used", + data: currentMostUsedWordCounts.map(({ count }) => calcWordSizeInPixels(count)), + }, + ], + }; + } + }; + const dateChartData: Accessor | undefined> = () => { const currentDmMessages = dmMessages(); const currentRecipients = recipients(); @@ -108,36 +138,50 @@ export const DmId: Component = (props) => { }; return ( - - {(currentDateChartData) => ( -
- + + {(currentDateChartData) => ( +
+ -
- )} - + }} + data={currentDateChartData()} + /> +
+ )} + + + {(currentMostUsedWordChartData) => ( +
+ +
+ )} + + ); }; diff --git a/src/pages/overview/index.tsx b/src/pages/overview/index.tsx index 3ceed3f..63a553b 100644 --- a/src/pages/overview/index.tsx +++ b/src/pages/overview/index.tsx @@ -1,4 +1,4 @@ -import { type Component, createEffect, createResource, Show } from "solid-js"; +import { type Component, createResource, Show } from "solid-js"; import type { RouteSectionProps } from "@solidjs/router"; import { allThreadsOverviewQuery, overallSentMessagesQuery, SELF_ID } from "~/db"; @@ -12,8 +12,6 @@ export const Overview: Component = () => { return (await allThreadsOverviewQuery()).rows.map((row) => { const isGroup = row.title !== null; - console.log(row); - // eslint-disable-next-line @typescript-eslint/no-non-null-assertion const name = ( isGroup @@ -35,10 +33,6 @@ export const Overview: Component = () => { }); }); - createEffect(() => { - console.log(roomOverview()); - }); - return (

All messages: {allSelfSentMessagesCount()?.messageCount as number}