duskflower/signalstats
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

feat(dm-stats): add most used words stat

Browse source
This commit is contained in:
Samuel 2024-12-13 17:10:44 +01:00
parent 8451a484ff
commit d21f7fa335
7 changed files with 163 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

1
package.json
View file

@ -42,6 +42,7 @@
"@solidjs/router": "^0.15.1",
"@tanstack/solid-table": "^8.20.5",
"chart.js": "^4.4.7",
"chartjs-chart-wordcloud": "^4.4.4",
"chartjs-plugin-deferred": "^2.0.0",
"chartjs-plugin-zoom": "^2.2.0",
"class-variance-authority": "^0.7.1",

38
pnpm-lock.yaml generated
View file

@ -26,6 +26,9 @@ importers:
chart.js:
specifier: ^4.4.7
version: 4.4.7
chartjs-chart-wordcloud:
specifier: ^4.4.4
version: 4.4.4(chart.js@4.4.7)
chartjs-plugin-deferred:
specifier: ^2.0.0
version: 2.0.0(chart.js@4.4.7)
@ -672,6 +675,12 @@ packages:
'@types/babel__traverse@7.20.6':
resolution: {integrity: sha512-r1bzfrm0tomOI8g1SzvCaQHo6Lcv6zu0EA+W2kHrt8dyrHQxGzBBL4kdkzIS+jBMV+EYcMAEAqXqYaLJq5rOZg==}
'@types/d3-cloud@1.2.9':
resolution: {integrity: sha512-5EWJvnlCrqTThGp8lYHx+DL00sOjx2HTlXH1WRe93k5pfOIhPQaL63NttaKYIbT7bTXp/USiunjNS/N4ipttIQ==}
'@types/d3@3.5.53':
resolution: {integrity: sha512-8yKQA9cAS6+wGsJpBysmnhlaaxlN42Qizqkw+h2nILSlS+MAG2z4JdO6p+PJrJ+ACvimkmLJL281h157e52psQ==}
'@types/emscripten@1.39.13':
resolution: {integrity: sha512-cFq+fO/isvhvmuP/+Sl4K4jtU6E23DoivtbO4r50e3odaxAiVdbfSYRDdJ4gCdxx+3aRjhphS5ZMwIH4hFy/Cw==}
@ -875,6 +884,11 @@ packages:
resolution: {integrity: sha512-pwkcKfdzTMAU/+jNosKhNL2bHtJc/sSmYgVbuGTEDhzkrhmyihmP7vUc/5ZK9WopidMDHNe3Wm7jOd/WhuHWuw==}
engines: {pnpm: '>=8'}
chartjs-chart-wordcloud@4.4.4:
resolution: {integrity: sha512-9ruQX5CjfBwpO31mYqniD3BQ9YTCHxWe52eoUl9Z3ecNQWF3OCHeDVv7lHe8ca1NBihfCgoRa3Y0eBGwBmgwcw==}
peerDependencies:
chart.js: ^4.1.0
chartjs-plugin-deferred@2.0.0:
resolution: {integrity: sha512-jq6b8Wt23WS6zxiX8oVB1MXq4uaJX2KGTyiqnq6xo4ctZPgFkT/FuIEKpJjsF1WkYv7ZQrqrrRg1fLw6O5ZEfQ==}
peerDependencies:
@ -934,6 +948,12 @@ packages:
csstype@3.1.3:
resolution: {integrity: sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw==}
d3-cloud@1.2.7:
resolution: {integrity: sha512-8TrgcgwRIpoZYQp7s3fGB7tATWfhckRb8KcVd1bOgqkNdkJRDGWfdSf4HkHHzZxSczwQJdSxvfPudwir5IAJ3w==}
d3-dispatch@1.0.6:
resolution: {integrity: sha512-fVjoElzjhCEy+Hbn8KygnmMS7Or0a9sI2UzGwoB7cCtvI1XpVN9GpoYlnb3xt2YV66oXYb1fLJ8GMvP4hdU1RA==}
date-fns@4.1.0:
resolution: {integrity: sha512-Ukq0owbQXxa/U3EGtsdVBkR1w7KOQ5gIBqdH2hkvknzZPYvBxb/aa6E8L7tmjFtkwZBu3UXBbjIgPo/Ez4xaNg==}
@ -2489,6 +2509,12 @@ snapshots:
dependencies:
'@babel/types': 7.26.3
'@types/d3-cloud@1.2.9':
dependencies:
'@types/d3': 3.5.53
'@types/d3@3.5.53': {}
'@types/emscripten@1.39.13': {}
'@types/estree@1.0.6': {}
@ -2718,6 +2744,12 @@ snapshots:
dependencies:
'@kurkle/color': 0.3.4
chartjs-chart-wordcloud@4.4.4(chart.js@4.4.7):
dependencies:
'@types/d3-cloud': 1.2.9
chart.js: 4.4.7
d3-cloud: 1.2.7
chartjs-plugin-deferred@2.0.0(chart.js@4.4.7):
dependencies:
chart.js: 4.4.7
@ -2776,6 +2808,12 @@ snapshots:
csstype@3.1.3: {}
d3-cloud@1.2.7:
dependencies:
d3-dispatch: 1.0.6
d3-dispatch@1.0.6: {}
date-fns@4.1.0: {}
debug@4.4.0:

3
src/components/ui/charts.tsx
View file

@ -36,6 +36,7 @@ import {
ScatterController,
Tooltip,
} from "chart.js";
import { WordCloudController, WordElement } from "chartjs-chart-wordcloud";
import ChartDeferred from "chartjs-plugin-deferred";
import ChartZoom from "chartjs-plugin-zoom";
@ -276,6 +277,7 @@ const RadarChart = /* #__PURE__ */ createTypedChart("radar", [
RadialLinearScale,
]);
const ScatterChart = /* #__PURE__ */ createTypedChart("scatter", [ScatterController, PointElement, LinearScale]);
const WordCloudChart = /* #__PURE__ */ createTypedChart("wordCloud", [WordCloudController, WordElement]);
export {
BarChart,
@ -287,4 +289,5 @@ export {
PolarAreaChart,
RadarChart,
ScatterChart,
WordCloudChart,
};

49
src/db.ts
View file

@ -1,4 +1,4 @@
import { type Accessor, createMemo, createSignal, DEV, type Setter } from "solid-js";
import { type Accessor, createEffect, createMemo, createSignal, DEV, type Setter } from "solid-js";
import { Kysely, type NotNull, sql } from "kysely";
import type { DB } from "kysely-codegen";
@ -41,6 +41,16 @@ const sqlJsDialect = () => {
}
};
createEffect(() => {
const db = rawDb();
if (db) {
db.create_function("is_not_empty", (str: string | null) => {
return str !== null && str !== "";
});
}
});
const kyselyDb = createMemo(() => {
const currentSqlJsDialect = sqlJsDialect();
@ -131,7 +141,7 @@ const dmOverviewQueryRaw = (dmId: number) =>
export const dmOverviewQuery = cached(dmOverviewQueryRaw);
const threadSentMessagesPerPersonOverviewQueryRaw = (dmId: number) =>
const threadSentMessagesPerPersonOverviewQueryRaw = (threadId: number) =>
kyselyDb()
.selectFrom("message")
.select((eb) => [
@ -141,10 +151,43 @@ const threadSentMessagesPerPersonOverviewQueryRaw = (dmId: number) =>
])
.groupBy(["from_recipient_id", "message_date"])
.orderBy(["message_date"])
.where((eb) => eb.and([eb("body", "is not", null), eb("body", "!=", ""), eb("thread_id", "=", dmId)]))
.where((eb) => eb.and([eb("body", "is not", null), eb("body", "!=", ""), eb("thread_id", "=", threadId)]))
.$narrowType<{
message_count: number;
}>()
.execute();
export const dmSentMessagesPerPersonOverviewQuery = cached(threadSentMessagesPerPersonOverviewQueryRaw);
const threadMostUsedWordsQueryRaw = (threadId: number, limit = 10) =>
kyselyDb()
.withRecursive("words", (eb) => {
return eb
.selectFrom("message")
.select([
sql`LOWER(substr(body, 1, instr(body || " ", " ") - 1))`.as("word"),
sql`(substr(body, instr(body || " ", " ") + 1))`.as("rest"),
])
.where((eb) => eb.and([eb("body", "is not", null), eb("thread_id", "=", threadId)]))
.unionAll((ebInner) => {
return ebInner
.selectFrom("words")
.select([
sql`LOWER(substr(rest, 1, instr(rest || " ", " ") - 1))`.as("word"),
sql`(substr(rest, instr(rest || " ", " ") + 1))`.as("rest"),
])
.where("rest", "<>", "");
});
})
.selectFrom("words")
.select((eb) => ["word", eb.fn.countAll().as("count")])
.where("word", "<>", "")
.groupBy("word")
.orderBy("count desc")
.limit(limit)
.$narrowType<{
count: number;
}>()
.execute();
export const threadMostUsedWordsQuery = cached(threadMostUsedWordsQueryRaw);

4
src/lib/db-cache.ts
View file

@ -82,13 +82,13 @@ class LocalStorageCacheAdapter {
const cache = new LocalStorageCacheAdapter();
export const cached = <T, R, TT>(fn: (...args: T[]) => R, self?: ThisType<TT>): ((...args: T[]) => R) => {
export const cached = <T extends unknown[], R, TT>(fn: (...args: T) => R, self?: ThisType<TT>): ((...args: T) => R) => {
const cacheName = hashString(fn.toString()).toString();
// important to return a promise on follow-up calls even if the data is immediately available
let isPromise: boolean;
return (...args: T[]) => {
return (...args: T) => {
const cacheKey = JSON.stringify(args);
const cachedValue = cache.get<R>(cacheName, cacheKey);

100
src/pages/dm/dm-id.tsx
View file

@ -1,15 +1,16 @@
import { type Accessor, type Component, createResource, Show } from "solid-js";
import { type Accessor, type Component, createEffect, createResource, Show } from "solid-js";
import type { RouteSectionProps } from "@solidjs/router";
import { type ChartData } from "chart.js";
import { LineChart } from "~/components/ui/charts";
import { LineChart, WordCloudChart } from "~/components/ui/charts";
import { dmPartnerRecipientQuery, dmSentMessagesPerPersonOverviewQuery, SELF_ID } from "~/db";
import { dmPartnerRecipientQuery, dmSentMessagesPerPersonOverviewQuery, SELF_ID, threadMostUsedWordsQuery } from "~/db";
export const DmId: Component<RouteSectionProps> = (props) => {
const dmId = () => Number(props.params.dmid);
// the other person in the chat with name and id
const [dmPartner] = createResource(async () => {
const dmPartner = await dmPartnerRecipientQuery(dmId());
@ -26,15 +27,17 @@ export const DmId: Component<RouteSectionProps> = (props) => {
const [dmMessagesPerPerson] = createResource(() => dmSentMessagesPerPersonOverviewQuery(dmId()));
// maps all the message counts to dates
const dmMessages = () => {
return dmMessagesPerPerson()?.reduce<
{
date: Date;
totalMessages: number;
[key: number]: number;
[recipientId: number]: number;
}[]
>((prev, curr) => {
const existingDate = prev.find(({ date }) => date === curr.message_date);
if (existingDate) {
existingDate[curr.from_recipient_id] = curr.message_count;
@ -51,6 +54,8 @@ export const DmId: Component<RouteSectionProps> = (props) => {
}, []);
};
const [mostUsedWordCounts] = createResource(async () => threadMostUsedWordsQuery(dmId(), 300));
const recipients = () => {
const currentDmPartner = dmPartner();
@ -72,6 +77,31 @@ export const DmId: Component<RouteSectionProps> = (props) => {
];
};
const maxWordSize = 100;
const mostUsedWordChartData: Accessor<ChartData<"wordCloud"> | undefined> = () => {
const currentMostUsedWordCounts = mostUsedWordCounts();
if (currentMostUsedWordCounts) {
// ordered descending in db query
const highestWordCount = currentMostUsedWordCounts[0].count;
const calcWordSizeInPixels = (count: number) => {
return 10 + Math.round((maxWordSize / highestWordCount) * count);
};
return {
labels: currentMostUsedWordCounts.map(({ word }) => word),
datasets: [
{
label: "Used",
data: currentMostUsedWordCounts.map(({ count }) => calcWordSizeInPixels(count)),
},
],
};
}
};
const dateChartData: Accessor<ChartData<"line"> | undefined> = () => {
const currentDmMessages = dmMessages();
const currentRecipients = recipients();
@ -108,36 +138,50 @@ export const DmId: Component<RouteSectionProps> = (props) => {
};
return (
<Show when={dateChartData()}>
{(currentDateChartData) => (
<div class="max-h-96">
<LineChart
options={{
normalized: true,
aspectRatio: 2,
plugins: {
zoom: {
pan: {
enabled: true,
mode: "xy",
},
<div>
<Show when={dateChartData()}>
{(currentDateChartData) => (
<div class="max-h-96">
<LineChart
options={{
normalized: true,
aspectRatio: 2,
plugins: {
zoom: {
wheel: {
pan: {
enabled: true,
mode: "xy",
},
pinch: {
enabled: true,
zoom: {
wheel: {
enabled: true,
},
pinch: {
enabled: true,
},
mode: "xy",
},
mode: "xy",
},
},
},
}}
data={currentDateChartData()}
/>
</div>
)}
</Show>
}}
data={currentDateChartData()}
/>
</div>
)}
</Show>
<Show when={mostUsedWordChartData()}>
{(currentMostUsedWordChartData) => (
<div class="max-w-3xl">
<WordCloudChart
options={{
normalized: true,
}}
data={currentMostUsedWordChartData()}
/>
</div>
)}
</Show>
</div>
);
};

8
src/pages/overview/index.tsx
View file

@ -1,4 +1,4 @@
import { type Component, createEffect, createResource, Show } from "solid-js";
import { type Component, createResource, Show } from "solid-js";
import type { RouteSectionProps } from "@solidjs/router";
import { allThreadsOverviewQuery, overallSentMessagesQuery, SELF_ID } from "~/db";
@ -12,8 +12,6 @@ export const Overview: Component<RouteSectionProps> = () => {
return (await allThreadsOverviewQuery()).rows.map((row) => {
const isGroup = row.title !== null;
console.log(row);
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
const name = (
isGroup
@ -35,10 +33,6 @@ export const Overview: Component<RouteSectionProps> = () => {
});
});
createEffect(() => {
console.log(roomOverview());
});
return (
<div>
<p>All messages: {allSelfSentMessagesCount()?.messageCount as number}</p>