perf: offload long running tasks to workers, preloading dm page data
This commit is contained in:
Samuel
parent
cad305aa66
commit
df218b9a56
18 changed files with 524 additions and 297 deletions
|
|
@ -1,5 +1,7 @@
|
|||
import { createRoot, on, createDeferred } from "solid-js";
|
||||
import { on, createSignal, createEffect, createRoot, createMemo } from "solid-js";
|
||||
import { serialize, deserialize } from "seroval";
|
||||
import { createSignaledWorker } from "@solid-primitives/workers";
|
||||
import { db } from "~/db";
|
||||
|
||||
const DATABASE_HASH_PREFIX = "database";
|
||||
|
||||
|
|
@ -30,31 +32,48 @@ const hashString = (str: string) => {
|
|||
|
||||
const HASH_STORE_KEY = `${DATABASE_HASH_PREFIX}_hash`;
|
||||
|
||||
// cannot import `db` the normal way because this file is imported in ~/db.ts before the initialisation of `db` has happened
|
||||
createRoot(() => {
|
||||
void import("~/db").then(({ db }) => {
|
||||
// we use create deferred because hasing can take very long and we don't want to block the mainthread
|
||||
createDeferred(
|
||||
on(db, (currentDb) => {
|
||||
if (currentDb) {
|
||||
const newHash = hashString(new TextDecoder().decode(currentDb.export())).toString();
|
||||
const [dbHash, setDbHash] = createSignal(localStorage.getItem(HASH_STORE_KEY));
|
||||
|
||||
const oldHash = localStorage.getItem(HASH_STORE_KEY);
|
||||
|
||||
if (newHash !== oldHash) {
|
||||
clearDbCache();
|
||||
|
||||
localStorage.setItem(HASH_STORE_KEY, newHash);
|
||||
}
|
||||
// offloaded because this can take a long time (>1s easily) and would block the mainthread
|
||||
createSignaledWorker({
|
||||
input: db,
|
||||
output: setDbHash,
|
||||
func: function hashDb(currentDb: ReturnType<typeof db>) {
|
||||
const hashString = (str: string) => {
|
||||
let hash = 0,
|
||||
i,
|
||||
chr;
|
||||
if (str.length === 0) return hash;
|
||||
for (i = 0; i < str.length; i++) {
|
||||
chr = str.charCodeAt(i);
|
||||
hash = (hash << 5) - hash + chr;
|
||||
hash |= 0; // Convert to 32bit integer
|
||||
}
|
||||
}),
|
||||
);
|
||||
return hash;
|
||||
};
|
||||
|
||||
if (currentDb?.export) {
|
||||
return hashString(new TextDecoder().decode(currentDb.export())).toString();
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
createEffect(() => {
|
||||
on(dbHash, (currentDbHash) => {
|
||||
if (currentDbHash) {
|
||||
clearDbCache();
|
||||
|
||||
localStorage.setItem(HASH_STORE_KEY, currentDbHash);
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
class LocalStorageCacheAdapter {
|
||||
keys = new Set<string>(Object.keys(localStorage).filter((key) => key.startsWith(this.prefix)));
|
||||
prefix = "database";
|
||||
#dbLoaded = createMemo(() => !!db());
|
||||
|
||||
#createKey(cacheName: string, key: string): string {
|
||||
return `${this.prefix}-${cacheName}-${key}`;
|
||||
|
|
@ -76,14 +95,24 @@ class LocalStorageCacheAdapter {
|
|||
}
|
||||
|
||||
has(cacheName: string, key: string): boolean {
|
||||
return this.keys.has(this.#createKey(cacheName, key));
|
||||
if (this.#dbLoaded()) {
|
||||
return this.keys.has(this.#createKey(cacheName, key));
|
||||
}
|
||||
|
||||
console.info("No database loaded");
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-parameters
|
||||
get<R>(cacheName: string, key: string): R | undefined {
|
||||
const item = localStorage.getItem(this.#createKey(cacheName, key));
|
||||
if (item) {
|
||||
return deserialize(item) as R;
|
||||
if (this.#dbLoaded()) {
|
||||
const item = localStorage.getItem(this.#createKey(cacheName, key));
|
||||
|
||||
if (item) {
|
||||
return deserialize(item) as R;
|
||||
}
|
||||
} else {
|
||||
console.info("No database loaded");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
109
src/lib/messages-worker.ts
Normal file
109
src/lib/messages-worker.ts
Normal file
|
|
@ -0,0 +1,109 @@
|
|||
import { getHourList, getMonthList, getWeekdayList } from "./date";
|
||||
import type { MessageOverview, MessageStats, Recipients } from "~/types";
|
||||
|
||||
const hourNames = getHourList();
|
||||
|
||||
const initialHourMap = [...hourNames.keys()];
|
||||
|
||||
const monthNames = getMonthList();
|
||||
|
||||
const initialMonthMap = [...monthNames.keys()];
|
||||
|
||||
const weekdayNames = getWeekdayList();
|
||||
|
||||
const initialWeekdayMap = [...weekdayNames.keys()];
|
||||
|
||||
function createMessageStatsSourcesRaw(messageOverview: MessageOverview, recipients: Recipients) {
|
||||
const getDateList = (startDate: Date, endDate: Date): Date[] => {
|
||||
const dateArray = new Array();
|
||||
|
||||
// end date for loop has to be one date after because we increment after adding the date in the loop
|
||||
const endDateForLoop = new Date(endDate);
|
||||
|
||||
endDateForLoop.setDate(endDateForLoop.getDate() + 1);
|
||||
|
||||
let currentDate = startDate;
|
||||
|
||||
while (currentDate <= endDateForLoop) {
|
||||
dateArray.push(new Date(currentDate));
|
||||
|
||||
const newDate = new Date(currentDate);
|
||||
newDate.setDate(newDate.getDate() + 1);
|
||||
|
||||
currentDate = newDate;
|
||||
}
|
||||
|
||||
return dateArray;
|
||||
};
|
||||
|
||||
const initialRecipientMap = () => {
|
||||
return Object.fromEntries(recipients.map(({ recipientId }) => [recipientId, 0]));
|
||||
};
|
||||
|
||||
const dateList = () => {
|
||||
const firstDate = messageOverview?.at(0)?.messageDate;
|
||||
const lastDate = messageOverview?.at(-1)?.messageDate;
|
||||
if (firstDate && lastDate) {
|
||||
return getDateList(firstDate, lastDate).map((date) => ({
|
||||
totalMessages: 0,
|
||||
date,
|
||||
...initialRecipientMap(),
|
||||
}));
|
||||
}
|
||||
};
|
||||
|
||||
const currentDateList = dateList();
|
||||
const currentInitialRecipientMap = initialRecipientMap();
|
||||
|
||||
const messageStats: MessageStats = {
|
||||
person: { ...currentInitialRecipientMap },
|
||||
month: initialMonthMap.map(() => ({ ...currentInitialRecipientMap })),
|
||||
date: currentDateList ?? [],
|
||||
weekday: initialWeekdayMap.map(() => ({ ...currentInitialRecipientMap })),
|
||||
daytime: initialHourMap.map(() => ({ ...currentInitialRecipientMap })),
|
||||
};
|
||||
|
||||
if (currentDateList) {
|
||||
const { person, month, date, weekday, daytime } = messageStats;
|
||||
|
||||
for (const message of messageOverview) {
|
||||
const { messageDate } = message;
|
||||
|
||||
// increment overall message count of a person
|
||||
person[message.fromRecipientId] += 1;
|
||||
|
||||
// increment the message count of the message's month for this recipient
|
||||
month[messageDate.getMonth()][message.fromRecipientId] += 1;
|
||||
|
||||
// biome-ignore lint/style/noNonNullAssertion: <explanation>
|
||||
const dateStatsEntry = date.find(({ date }) => date.toDateString() === messageDate.toDateString())!;
|
||||
|
||||
// increment the message count of the message's date for this recipient
|
||||
dateStatsEntry[message.fromRecipientId] += 1;
|
||||
|
||||
// increment the overall message count of the message's date
|
||||
dateStatsEntry.totalMessages += 1;
|
||||
|
||||
const weekdayOfDate = messageDate.getDay();
|
||||
// we index starting with monday while the `Date` object indexes starting with Sunday
|
||||
const weekdayIndex = weekdayOfDate === 0 ? 6 : weekdayOfDate - 1;
|
||||
|
||||
// increment the message count of the message's weekday for this recipient
|
||||
weekday[weekdayIndex][message.fromRecipientId] += 1;
|
||||
|
||||
// increment the message count of the message's daytime for this recipient
|
||||
daytime[messageDate.getHours()][message.fromRecipientId] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return messageStats;
|
||||
}
|
||||
|
||||
self.onmessage = (event: MessageEvent<{ dmId: number; messageOverview: MessageOverview; recipients: Recipients }>) => {
|
||||
const result = createMessageStatsSourcesRaw(event.data.messageOverview, event.data.recipients);
|
||||
|
||||
postMessage({
|
||||
dmId: event.data.dmId,
|
||||
messageStatsSources: result,
|
||||
});
|
||||
};
|
||||
|
|
@ -1,82 +1,37 @@
|
|||
import { getDateList, getHourList, getMonthList, getWeekdayList } from "./date";
|
||||
import type { MessageOverview, MessageStats, Recipients } from "~/types";
|
||||
import { isSameDay } from "date-fns";
|
||||
import { cached } from "./db-cache";
|
||||
import { getHourList, getMonthList, getWeekdayList } from "./date";
|
||||
import MessageStatsWorker from "./messages-worker?worker";
|
||||
|
||||
export const hourNames = getHourList();
|
||||
|
||||
const initialHoursMap = [...hourNames.keys()];
|
||||
|
||||
export const monthNames = getMonthList();
|
||||
|
||||
const initialMonthMap = [...monthNames.keys()];
|
||||
|
||||
export const weekdayNames = getWeekdayList();
|
||||
|
||||
const initialWeekdayMap = [...weekdayNames.keys()];
|
||||
const messageStatsWorker = new MessageStatsWorker();
|
||||
|
||||
const createMessageStatsSourcesRaw = (messageOverview: MessageOverview, recipients: Recipients) => {
|
||||
const initialRecipientMap = () => {
|
||||
return Object.fromEntries(recipients.map(({ recipientId }) => [recipientId, 0]));
|
||||
};
|
||||
export const createMessageStatsSources = cached(
|
||||
(dmId: number, messageOverview: MessageOverview, recipients: Recipients) => {
|
||||
messageStatsWorker.postMessage({
|
||||
dmId,
|
||||
messageOverview,
|
||||
recipients,
|
||||
});
|
||||
|
||||
const dateList = () => {
|
||||
const firstDate = messageOverview?.at(0)?.messageDate;
|
||||
const lastDate = messageOverview?.at(-1)?.messageDate;
|
||||
if (firstDate && lastDate) {
|
||||
return getDateList(firstDate, lastDate).map((date) => ({
|
||||
totalMessages: 0,
|
||||
date,
|
||||
...initialRecipientMap(),
|
||||
}));
|
||||
}
|
||||
};
|
||||
return new Promise<MessageStats>((resolve) => {
|
||||
const listener = (
|
||||
event: MessageEvent<{
|
||||
dmId: number;
|
||||
messageStatsSources: MessageStats;
|
||||
}>,
|
||||
) => {
|
||||
if (event.data.dmId === dmId) {
|
||||
resolve(event.data.messageStatsSources);
|
||||
}
|
||||
};
|
||||
|
||||
const currentDateList = dateList();
|
||||
const currentInitialRecipientMap = initialRecipientMap();
|
||||
|
||||
const messageStats: MessageStats = {
|
||||
person: { ...currentInitialRecipientMap },
|
||||
month: initialMonthMap.map(() => ({ ...currentInitialRecipientMap })),
|
||||
date: currentDateList ?? [],
|
||||
weekday: initialWeekdayMap.map(() => ({ ...currentInitialRecipientMap })),
|
||||
daytime: initialHoursMap.map(() => ({ ...currentInitialRecipientMap })),
|
||||
};
|
||||
|
||||
if (currentDateList) {
|
||||
const { person, month, date, weekday, daytime } = messageStats;
|
||||
|
||||
for (const message of messageOverview) {
|
||||
const { messageDate } = message;
|
||||
|
||||
// increment overall message count of a person
|
||||
person[message.fromRecipientId] += 1;
|
||||
|
||||
// increment the message count of the message's month for this recipient
|
||||
month[messageDate.getMonth()][message.fromRecipientId] += 1;
|
||||
|
||||
// biome-ignore lint/style/noNonNullAssertion: <explanation>
|
||||
const dateStatsEntry = date.find(({ date }) => isSameDay(date, messageDate))!;
|
||||
|
||||
// increment the message count of the message's date for this recipient
|
||||
dateStatsEntry[message.fromRecipientId] += 1;
|
||||
|
||||
// increment the overall message count of the message's date
|
||||
dateStatsEntry.totalMessages += 1;
|
||||
|
||||
const weekdayOfDate = messageDate.getDay();
|
||||
// we index starting with monday while the `Date` object indexes starting with Sunday
|
||||
const weekdayIndex = weekdayOfDate === 0 ? 6 : weekdayOfDate - 1;
|
||||
|
||||
// increment the message count of the message's weekday for this recipient
|
||||
weekday[weekdayIndex][message.fromRecipientId] += 1;
|
||||
|
||||
// increment the message count of the message's daytime for this recipient
|
||||
daytime[messageDate.getHours()][message.fromRecipientId] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
return messageStats;
|
||||
};
|
||||
|
||||
export const createMessageStatsSources = cached(createMessageStatsSourcesRaw);
|
||||
messageStatsWorker.addEventListener("message", listener);
|
||||
});
|
||||
},
|
||||
);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue