fix: chunked processing (partly)
This commit is contained in:
Samuel
parent
a581e2326b
commit
b3e3eb7270
10 changed files with 758 additions and 108 deletions
175
test/.gitignore
vendored
Normal file
175
test/.gitignore
vendored
Normal file
|
|
@ -0,0 +1,175 @@
|
|||
# Based on https://raw.githubusercontent.com/github/gitignore/main/Node.gitignore
|
||||
|
||||
# Logs
|
||||
|
||||
logs
|
||||
_.log
|
||||
npm-debug.log_
|
||||
yarn-debug.log*
|
||||
yarn-error.log*
|
||||
lerna-debug.log*
|
||||
.pnpm-debug.log*
|
||||
|
||||
# Caches
|
||||
|
||||
.cache
|
||||
|
||||
# Diagnostic reports (https://nodejs.org/api/report.html)
|
||||
|
||||
report.[0-9]_.[0-9]_.[0-9]_.[0-9]_.json
|
||||
|
||||
# Runtime data
|
||||
|
||||
pids
|
||||
_.pid
|
||||
_.seed
|
||||
*.pid.lock
|
||||
|
||||
# Directory for instrumented libs generated by jscoverage/JSCover
|
||||
|
||||
lib-cov
|
||||
|
||||
# Coverage directory used by tools like istanbul
|
||||
|
||||
coverage
|
||||
*.lcov
|
||||
|
||||
# nyc test coverage
|
||||
|
||||
.nyc_output
|
||||
|
||||
# Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files)
|
||||
|
||||
.grunt
|
||||
|
||||
# Bower dependency directory (https://bower.io/)
|
||||
|
||||
bower_components
|
||||
|
||||
# node-waf configuration
|
||||
|
||||
.lock-wscript
|
||||
|
||||
# Compiled binary addons (https://nodejs.org/api/addons.html)
|
||||
|
||||
build/Release
|
||||
|
||||
# Dependency directories
|
||||
|
||||
node_modules/
|
||||
jspm_packages/
|
||||
|
||||
# Snowpack dependency directory (https://snowpack.dev/)
|
||||
|
||||
web_modules/
|
||||
|
||||
# TypeScript cache
|
||||
|
||||
*.tsbuildinfo
|
||||
|
||||
# Optional npm cache directory
|
||||
|
||||
.npm
|
||||
|
||||
# Optional eslint cache
|
||||
|
||||
.eslintcache
|
||||
|
||||
# Optional stylelint cache
|
||||
|
||||
.stylelintcache
|
||||
|
||||
# Microbundle cache
|
||||
|
||||
.rpt2_cache/
|
||||
.rts2_cache_cjs/
|
||||
.rts2_cache_es/
|
||||
.rts2_cache_umd/
|
||||
|
||||
# Optional REPL history
|
||||
|
||||
.node_repl_history
|
||||
|
||||
# Output of 'npm pack'
|
||||
|
||||
*.tgz
|
||||
|
||||
# Yarn Integrity file
|
||||
|
||||
.yarn-integrity
|
||||
|
||||
# dotenv environment variable files
|
||||
|
||||
.env
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
.env.local
|
||||
|
||||
# parcel-bundler cache (https://parceljs.org/)
|
||||
|
||||
.parcel-cache
|
||||
|
||||
# Next.js build output
|
||||
|
||||
.next
|
||||
out
|
||||
|
||||
# Nuxt.js build / generate output
|
||||
|
||||
.nuxt
|
||||
|
||||
|
||||
# Gatsby files
|
||||
|
||||
# Comment in the public line in if your project uses Gatsby and not Next.js
|
||||
|
||||
# https://nextjs.org/blog/next-9-1#public-directory-support
|
||||
|
||||
# public
|
||||
|
||||
# vuepress build output
|
||||
|
||||
.vuepress/dist
|
||||
|
||||
# vuepress v2.x temp and cache directory
|
||||
|
||||
.temp
|
||||
|
||||
# Docusaurus cache and generated files
|
||||
|
||||
.docusaurus
|
||||
|
||||
# Serverless directories
|
||||
|
||||
.serverless/
|
||||
|
||||
# FuseBox cache
|
||||
|
||||
.fusebox/
|
||||
|
||||
# DynamoDB Local files
|
||||
|
||||
.dynamodb/
|
||||
|
||||
# TernJS port file
|
||||
|
||||
.tern-port
|
||||
|
||||
# Stores VSCode versions used for testing VSCode extensions
|
||||
|
||||
.vscode-test
|
||||
|
||||
# yarn v2
|
||||
|
||||
.yarn/cache
|
||||
.yarn/unplugged
|
||||
.yarn/build-state.yml
|
||||
.yarn/install-state.gz
|
||||
.pnp.*
|
||||
|
||||
# IntelliJ based IDEs
|
||||
.idea
|
||||
|
||||
# Finder (MacOS) folder config
|
||||
.DS_Store
|
||||
15
test/README.md
Normal file
15
test/README.md
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
# test
|
||||
|
||||
To install dependencies:
|
||||
|
||||
```bash
|
||||
bun install
|
||||
```
|
||||
|
||||
To run:
|
||||
|
||||
```bash
|
||||
bun run index.js
|
||||
```
|
||||
|
||||
This project was created using `bun init` in bun v1.1.20. [Bun](https://bun.sh) is a fast all-in-one JavaScript runtime.
|
||||
21
test/dist/index.html
vendored
Normal file
21
test/dist/index.html
vendored
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||||
<meta name="theme-color" content="#000000" />
|
||||
<!-- <link rel="shortcut icon" type="image/ico" href="/src/assets/favicon.ico" /> -->
|
||||
</head>
|
||||
|
||||
<body>
|
||||
<noscript>You need to enable JavaScript to run this app.</noscript>
|
||||
<div id="root">
|
||||
<form id="test-form">
|
||||
<input type="password" id="passphrase-input" />
|
||||
<input type="file" id="backup-input" />
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<script src="./index.js" type="module"></script>
|
||||
</body>
|
||||
</html>
|
||||
99
test/dist/index.js
vendored
Normal file
99
test/dist/index.js
vendored
Normal file
|
|
@ -0,0 +1,99 @@
|
|||
import init, { BackupDecryptor } from "../pkg/signal_decrypt_backup_wasm.js";
|
||||
|
||||
let isInitialized = false;
|
||||
|
||||
async function initialize() {
|
||||
if (!isInitialized) {
|
||||
await init().catch((err) =>
|
||||
console.error("Failed to initialize WASM module: ", err),
|
||||
);
|
||||
isInitialized = true;
|
||||
}
|
||||
}
|
||||
|
||||
export async function decryptBackup(file, passphrase, progressCallback) {
|
||||
await initialize();
|
||||
|
||||
console.log("Starting decryption of file size:", file.size);
|
||||
const decryptor = new BackupDecryptor();
|
||||
const chunkSize = 1024 * 1024 * 40; // 40MB chunks
|
||||
let offset = 0;
|
||||
let percent;
|
||||
|
||||
try {
|
||||
while (offset < file.size) {
|
||||
const newPercent = Math.round((100 / file.size) * offset);
|
||||
|
||||
if (newPercent !== percent) {
|
||||
percent = newPercent;
|
||||
console.info(`${percent}% done`);
|
||||
}
|
||||
|
||||
console.log(`Processing chunk at offset ${offset}`);
|
||||
const chunk = file.slice(offset, offset + chunkSize);
|
||||
const arrayBuffer = await chunk.arrayBuffer();
|
||||
const uint8Array = new Uint8Array(arrayBuffer);
|
||||
|
||||
decryptor.feed_data(uint8Array);
|
||||
|
||||
let done = false;
|
||||
while (!done) {
|
||||
try {
|
||||
done = await decryptor.process_chunk(passphrase, progressCallback);
|
||||
} catch (e) {
|
||||
console.error("Error processing chunk:", e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
offset += chunkSize;
|
||||
console.log(`Completed chunk, new offset: ${offset}`);
|
||||
if (performance.memory) {
|
||||
const memoryInfo = performance.memory;
|
||||
console.log(`Total JS Heap Size: ${memoryInfo.totalJSHeapSize} bytes`);
|
||||
console.log(`Used JS Heap Size: ${memoryInfo.usedJSHeapSize} bytes`);
|
||||
console.log(`JS Heap Size Limit: ${memoryInfo.jsHeapSizeLimit} bytes`);
|
||||
} else {
|
||||
console.log("Memory information is not available in this environment.");
|
||||
}
|
||||
}
|
||||
|
||||
console.log("All chunks processed, finishing up");
|
||||
console.log(window.performance.measureUserAgentSpecificMemory());
|
||||
return decryptor.finish();
|
||||
} catch (e) {
|
||||
console.error("Decryption failed:", e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
|
||||
async function decrypt(file, passphrase) {
|
||||
try {
|
||||
const result = await decryptBackup(file, passphrase);
|
||||
|
||||
console.log("Database bytes length:", result.databaseBytes.length);
|
||||
console.log("Preferences:", result.preferences);
|
||||
console.log("Key values:", result.keyValues);
|
||||
|
||||
// Example: Convert database bytes to SQL statements
|
||||
const sqlStatements = new TextDecoder().decode(result.databaseBytes);
|
||||
console.log("SQL statements:", sqlStatements);
|
||||
} catch (error) {
|
||||
console.error("Decryption failed:", error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @type {HTMLInputElement}
|
||||
*/
|
||||
const passphraseInput = document.querySelector("#passphrase-input");
|
||||
/**
|
||||
* @type {HTMLInputElement}
|
||||
*/
|
||||
const backupInput = document.querySelector("#backup-input");
|
||||
|
||||
backupInput.addEventListener("change", (event) => {
|
||||
const file = event.currentTarget.files[0];
|
||||
|
||||
decrypt(file, passphraseInput.value);
|
||||
});
|
||||
34
test/index.js
Normal file
34
test/index.js
Normal file
|
|
@ -0,0 +1,34 @@
|
|||
const server = Bun.serve({
|
||||
static: {
|
||||
"/": new Response(await Bun.file("./dist/index.html").bytes(), {
|
||||
headers: {
|
||||
"Content-Type": "text/html",
|
||||
},
|
||||
}),
|
||||
"/index.js": new Response(await Bun.file("./dist/index.js").bytes(), {
|
||||
headers: {
|
||||
"Content-Type": "text/javascript",
|
||||
},
|
||||
}),
|
||||
"/pkg/signal_decrypt_backup_wasm.js": new Response(
|
||||
await Bun.file("../pkg/signal_decrypt_backup_wasm.js").bytes(),
|
||||
{
|
||||
headers: {
|
||||
"Content-Type": "text/javascript",
|
||||
},
|
||||
},
|
||||
),
|
||||
"/pkg/signal_decrypt_backup_wasm_bg.wasm": new Response(
|
||||
await Bun.file("../pkg/signal_decrypt_backup_wasm_bg.wasm").bytes(),
|
||||
{
|
||||
headers: {
|
||||
"Content-Type": "application/wasm",
|
||||
},
|
||||
},
|
||||
),
|
||||
},
|
||||
fetch(req) {
|
||||
console.log(req);
|
||||
return new Response("404!");
|
||||
},
|
||||
});
|
||||
27
test/jsconfig.json
Normal file
27
test/jsconfig.json
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
{
|
||||
"compilerOptions": {
|
||||
// Enable latest features
|
||||
"lib": ["ESNext", "DOM"],
|
||||
"target": "ESNext",
|
||||
"module": "ESNext",
|
||||
"moduleDetection": "force",
|
||||
"jsx": "react-jsx",
|
||||
"allowJs": true,
|
||||
|
||||
// Bundler mode
|
||||
"moduleResolution": "bundler",
|
||||
"allowImportingTsExtensions": true,
|
||||
"verbatimModuleSyntax": true,
|
||||
"noEmit": true,
|
||||
|
||||
// Best practices
|
||||
"strict": true,
|
||||
"skipLibCheck": true,
|
||||
"noFallthroughCasesInSwitch": true,
|
||||
|
||||
// Some stricter flags (disabled by default)
|
||||
"noUnusedLocals": false,
|
||||
"noUnusedParameters": false,
|
||||
"noPropertyAccessFromIndexSignature": false
|
||||
}
|
||||
}
|
||||
11
test/package.json
Normal file
11
test/package.json
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
{
|
||||
"name": "test",
|
||||
"module": "index.js",
|
||||
"type": "module",
|
||||
"devDependencies": {
|
||||
"@types/bun": "latest"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"typescript": "^5.0.0"
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue