mirror of
https://github.com/spacebarchat/server.git
synced 2026-03-29 07:39:53 +00:00
temp unfinished json stuff in ts
This commit is contained in:
Rory&
@@ -24,7 +24,7 @@
|
||||
"generate:openapi": "node scripts/openapi.js",
|
||||
"add:license": "node scripts/license.js",
|
||||
"migrate-from-staging": "node -r dotenv/config -r module-alias/register scripts/stagingMigration/index.js",
|
||||
"node:tests": "npm run build:src && node -r dotenv/config -r module-alias/register --enable-source-maps --test --experimental-test-coverage dist/**/*.test.js"
|
||||
"node:tests": "npm run build:src:tsgo && node -r dotenv/config -r module-alias/register --enable-source-maps --test --experimental-test-coverage dist/**/*.test.js"
|
||||
},
|
||||
"main": "dist/bundle/index.js",
|
||||
"types": "src/bundle/index.ts",
|
||||
|
||||
31
src/util/util/json/JsonNode.ts
Normal file
31
src/util/util/json/JsonNode.ts
Normal file
@@ -0,0 +1,31 @@
|
||||
/// <summary>Defines the various JSON tokens that make up a JSON text.</summary>
|
||||
export enum JsonTokenType {
|
||||
/// <summary>There is no value (as distinct from <see cref="F:System.Text.Json.JsonTokenType.Null" />). This is the default token type if no data has been read by the <see cref="T:System.Text.Json.Utf8JsonReader" />.</summary>
|
||||
None,
|
||||
/// <summary>The token type is the start of a JSON object.</summary>
|
||||
StartObject,
|
||||
/// <summary>The token type is the end of a JSON object.</summary>
|
||||
EndObject,
|
||||
/// <summary>The token type is the start of a JSON array.</summary>
|
||||
StartArray,
|
||||
/// <summary>The token type is the end of a JSON array.</summary>
|
||||
EndArray,
|
||||
/// <summary>The token type is a JSON property name.</summary>
|
||||
PropertyName,
|
||||
/// <summary>The token type is a comment string.</summary>
|
||||
Comment,
|
||||
/// <summary>The token type is a JSON string.</summary>
|
||||
String,
|
||||
/// <summary>The token type is a JSON number.</summary>
|
||||
Number,
|
||||
/// <summary>The token type is the JSON literal true.</summary>
|
||||
True,
|
||||
/// <summary>The token type is the JSON literal false.</summary>
|
||||
False,
|
||||
/// <summary>The token type is the JSON literal null.</summary>
|
||||
Null,
|
||||
}
|
||||
|
||||
export class JsonNode {
|
||||
type: JsonTokenType = JsonTokenType.None;
|
||||
}
|
||||
158
src/util/util/json/JsonSerializer.test.ts
Normal file
158
src/util/util/json/JsonSerializer.test.ts
Normal file
@@ -0,0 +1,158 @@
|
||||
import { JsonSerializer } from "./JsonSerializer";
|
||||
import { describe, it } from "node:test";
|
||||
import { strict as assert } from "assert";
|
||||
import fs from "fs/promises";
|
||||
import { Stopwatch } from "../Stopwatch";
|
||||
import { JsonValue } from "@protobuf-ts/runtime";
|
||||
|
||||
describe("JsonSerializer", () => {
|
||||
it("should serialize synchronously", () => {
|
||||
const obj = { a: 1, b: "test" };
|
||||
const result = JsonSerializer.Serialize(obj);
|
||||
assert.equal(result, '{"a":1,"b":"test"}');
|
||||
});
|
||||
|
||||
it("should deserialize synchronously", () => {
|
||||
const json = '{"a":1,"b":"test"}';
|
||||
const result = JsonSerializer.Deserialize(json);
|
||||
assert.deepEqual(result, { a: 1, b: "test" });
|
||||
});
|
||||
|
||||
it("should serialize asynchronously", async () => {
|
||||
const obj = { a: 1, b: "test" };
|
||||
const result = await JsonSerializer.SerializeAsync(obj);
|
||||
assert.equal(result, '{"a":1,"b":"test"}');
|
||||
});
|
||||
|
||||
it("should deserialize asynchronously", async () => {
|
||||
const json = '{"a":1,"b":"test"}';
|
||||
const result = await JsonSerializer.DeserializeAsync(json);
|
||||
assert.deepEqual(result, { a: 1, b: "test" });
|
||||
});
|
||||
|
||||
it("should be able to read large file", async () => {
|
||||
// write a massive json file
|
||||
const sw = Stopwatch.startNew();
|
||||
const jsonfile = await fs.open("large.json", "w");
|
||||
await jsonfile.write("[");
|
||||
const getLargeObj = (index: number, depth: number) => {
|
||||
const obj: JsonValue = {};
|
||||
|
||||
if (depth === 0) {
|
||||
return obj;
|
||||
}
|
||||
for (let i = 0; i < 10; i++) {
|
||||
obj[`key${i}`] = getLargeObj(index * 10 + i, depth - 1);
|
||||
}
|
||||
return obj;
|
||||
};
|
||||
for (let i = 0; i < 100; i++) {
|
||||
const entry = JSON.stringify(getLargeObj(i, 5));
|
||||
await jsonfile.write(entry);
|
||||
if (i < 99) {
|
||||
await jsonfile.write(",");
|
||||
}
|
||||
}
|
||||
await jsonfile.write("]");
|
||||
await jsonfile.close();
|
||||
process.stdout.write("Large file written in " + sw.elapsed().toString() + "\n");
|
||||
|
||||
const jsonData = await fs.readFile("large.json", "utf-8");
|
||||
|
||||
const start = process.hrtime.bigint();
|
||||
const obj = await JsonSerializer.DeserializeAsync<{ key: string; value: string }[]>(jsonData);
|
||||
const end = process.hrtime.bigint();
|
||||
const duration = end - start;
|
||||
console.log(`Deserialization took ${duration / BigInt(1e6)} ms`);
|
||||
|
||||
assert.equal(obj.length, 100);
|
||||
await fs.unlink("large.json");
|
||||
});
|
||||
|
||||
it("should be able to parallelise", async () => {
|
||||
// write a massive json file
|
||||
const sw = Stopwatch.startNew();
|
||||
const jsonfile = await fs.open("large.json", "w");
|
||||
await jsonfile.write("[");
|
||||
const getLargeObj = (index: number, depth: number) => {
|
||||
const obj: JsonValue = {};
|
||||
|
||||
if (depth === 0) {
|
||||
return obj;
|
||||
}
|
||||
for (let i = 0; i < 5; i++) {
|
||||
obj[`key${i}`] = getLargeObj(index * 10 + i, depth - 1);
|
||||
}
|
||||
return obj;
|
||||
};
|
||||
for (let i = 0; i < 50; i++) {
|
||||
const entry = JSON.stringify(getLargeObj(i, 5));
|
||||
await jsonfile.write(entry);
|
||||
if (i < 49) {
|
||||
await jsonfile.write(",");
|
||||
}
|
||||
}
|
||||
await jsonfile.write("]");
|
||||
await jsonfile.close();
|
||||
process.stdout.write("Large file written in " + sw.elapsed().toString() + "\n");
|
||||
|
||||
const tasks = [];
|
||||
const start = process.hrtime.bigint();
|
||||
for (let i = 0; i < 64; i++) {
|
||||
tasks.push(
|
||||
(async () => {
|
||||
const jsonData = await fs.readFile("large.json", "utf-8");
|
||||
|
||||
const obj = await JsonSerializer.DeserializeAsync<{ key: string; value: string }[]>(jsonData);
|
||||
const end = process.hrtime.bigint();
|
||||
const duration = end - start;
|
||||
console.log(`Deserialization took ${duration / BigInt(1e6)} ms`);
|
||||
|
||||
assert.equal(obj.length, 50);
|
||||
})(),
|
||||
);
|
||||
}
|
||||
await Promise.all(tasks);
|
||||
await fs.unlink("large.json");
|
||||
});
|
||||
|
||||
// TODO: broken
|
||||
// it("should be able to stream large file", async () => {
|
||||
// // write a massive json file
|
||||
// const sw = Stopwatch.startNew();
|
||||
// const jsonfile = await fs.open("large.json", "w");
|
||||
// await jsonfile.write("[");
|
||||
// const getLargeObj = (index: number, depth: number) => {
|
||||
// const obj: JsonValue = {};
|
||||
//
|
||||
// if (depth === 0) {
|
||||
// return obj;
|
||||
// }
|
||||
// for (let i = 0; i < 10; i++) {
|
||||
// obj[`key${i}`] = getLargeObj(index * 10 + i, depth - 1);
|
||||
// }
|
||||
// return obj;
|
||||
// };
|
||||
// for (let i = 0; i < 100; i++) {
|
||||
// const entry = JSON.stringify(getLargeObj(i, 5));
|
||||
// await jsonfile.write(entry);
|
||||
// if (i < 99) {
|
||||
// await jsonfile.write(",");
|
||||
// }
|
||||
// }
|
||||
// await jsonfile.write("]");
|
||||
// await jsonfile.close();
|
||||
// process.stdout.write("Large file written in " + sw.elapsed().toString() + "\n");
|
||||
//
|
||||
// const jsonData = await fs.open("large.json", "r").then((f) => f.createReadStream());
|
||||
//
|
||||
// const start = process.hrtime.bigint();
|
||||
// const obj = await JsonSerializer.DeserializeAsync<{ key: string; value: string }[]>(jsonData);
|
||||
// const end = process.hrtime.bigint();
|
||||
// const duration = end - start;
|
||||
// console.log(`Deserialization took ${duration / BigInt(1e6)} ms`);
|
||||
//
|
||||
// assert.equal(obj.length, 100);
|
||||
// await fs.unlink("large.json");
|
||||
// });
|
||||
});
|
||||
166
src/util/util/json/JsonSerializer.ts
Normal file
166
src/util/util/json/JsonSerializer.ts
Normal file
@@ -0,0 +1,166 @@
|
||||
import { JsonSerializerOptions } from "./JsonSerializerOptions";
|
||||
import { Worker } from "worker_threads";
|
||||
import { join } from "path";
|
||||
import os from "os";
|
||||
import Stream from "node:stream";
|
||||
import { ReadStream, WriteStream } from "node:fs";
|
||||
|
||||
// const worker = new Worker(join(process.cwd(), 'dist', 'util', 'util', 'json', 'jsonWorker.js'));
|
||||
const workerPool: Worker[] = [];
|
||||
const numWorkers = process.env.JSON_WORKERS ? parseInt(process.env.JSON_WORKERS) : os.cpus().length;
|
||||
|
||||
for (let i = 0; i < numWorkers; i++) {
|
||||
console.log("[JsonSerializer] Starting JSON worker", i);
|
||||
workerPool.push(new Worker(join(__dirname, "jsonWorker.js")));
|
||||
workerPool[i].unref();
|
||||
workerPool[i].setMaxListeners(64);
|
||||
}
|
||||
let currentWorkerIndex = 0;
|
||||
|
||||
function getNextWorker(): Worker {
|
||||
const worker = workerPool[currentWorkerIndex];
|
||||
currentWorkerIndex = (currentWorkerIndex + 1) % numWorkers;
|
||||
return worker;
|
||||
}
|
||||
|
||||
export class JsonSerializer {
|
||||
public static Serialize<T>(value: T, opts?: JsonSerializerOptions): string {
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
public static async SerializeAsync<T>(value: T, opts?: JsonSerializerOptions): Promise<string> {
|
||||
const worker = getNextWorker();
|
||||
worker.postMessage({ type: "serialize", value });
|
||||
return new Promise((resolve, reject) => {
|
||||
const handler = (msg: { result?: string; error?: string }) => {
|
||||
clearTimeout(timeout);
|
||||
worker.removeListener("message", handler);
|
||||
if (msg.error) {
|
||||
reject(new Error(msg.error));
|
||||
} else {
|
||||
resolve(msg.result!);
|
||||
}
|
||||
};
|
||||
worker.on("message", handler);
|
||||
const timeout = setTimeout(() => {
|
||||
worker.removeListener("message", handler);
|
||||
reject(new Error("Worker timeout"));
|
||||
}, 60000);
|
||||
});
|
||||
}
|
||||
public static Deserialize<T>(json: string, opts?: JsonSerializerOptions): T {
|
||||
return JSON.parse(json) as T;
|
||||
}
|
||||
public static async DeserializeAsync<T>(json: string | ReadableStream | ReadStream, opts?: JsonSerializerOptions): Promise<T> {
|
||||
if (json instanceof ReadableStream) return this.DeserializeAsyncReadableStream<T>(json, opts);
|
||||
if (json instanceof ReadStream) return this.DeserializeAsyncReadStream<T>(json, opts);
|
||||
|
||||
const worker = getNextWorker();
|
||||
worker.postMessage({ type: "deserialize", json });
|
||||
return new Promise((resolve, reject) => {
|
||||
const handler = (msg: { result?: string; error?: string }) => {
|
||||
clearTimeout(timeout);
|
||||
worker.removeListener("message", handler);
|
||||
if (msg.error) {
|
||||
reject(new Error(msg.error));
|
||||
} else {
|
||||
resolve(JSON.parse(msg.result!) as T);
|
||||
}
|
||||
};
|
||||
worker.on("message", handler);
|
||||
const timeout = setTimeout(() => {
|
||||
worker.removeListener("message", handler);
|
||||
reject(new Error("Worker timeout"));
|
||||
}, 60000);
|
||||
});
|
||||
}
|
||||
|
||||
private static async DeserializeAsyncReadableStream<T>(jsonStream: ReadableStream, opts?: JsonSerializerOptions): Promise<T> {
|
||||
const reader = jsonStream.getReader();
|
||||
let jsonData = "";
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
jsonData += new TextDecoder().decode(value);
|
||||
}
|
||||
return this.DeserializeAsync<T>(jsonData, opts);
|
||||
}
|
||||
|
||||
private static async DeserializeAsyncReadStream<T>(jsonStream: ReadStream, opts?: JsonSerializerOptions): Promise<T> {
|
||||
let jsonData = "";
|
||||
for await (const chunk of jsonStream) {
|
||||
jsonData += chunk.toString();
|
||||
}
|
||||
return this.DeserializeAsync<T>(jsonData, opts);
|
||||
}
|
||||
|
||||
public static async *DeserializeAsyncEnumerable<T>(json: string | ReadStream | ReadableStream, opts?: JsonSerializerOptions): AsyncGenerator<T, void, unknown> {
|
||||
if (json instanceof ReadableStream) return yield* this.DeserializeAsyncEnumerableReadableStream<T>(json, opts);
|
||||
if (json instanceof ReadStream) return yield* this.DeserializeAsyncEnumerableReadStream<T>(json, opts);
|
||||
|
||||
const arr = await this.DeserializeAsync<T[]>(json, opts);
|
||||
for (const item of arr) {
|
||||
yield item;
|
||||
}
|
||||
}
|
||||
|
||||
private static async *DeserializeAsyncEnumerableReadableStream<T>(json: ReadableStream, opts?: JsonSerializerOptions) {
|
||||
const reader = json.getReader();
|
||||
//TODO: implement
|
||||
yield undefined as unknown as T;
|
||||
}
|
||||
|
||||
private static async *DeserializeAsyncEnumerableReadStream<T>(json: ReadStream, opts?: JsonSerializerOptions) {
|
||||
// TODO: implement
|
||||
yield undefined as unknown as T;
|
||||
}
|
||||
|
||||
public static async SerializeAsyncEnumerableToStringAsync<T>(items: AsyncIterable<T>, opts?: JsonSerializerOptions): Promise<string> {
|
||||
let jsonData = "[";
|
||||
let first = true;
|
||||
for await (const item of items) {
|
||||
if (!first) {
|
||||
jsonData += ",";
|
||||
} else {
|
||||
first = false;
|
||||
}
|
||||
jsonData += await this.SerializeAsync(item, opts);
|
||||
}
|
||||
jsonData += "]";
|
||||
return jsonData;
|
||||
}
|
||||
|
||||
public static async SerializeAsyncEnumerableAsync<T>(items: AsyncIterable<T>, stream: WriteStream | WritableStream, opts?: JsonSerializerOptions): Promise<void> {}
|
||||
|
||||
private static async SerializeAsyncEnumerableToWritableStreamAsync<T>(items: AsyncIterable<T>, stream: WritableStream, opts?: JsonSerializerOptions): Promise<void> {
|
||||
const writer = stream.getWriter();
|
||||
let first = true;
|
||||
await writer.write(new TextEncoder().encode("["));
|
||||
for await (const item of items) {
|
||||
if (!first) {
|
||||
await writer.write(new TextEncoder().encode(","));
|
||||
} else {
|
||||
first = false;
|
||||
}
|
||||
const jsonItem = await this.SerializeAsync(item, opts);
|
||||
await writer.write(new TextEncoder().encode(jsonItem));
|
||||
}
|
||||
await writer.write(new TextEncoder().encode("]"));
|
||||
await writer.close();
|
||||
}
|
||||
|
||||
private static async SerializeAsyncEnumerableToWriteStreamAsync<T>(items: AsyncIterable<T>, stream: WriteStream, opts?: JsonSerializerOptions): Promise<void> {
|
||||
let first = true;
|
||||
stream.write("[");
|
||||
for await (const item of items) {
|
||||
if (!first) {
|
||||
stream.write(",");
|
||||
} else {
|
||||
first = false;
|
||||
}
|
||||
const jsonItem = await this.SerializeAsync(item, opts);
|
||||
stream.write(jsonItem);
|
||||
}
|
||||
stream.write("]");
|
||||
stream.end();
|
||||
}
|
||||
}
|
||||
1
src/util/util/json/JsonSerializerOptions.ts
Normal file
1
src/util/util/json/JsonSerializerOptions.ts
Normal file
@@ -0,0 +1 @@
|
||||
export class JsonSerializerOptions {}
|
||||
4
src/util/util/json/README.RST
Normal file
4
src/util/util/json/README.RST
Normal file
@@ -0,0 +1,4 @@
|
||||
JSON utils
|
||||
=============
|
||||
|
||||
JSON library in typescript, based around Dotnet's System.Text.Json api.
|
||||
16
src/util/util/json/jsonWorker.ts
Normal file
16
src/util/util/json/jsonWorker.ts
Normal file
@@ -0,0 +1,16 @@
|
||||
import { parentPort } from "worker_threads";
|
||||
|
||||
parentPort?.on("message", (message) => {
|
||||
try {
|
||||
if (message.type === "serialize") {
|
||||
const result = JSON.stringify(message.value);
|
||||
parentPort?.postMessage({ result });
|
||||
} else if (message.type === "deserialize") {
|
||||
const parsed = JSON.parse(message.json);
|
||||
const result = JSON.stringify(parsed);
|
||||
parentPort?.postMessage({ result });
|
||||
}
|
||||
} catch (error) {
|
||||
parentPort?.postMessage({ error: (error as Error).message });
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user