diff --git a/packages/simplex-chat-nodejs/src/libs.ts b/packages/simplex-chat-nodejs/src/libs.ts index 43980cccb2..b8888ede1d 100644 --- a/packages/simplex-chat-nodejs/src/libs.ts +++ b/packages/simplex-chat-nodejs/src/libs.ts @@ -1,3 +1,4 @@ +import * as crypto from "crypto" import * as fs from "fs" import * as http from "http" import * as https from "https" @@ -6,8 +7,7 @@ import * as path from "path" import {pipeline} from "stream/promises" import extract = require("extract-zip") import type {Backend} from "./core" - -export const LIBS_RELEASE = "core-7.1.0.11" +import {ADDON_RELEASE, LIBS_RELEASE, SHA256} from "./release" const GITHUB_REPO = "simplex-chat/simplex-chat-libs" const REQUEST_TIMEOUT_MS = 60_000 @@ -45,21 +45,23 @@ export function cacheRoot(platform: string, env: NodeJS.ProcessEnv, home: string return path.join(env.XDG_CACHE_HOME || path.join(home, ".cache"), "simplex-chat") } -function releaseUrl(asset: string): string { - return `https://github.com/${GITHUB_REPO}/releases/download/${LIBS_RELEASE}/${asset}` -} - -export function libsUrl(backend: Backend, tag: string): string { +export function libsAsset(backend: Backend, tag: string): string { const suffix = backend === "postgres" ? "-postgres" : "" - return releaseUrl(`simplex-chat-libs-${tag}${suffix}.zip`) + return `simplex-chat-libs-${tag}${suffix}.zip` } -function addonName(tag: string): string { +export function addonAsset(tag: string): string { return `simplex-chat-nodejs-${tag}.node` } -export function addonUrl(tag: string): string { - return releaseUrl(addonName(tag)) +export function releaseUrl(release: string, asset: string): string { + return `https://github.com/${GITHUB_REPO}/releases/download/${release}/${asset}` +} + +function pinnedSha256(asset: string): string { + const sha256 = SHA256[asset] + if (!sha256) throw new Error(`no SHA-256 is pinned for ${asset}`) + return sha256 } export function libPath(dir: string): string { @@ -83,7 +85,10 @@ export async function resolveLibsDir( throw new Error(`postgres backend is only supported on linux-x86_64; current platform is ${tag}`) } const target = path.resolve(cacheRoot(platform, env), LIBS_RELEASE, backend) - if (!fs.existsSync(path.join(target, lib))) await installLibs(libsUrl(backend, tag), target, lib) + if (!fs.existsSync(path.join(target, lib))) { + const asset = libsAsset(backend, tag) + await installLibs(releaseUrl(LIBS_RELEASE, asset), pinnedSha256(asset), target, lib) + } return target } @@ -97,17 +102,19 @@ export async function resolveAddonPath( if (!fs.existsSync(file)) throw new Error(`SIMPLEX_ADDON_PATH does not exist: ${file}`) return file } - const tag = platformTag(platform, arch) - const target = path.resolve(cacheRoot(platform, env), LIBS_RELEASE, addonName(tag)) - if (!fs.existsSync(target)) await installAddon(addonUrl(tag), target) + if (!ADDON_RELEASE) throw new Error("no prebuilt Node.js add-on is pinned in this version; set SIMPLEX_ADDON_PATH") + const asset = addonAsset(platformTag(platform, arch)) + const target = path.resolve(cacheRoot(platform, env), ADDON_RELEASE, asset) + if (!fs.existsSync(target)) await installAddon(releaseUrl(ADDON_RELEASE, asset), pinnedSha256(asset), target) return target } -export function installLibs(url: string, target: string, lib: string, timeoutMs: number = REQUEST_TIMEOUT_MS): Promise { +export function installLibs(url: string, sha256: string, target: string, lib: string, timeoutMs: number = REQUEST_TIMEOUT_MS): Promise { return inTempDir(target, async tmp => { console.error(`Downloading libsimplex from ${url} ...`) const zipPath = path.join(tmp, "libs.zip") await download(url, zipPath, timeoutMs) + await verifySha256(zipPath, sha256, url) await extract(zipPath, {dir: tmp}) const extracted = path.join(tmp, "libs") if (!fs.existsSync(path.join(extracted, lib))) throw new Error(`libs/${lib} missing from ${url}`) @@ -115,15 +122,23 @@ export function installLibs(url: string, target: string, lib: string, timeoutMs: }) } -export function installAddon(url: string, target: string, timeoutMs: number = REQUEST_TIMEOUT_MS): Promise { +export function installAddon(url: string, sha256: string, target: string, timeoutMs: number = REQUEST_TIMEOUT_MS): Promise { return inTempDir(target, async tmp => { console.error(`Downloading the Node.js add-on from ${url} ...`) const file = path.join(tmp, path.basename(target)) await download(url, file, timeoutMs) + await verifySha256(file, sha256, url) await renameInto(file, target, target) }) } +async function verifySha256(file: string, expected: string, url: string): Promise { + const hash = crypto.createHash("sha256") + for await (const chunk of fs.createReadStream(file)) hash.update(chunk) + const actual = hash.digest("hex") + if (actual !== expected) throw new Error(`SHA-256 of ${url} is ${actual}, expected ${expected}`) +} + // The temp dir is next to the target, so the final rename stays on one filesystem. async function inTempDir(target: string, install: (tmp: string) => Promise): Promise { const parent = path.dirname(target) diff --git a/packages/simplex-chat-nodejs/src/release.ts b/packages/simplex-chat-nodejs/src/release.ts new file mode 100644 index 0000000000..ccd436e250 --- /dev/null +++ b/packages/simplex-chat-nodejs/src/release.ts @@ -0,0 +1,14 @@ +// Written by scripts/ci/packages_release.py from the simplex-chat-libs releases. + +export const LIBS_RELEASE = "core-7.1.0.11" + +export const ADDON_RELEASE: string | undefined = undefined + +export const SHA256: {readonly [asset: string]: string} = { + "simplex-chat-libs-linux-aarch64.zip": "d66aa1b30b9fa20225dad8ef75a9c605961e1c0adde2ba0283e2fcac4e15d260", + "simplex-chat-libs-linux-x86_64-postgres.zip": "925b3166f969b12082ff8ee659e5f2a9933fe40120deb80c9aa3002a5ba6c803", + "simplex-chat-libs-linux-x86_64.zip": "dabad0824e8813ea4a084fbf37e1ba4cf6dcea9cd8ccadc521fec200f246208a", + "simplex-chat-libs-macos-aarch64.zip": "aa74608b149fff7508a47eaac4284dcc6d189c6ed8b7c18428df9158bc176a61", + "simplex-chat-libs-macos-x86_64.zip": "a0e21457e1077be7f71b46c1c27452945b2cee2b6c50efb0004412ce5b7042ec", + "simplex-chat-libs-windows-x86_64.zip": "c01dad574daa101a1cbfd31ddbf2b3d0d986a4a42f97d789d2f878b2050b2b4b", +} diff --git a/packages/simplex-chat-nodejs/tests/libs.test.ts b/packages/simplex-chat-nodejs/tests/libs.test.ts index 94030095c2..c8a65f6bdd 100644 --- a/packages/simplex-chat-nodejs/tests/libs.test.ts +++ b/packages/simplex-chat-nodejs/tests/libs.test.ts @@ -4,12 +4,22 @@ import * as https from "https" import * as os from "os" import * as path from "path" import {AddressInfo} from "net" -import {addonUrl, cacheRoot, installAddon, installLibs, LIBS_RELEASE, libPath, libsUrl, MAX_REDIRECTS, platformTag, resolveAddonPath, resolveLibsDir} from "../src/libs" +import * as crypto from "crypto" +import {addonAsset, cacheRoot, installAddon, installLibs, libPath, libsAsset, MAX_REDIRECTS, platformTag, releaseUrl, resolveAddonPath, resolveLibsDir} from "../src/libs" +import * as release from "../src/release" import {main, parseInstallArgs} from "../src/cli" import * as libs from "../src/libs" import {storedZip} from "./zip" +jest.mock("../src/release", () => ({LIBS_RELEASE: "core-test", ADDON_RELEASE: "nodejs-addon-test", SHA256: {}})) + const LIB = "libtest.so" +const ADDON = Buffer.from("addon") +const PINNED = release.SHA256 as {[asset: string]: string} + +function sha256(data: Buffer): string { + return crypto.createHash("sha256").update(data).digest("hex") +} const LINUX_LIB = "libsimplex.so" const {XDG_CACHE_HOME, LOCALAPPDATA, SIMPLEX_LIBS_DIR, SIMPLEX_ADDON_PATH} = process.env @@ -74,10 +84,10 @@ describe("paths", () => { }) it("uses the release URLs", () => { - const release = `https://github.com/simplex-chat/simplex-chat-libs/releases/download/${LIBS_RELEASE}` - expect(libsUrl("sqlite", "macos-aarch64")).toBe(`${release}/simplex-chat-libs-macos-aarch64.zip`) - expect(libsUrl("postgres", "linux-x86_64")).toBe(`${release}/simplex-chat-libs-linux-x86_64-postgres.zip`) - expect(addonUrl("windows-x86_64")).toBe(`${release}/simplex-chat-nodejs-windows-x86_64.node`) + const releases = "https://github.com/simplex-chat/simplex-chat-libs/releases/download" + expect(releaseUrl("core-7.1.0.11", libsAsset("sqlite", "macos-aarch64"))).toBe(`${releases}/core-7.1.0.11/simplex-chat-libs-macos-aarch64.zip`) + expect(releaseUrl("core-7.1.0.11", libsAsset("postgres", "linux-x86_64"))).toBe(`${releases}/core-7.1.0.11/simplex-chat-libs-linux-x86_64-postgres.zip`) + expect(releaseUrl("nodejs-addon-1", addonAsset("windows-x86_64"))).toBe(`${releases}/nodejs-addon-1/simplex-chat-nodejs-windows-x86_64.node`) }) it("resolves an absolute SIMPLEX_LIBS_DIR", async () => { @@ -146,9 +156,11 @@ describe("installLibs", () => { afterEach(() => fs.rmSync(tmp, {recursive: true, force: true})) const good = storedZip({[`libs/${LIB}`]: "lib", "libs/libHSdep.so": "dep"}) + const GOOD_SHA = sha256(good) routes["/good.zip"] = res => res.writeHead(200).end(good) routes["/redirect.zip"] = res => res.writeHead(302, {location: "/good.zip"}).end() - routes["/nolib.zip"] = res => res.writeHead(200).end(storedZip({"other/file": "x"})) + const nolib = storedZip({"other/file": "x"}) + routes["/nolib.zip"] = res => res.writeHead(200).end(nolib) routes["/stall.zip"] = res => { res.writeHead(200); res.write(good.subarray(0, 10)) } let loopRequests = 0 routes["/loop.zip"] = res => { loopRequests++; res.writeHead(302, {location: "/loop.zip"}).end() } @@ -182,20 +194,20 @@ describe("installLibs", () => { it("follows a redirect and installs libs/", async () => { const target = path.join(tmp, "v1", "sqlite") - await installLibs(`${base}/redirect.zip`, target, LIB) + await installLibs(`${base}/redirect.zip`, GOOD_SHA, target, LIB) expect(fs.readFileSync(path.join(target, LIB), "utf8")).toBe("lib") expect(fs.existsSync(path.join(target, "libHSdep.so"))).toBe(true) }) it("downloads into a temp dir next to the target", async () => { - await installLibs(`${base}/observed.zip`, path.join(tmp, "sqlite"), LIB) + await installLibs(`${base}/observed.zip`, GOOD_SHA, path.join(tmp, "sqlite"), LIB) expect(downloadDirs).toHaveLength(1) expect(leftovers()).toEqual([]) }) it("stops following a redirect loop", async () => { loopRequests = 0 - await expect(installLibs(`${base}/loop.zip`, path.join(tmp, "sqlite"), LIB)).rejects.toThrow("too many redirects") + await expect(installLibs(`${base}/loop.zip`, GOOD_SHA, path.join(tmp, "sqlite"), LIB)).rejects.toThrow("too many redirects") expect(loopRequests).toBe(MAX_REDIRECTS + 1) expect(leftovers()).toEqual([]) }) @@ -203,69 +215,72 @@ describe("installLibs", () => { it("follows an https redirect to another host", async () => { const requested = mockHttps() const target = path.join(tmp, "sqlite") - await installLibs("https://libs.test/cross-host.zip", target, LIB) + await installLibs("https://libs.test/cross-host.zip", GOOD_SHA, target, LIB) expect(requested).toEqual(["https://libs.test/cross-host.zip", "https://objects.libs.test/good.zip"]) expect(fs.readFileSync(path.join(target, LIB), "utf8")).toBe("lib") }) it("rejects a redirect from https to http", async () => { mockHttps("/to-http.zip") - await expect(installLibs("https://libs.test/libs.zip", path.join(tmp, "sqlite"), LIB)).rejects.toThrow("changes protocol") + await expect(installLibs("https://libs.test/libs.zip", GOOD_SHA, path.join(tmp, "sqlite"), LIB)).rejects.toThrow("changes protocol") expect(leftovers()).toEqual([]) }) it("rejects an invalid redirect location", async () => { - await expect(installLibs(`${base}/bad-location.zip`, path.join(tmp, "sqlite"), LIB)).rejects.toThrow("Invalid URL") + await expect(installLibs(`${base}/bad-location.zip`, GOOD_SHA, path.join(tmp, "sqlite"), LIB)).rejects.toThrow("Invalid URL") expect(leftovers()).toEqual([]) }) it("rejects a redirect without a location", async () => { - await expect(installLibs(`${base}/no-location.zip`, path.join(tmp, "sqlite"), LIB)).rejects.toThrow("HTTP 302") + await expect(installLibs(`${base}/no-location.zip`, GOOD_SHA, path.join(tmp, "sqlite"), LIB)).rejects.toThrow("HTTP 302") expect(leftovers()).toEqual([]) }) it("rejects 404 and cleans up", async () => { const target = path.join(tmp, "sqlite") - await expect(installLibs(`${base}/missing.zip`, target, LIB)).rejects.toThrow("HTTP 404") + await expect(installLibs(`${base}/missing.zip`, GOOD_SHA, target, LIB)).rejects.toThrow("HTTP 404") expect(leftovers()).toEqual([]) expect(fs.existsSync(target)).toBe(false) }) it("rejects a zip without the lib", async () => { const target = path.join(tmp, "sqlite") - await expect(installLibs(`${base}/nolib.zip`, target, LIB)).rejects.toThrow(`libs/${LIB} missing`) + await expect(installLibs(`${base}/nolib.zip`, sha256(nolib), target, LIB)).rejects.toThrow(`libs/${LIB} missing`) expect(leftovers()).toEqual([]) }) it("times out a request without a response", async () => { - await expect(installLibs(`${base}/hang.zip`, path.join(tmp, "sqlite"), LIB, 200)).rejects.toThrow("timeout") + await expect(installLibs(`${base}/hang.zip`, GOOD_SHA, path.join(tmp, "sqlite"), LIB, 200)).rejects.toThrow("timeout") expect(leftovers()).toEqual([]) }) it("times out a stalled download", async () => { const target = path.join(tmp, "sqlite") - await expect(installLibs(`${base}/stall.zip`, target, LIB, 200)).rejects.toThrow("timeout") + await expect(installLibs(`${base}/stall.zip`, GOOD_SHA, target, LIB, 200)).rejects.toThrow("timeout") expect(leftovers()).toEqual([]) }) it.each(["sqlite", "postgres"] as const)("installs %s into a relative XDG_CACHE_HOME and returns an absolute path", async backend => { const zip = storedZip({[`libs/${LINUX_LIB}`]: "lib"}) + const asset = libsAsset(backend, "linux-x86_64") + PINNED[asset] = sha256(zip) routes["/libsimplex.zip"] = res => res.writeHead(200).end(zip) const requested = mockHttps("/libsimplex.zip") const cache = path.relative(process.cwd(), path.join(tmp, "cache")) const dir = await resolveLibsDir(backend, {XDG_CACHE_HOME: cache}, "linux", "x64") - expect(requested).toEqual([libsUrl(backend, "linux-x86_64")]) + expect(requested).toEqual([releaseUrl("core-test", asset)]) expect(path.isAbsolute(dir)).toBe(true) - expect(dir).toBe(path.join(tmp, "cache", "simplex-chat", LIBS_RELEASE, backend)) + expect(dir).toBe(path.join(tmp, "cache", "simplex-chat", "core-test", backend)) expect(fs.readFileSync(path.join(dir, LINUX_LIB), "utf8")).toBe("lib") }) it("downloads the add-on into the cache", async () => { - routes["/addon.node"] = res => res.writeHead(200).end("addon") + routes["/addon.node"] = res => res.writeHead(200).end(ADDON) + PINNED[addonAsset("linux-x86_64")] = sha256(ADDON) const requested = mockHttps("/addon.node") const addon = await resolveAddonPath({XDG_CACHE_HOME: tmp}, "linux", "x64") - const version = path.join(tmp, "simplex-chat", LIBS_RELEASE) - expect(requested).toEqual([addonUrl("linux-x86_64")]) + const version = path.join(tmp, "simplex-chat", "nodejs-addon-test") + expect(requested).toEqual([releaseUrl("nodejs-addon-test", addonAsset("linux-x86_64"))]) expect(addon).toBe(path.join(version, "simplex-chat-nodejs-linux-x86_64.node")) expect(fs.readFileSync(addon, "utf8")).toBe("addon") expect(fs.readdirSync(version)).toEqual(["simplex-chat-nodejs-linux-x86_64.node"]) @@ -273,31 +288,56 @@ describe("installLibs", () => { it("does not download a cached add-on", async () => { const requested = mockHttps("/addon.node") - const cached = path.join(cacheRoot("darwin", {}), LIBS_RELEASE, "simplex-chat-nodejs-macos-aarch64.node") + const cached = path.join(cacheRoot("darwin", {}), "nodejs-addon-test", "simplex-chat-nodejs-macos-aarch64.node") fs.mkdirSync(path.dirname(cached), {recursive: true}) fs.writeFileSync(cached, "addon") await expect(resolveAddonPath({}, "darwin", "arm64")).resolves.toBe(cached) expect(requested).toEqual([]) }) + it("rejects a download with another SHA-256 and cleans up", async () => { + const target = path.join(tmp, "sqlite") + await expect(installLibs(`${base}/good.zip`, sha256(ADDON), target, LIB)).rejects.toThrow(`SHA-256 of ${base}/good.zip is ${GOOD_SHA}, expected ${sha256(ADDON)}`) + expect(fs.readdirSync(tmp)).toEqual([]) + }) + + it("rejects an add-on with another SHA-256", async () => { + routes["/addon.node"] = res => res.writeHead(200).end(ADDON) + const target = path.join(tmp, "simplex.node") + await expect(installAddon(`${base}/addon.node`, GOOD_SHA, target)).rejects.toThrow("SHA-256 of") + expect(fs.readdirSync(tmp)).toEqual([]) + }) + + it("rejects an asset without a pinned SHA-256", async () => { + mockHttps("/good.zip") + await expect(resolveLibsDir("sqlite", {XDG_CACHE_HOME: tmp}, "darwin", "x64")) + .rejects.toThrow("no SHA-256 is pinned for simplex-chat-libs-macos-x86_64.zip") + }) + + it("requires SIMPLEX_ADDON_PATH without a pinned add-on release", async () => { + jest.replaceProperty(release, "ADDON_RELEASE", undefined) + await expect(resolveAddonPath({XDG_CACHE_HOME: tmp}, "linux", "x64")) + .rejects.toThrow("no prebuilt Node.js add-on is pinned in this version; set SIMPLEX_ADDON_PATH") + }) + it("rejects a missing add-on and cleans up", async () => { const target = path.join(tmp, "simplex.node") - await expect(installAddon(`${base}/missing.node`, target)).rejects.toThrow("HTTP 404") + await expect(installAddon(`${base}/missing.node`, sha256(ADDON), target)).rejects.toThrow("HTTP 404") expect(fs.readdirSync(tmp)).toEqual([]) }) it("accepts EPERM from rename onto an installed add-on", async () => { - routes["/addon.node"] = res => res.writeHead(200).end("addon") + routes["/addon.node"] = res => res.writeHead(200).end(ADDON) const target = path.join(tmp, "simplex.node") fs.writeFileSync(target, "addon") renameFails("EPERM") - await expect(installAddon(`${base}/addon.node`, target)).resolves.toBeUndefined() + await expect(installAddon(`${base}/addon.node`, sha256(ADDON), target)).resolves.toBeUndefined() expect(leftovers()).toEqual([]) }) it("does not download a cached lib", async () => { const requested = mockHttps("/good.zip") - const cached = path.join(tmp, "simplex-chat", LIBS_RELEASE, "postgres") + const cached = path.join(tmp, "simplex-chat", "core-test", "postgres") fs.mkdirSync(cached, {recursive: true}) fs.writeFileSync(path.join(cached, LINUX_LIB), "lib") await expect(resolveLibsDir("postgres", {XDG_CACHE_HOME: tmp}, "linux", "x64")).resolves.toBe(cached) @@ -308,7 +348,7 @@ describe("installLibs", () => { const target = path.join(tmp, "sqlite") fs.mkdirSync(target) fs.writeFileSync(path.join(target, "other"), "x") - await expect(installLibs(`${base}/good.zip`, target, LIB)).rejects.toThrow(`another process partially populated ${target}`) + await expect(installLibs(`${base}/good.zip`, GOOD_SHA, target, LIB)).rejects.toThrow(`another process partially populated ${target}`) expect(leftovers()).toEqual([]) }) @@ -323,21 +363,21 @@ describe("installLibs", () => { fs.mkdirSync(target) fs.writeFileSync(path.join(target, LIB), "lib") renameFails(code) - await expect(installLibs(`${base}/good.zip`, target, LIB)).resolves.toBeUndefined() + await expect(installLibs(`${base}/good.zip`, GOOD_SHA, target, LIB)).resolves.toBeUndefined() expect(leftovers()).toEqual([]) }) it("rethrows EPERM from rename when the target does not exist", async () => { renameFails("EPERM") - await expect(installLibs(`${base}/good.zip`, path.join(tmp, "sqlite"), LIB)).rejects.toThrow("EPERM") + await expect(installLibs(`${base}/good.zip`, GOOD_SHA, path.join(tmp, "sqlite"), LIB)).rejects.toThrow("EPERM") expect(leftovers()).toEqual([]) }) it("lets concurrent installs share one target", async () => { const target = path.join(tmp, "sqlite") await Promise.all([ - installLibs(`${base}/good.zip`, target, LIB), - installLibs(`${base}/good.zip`, target, LIB), + installLibs(`${base}/good.zip`, GOOD_SHA, target, LIB), + installLibs(`${base}/good.zip`, GOOD_SHA, target, LIB), ]) expect(fs.readFileSync(path.join(target, LIB), "utf8")).toBe("lib") expect(leftovers()).toEqual([])