mirror of
https://github.com/simplex-chat/simplexmq.git
synced 2026-10-05 20:57:16 +00:00
Merge branch 'master' into ep/socks-mode
This commit is contained in:
@@ -320,15 +320,14 @@ jobs:
|
||||
- name: Clone project
|
||||
uses: actions/checkout@v3
|
||||
|
||||
- name: Set up Python
|
||||
# Matches the runtime stage of scripts/resolver/service/Dockerfile.
|
||||
uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.13"
|
||||
- name: Set up uv
|
||||
# uv takes Python from .python-version, the version of the Dockerfile's runtime stage.
|
||||
uses: astral-sh/setup-uv@c18668ad3cf93ea998bef934396af7bb5c839dc7 # v10.2.0
|
||||
|
||||
- name: Install resolver dependencies
|
||||
# Must match scripts/resolver/service/pyproject.toml.
|
||||
run: python -m pip install "eth-hash[pycryptodome]>=0.7"
|
||||
- name: Lint
|
||||
working-directory: scripts/resolver/service
|
||||
run: uv run --frozen ruff check .
|
||||
|
||||
- name: Test
|
||||
run: python -m unittest discover -s scripts/resolver/service -v
|
||||
working-directory: scripts/resolver/service
|
||||
run: uv run --frozen pytest -v
|
||||
|
||||
@@ -1,4 +1,8 @@
|
||||
*.lock
|
||||
# the resolver's Docker image and CI install exactly these
|
||||
!/scripts/resolver/service/uv.lock
|
||||
# local overrides of scripts/resolver/.env.example
|
||||
/scripts/resolver/.env
|
||||
*.db
|
||||
*.db.bak
|
||||
*.session.sql
|
||||
|
||||
@@ -168,7 +168,7 @@ This document describes the cryptographic primitives and threat model for the Si
|
||||
|
||||
*cannot:*
|
||||
|
||||
- impersonate a sender and send messages to the user whose database was stolen. Doing so requires also compromising the router (to place the message in the queue, that is possible until the Double-Ratchet advances forward) or the user's device at a subsequent time (to place the message in the database).
|
||||
- impersonate a sender and send messages to the user whose database was stolen, unless the database contains a new rotation queue that is not secured yet. Doing so requires also compromising the router (to place the message in the queue, that is possible until the Double-Ratchet advances forward) or the user's device at a subsequent time (to place the message in the database).
|
||||
|
||||
- undetectably communicate at the same time as Alice with her contacts. Doing so would result in the contact getting different messages with repeated IDs.
|
||||
|
||||
|
||||
@@ -86,7 +86,7 @@ It's designed with the focus on communication security and integrity, under the
|
||||
|
||||
It is designed as a low level protocol for other application protocols to solve the problem of secure and private message transmission, making [MITM attack][1] very difficult at any part of the message transmission system.
|
||||
|
||||
This document describes SMP protocol version 22. Versions 1-5 are discontinued. The version history:
|
||||
This document describes SMP protocol version 23. Versions 1-5 are discontinued. The version history:
|
||||
|
||||
- v1: binary protocol encoding
|
||||
- v2: message flags (used to control notifications)
|
||||
@@ -109,6 +109,7 @@ This document describes SMP protocol version 22. Versions 1-5 are discontinued.
|
||||
- v20: public namespaces resolver (RSLV command, RNAME response) — direct or forwarded via PFWD
|
||||
- v21: server public information in handshake
|
||||
- v22: `RNAME` says whether a name can be registered, not only what it resolves to
|
||||
- v23: version in nonces of forwarded commands, random nonces for forwarded responses
|
||||
|
||||
## Introduction
|
||||
|
||||
@@ -1120,13 +1121,15 @@ When the client receives PKEY response it MUST validate that:
|
||||
- the fingerprint of the received certificate matches fingerprint in the router address - it mitigates MITM attack by proxy.
|
||||
- the router session key is correctly signed with the received certificate.
|
||||
|
||||
`smpVersionRange` in `PKEY` is not signed.
|
||||
|
||||
The proxy router may respond with error response in case the destination router is not available or in case it has an earlier version that does not support proxied commands.
|
||||
|
||||
#### Send command via proxy
|
||||
|
||||
Sender can send `SKEY` and `SEND` commands via proxy after obtaining the session ID with `PRXY` command (see [Request proxied session](#request-proxied-session)).
|
||||
|
||||
Transmission sent to proxy router should use session ID as entity ID and use a random correlation ID of 24 bytes as a nonce for crypto_box encryption of transmission to the destination router. The random ephemeral X25519 key to encrypt transmission should be unique per command, and it should be combined with the key sent by the router in the handshake header to proxy and to the client in `PKEY` command.
|
||||
Transmission sent to proxy router should use session ID as entity ID and use a random correlation ID of 24 bytes as a nonce for crypto_box encryption of transmission to the destination router. When `smpVersion` in `PFWD` is 23 or higher, the first 2 bytes of this nonce are XOR-ed with `smpVersion`. The random ephemeral X25519 key to encrypt transmission should be unique per command, and it should be combined with the key sent by the router in the handshake header to proxy and to the client in `PKEY` command.
|
||||
|
||||
Encrypted transmission should use the received session ID from the connection between proxy router and destination router in the authorized body.
|
||||
|
||||
@@ -1138,10 +1141,11 @@ commandKey = length x509encoded
|
||||
|
||||
The proxy router will forward the encrypted transmission in `RFWD` command (see below).
|
||||
|
||||
Having received the `RRES` response from the destination router, proxy router will forward `PRES` response to the client. `PRES` response should use the same correlation ID as `PFWD` command. The destination router will use this correlation ID increased by 1 as a nonce for encryption of the response.
|
||||
Having received the `RRES` response from the destination router, proxy router will forward `PRES` response to the client. `PRES` response should use the same correlation ID as `PFWD` command. The destination router will use this correlation ID increased by 1 as a nonce for encryption of the response. When `smpVersion` in `PFWD` is 23 or higher, the destination router uses a random nonce instead, and sends it before the encrypted response.
|
||||
|
||||
```abnf
|
||||
proxyResponse = %s"PRES" SP <encrypted padded(forwardedResponse, 16226)>
|
||||
proxyResponse = %s"PRES" SP [responseNonce] <encrypted padded(forwardedResponse, 16226)>
|
||||
responseNonce = %s"0" / (%s"1" 24*24 OCTET) ; from v23
|
||||
forwardedResponse = *OCTET ; client-encrypted SMP response, decrypted by client using per-command DH secret
|
||||
```
|
||||
|
||||
@@ -1168,7 +1172,7 @@ The shared secret for encrypting transmission bodies between proxy router and de
|
||||
|
||||
|
||||
```abnf
|
||||
relayResponse = %s"RRES" SP <encrypted(responseTransmission)>
|
||||
relayResponse = %s"RRES" SP [responseNonce] <encrypted(responseTransmission)>
|
||||
responseTransmission = fwdCorrId forwardedResponse
|
||||
; fwdCorrId and forwardedResponse defined above in RFWD section
|
||||
```
|
||||
|
||||
@@ -1,5 +1,8 @@
|
||||
# Copy to .env (next to docker-compose.yml) only to change a setting; without it
|
||||
# docker-compose.yml uses the values shown here.
|
||||
|
||||
# ============================================================================
|
||||
# Required settings — the stack will not start without these.
|
||||
# Chain settings.
|
||||
# ============================================================================
|
||||
|
||||
# Ethereum network: mainnet (the SNRC `.testing` contracts live on mainnet)
|
||||
@@ -14,9 +17,16 @@ NETWORK=mainnet
|
||||
TRUSTED_NODE_URL=https://mainnet-checkpoint-sync.attestant.io
|
||||
|
||||
# ============================================================================
|
||||
# Optional overrides — sensible defaults are baked into docker-compose.yml,
|
||||
# so leave these commented unless you need to change them.
|
||||
# Optional overrides — leave commented unless you need to change them.
|
||||
# ============================================================================
|
||||
|
||||
# Nimbus NAT (default: any). For a stable public node set an explicit IP:
|
||||
# NAT=extip:1.2.3.4 # your public IPv4: curl -s ifconfig.me
|
||||
|
||||
# Resolver (defaults in the README's "Load and scaling" and "Logs" sections)
|
||||
# SNRC_WORKERS=4 # worker processes; default CPU count, at most 4
|
||||
# SNRC_RPC_TIMEOUT=5 # seconds per request to reth
|
||||
# SNRC_LOG_FORMAT=json # default text (key=value)
|
||||
# SNRC_LOG_COLOR=always # coloured `docker compose logs`
|
||||
# SNRC_LOG_LEVEL=debug # also log health checks and idle connections
|
||||
# SNRC_TRUSTED_PROXIES=172.16.0.0/12 # behind a reverse proxy on the host; see README
|
||||
@@ -19,15 +19,15 @@ against **Ethereum mainnet** (where the `.testing` contracts live):
|
||||
|
||||
## 1. Configure
|
||||
|
||||
Edit `.env` — the defaults work as-is; override only if needed:
|
||||
The defaults in `docker-compose.yml` work as-is:
|
||||
|
||||
```sh
|
||||
NETWORK=mainnet # default
|
||||
TRUSTED_NODE_URL=https://mainnet-checkpoint-sync.attestant.io # default
|
||||
```
|
||||
|
||||
Everything else (NAT) has a working default baked into `docker-compose.yml`;
|
||||
uncomment the hints in `.env` only to override.
|
||||
To override these or NAT, copy `.env.example` to `.env` and edit it; `.env` is
|
||||
not committed.
|
||||
|
||||
## 2. Run
|
||||
|
||||
@@ -121,6 +121,15 @@ standalone for local dev (no Docker), via [`uv`](https://docs.astral.sh/uv/):
|
||||
uv run scripts/resolver/service/snrc-resolve.py # defaults to local reth + mainnet .testing
|
||||
```
|
||||
|
||||
Tests and lint run from `scripts/resolver/service`:
|
||||
|
||||
```sh
|
||||
uv run pytest
|
||||
uv run ruff check .
|
||||
```
|
||||
|
||||
Dependencies are locked in `uv.lock`; the Docker image installs exactly those.
|
||||
|
||||
Three routes, versioned separately from the protocol so each only changes when
|
||||
its own shape does:
|
||||
|
||||
@@ -243,7 +252,7 @@ wrong, `error` and `status` hold the same value, so one field is enough to read.
|
||||
|
||||
`upstreamError` says only which exception type the RPC call raised. The text
|
||||
goes to the resolver's log instead, because `SNRC_RPC` can carry a provider key
|
||||
and urlopen puts the URL it failed on into the message. It is also the answer
|
||||
and the exception can carry the URL it failed on. It is also the answer
|
||||
when a registrar, controller or oracle address has no contract behind it: the
|
||||
empty reply is refused rather than read as zero, which would make every name
|
||||
look free.
|
||||
@@ -374,3 +383,87 @@ here.
|
||||
To override any of them, set `SNRC_REGISTRY_<TLD>`, `SNRC_REGISTRAR_<TLD>` or
|
||||
`SNRC_CONTROLLER_<TLD>` on the `resolver` service in `docker-compose.yml`, or
|
||||
as env vars when you run the script directly.
|
||||
|
||||
### Load and scaling
|
||||
|
||||
A lookup reads the chain in at most three rounds: the name's status and
|
||||
registry entries, then its record from its resolver, then prices for a name
|
||||
that is free. The node runs the calls of a JSON-RPC batch one after another,
|
||||
so the contract reads of a round go to the chain as one `eth_call` to
|
||||
[Multicall3](https://github.com/mds1/multicall). A node that is slow for a
|
||||
moment therefore delays a lookup a few times, not once per read. Connections to
|
||||
the node are kept open and reused.
|
||||
|
||||
The resolver answers HTTP/1.1, so each smp-server keeps its connections to it
|
||||
open instead of connecting for every lookup. An idle connection is closed after
|
||||
60 s; the smp-server drops its own idle ones after 30 s, so it closes them first.
|
||||
|
||||
Each access log line ends with how long the request took, so slow requests
|
||||
show up in `docker compose logs resolver`.
|
||||
|
||||
| Variable | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `SNRC_WORKERS` | CPU count, at most 4 | processes sharing the port. If one exits, the others stop, so the container restarts |
|
||||
| `SNRC_RPC_TIMEOUT` | `5` | seconds to wait for each request to the node; the smp-server gives up after 3 |
|
||||
| `SNRC_MULTICALL` | `0xcA11bde05977b3631167028862bE2a173976CA11` | Multicall3 address. If it does not answer, each round is sent as a plain batch and the log says so once |
|
||||
|
||||
### Logs
|
||||
|
||||
Each event is one line: UTC time, level, event name, then `key=value` fields.
|
||||
|
||||
```
|
||||
2026-09-26T08:06:19.992Z INFO request client=203.0.113.7 worker=12 method=GET path=/v2/resolve/[4fdd…].testing status=200 bytes=701 ms=35
|
||||
2026-09-26T08:06:20.051Z WARN upstream_error name=foobar.testing error=ConnectionRefusedError message="[Errno 111] Connection refused"
|
||||
```
|
||||
|
||||
| Event | Level | Meaning |
|
||||
|---|---|---|
|
||||
| `listening` | info | started: address, workers, RPC endpoint, registries, trusted proxies |
|
||||
| `request` | info | one answered request; `/health` only at debug |
|
||||
| `upstream_error` | warn | a read from the node failed; the caller got 502 |
|
||||
| `multicall_unavailable` | warn | Multicall3 did not answer; rounds are sent as plain batches |
|
||||
| `client_gone` | warn | the caller hung up before the answer, usually an smp-server past its timeout |
|
||||
| `http_error` | warn | a malformed request; idle keep-alive timeouts only at debug |
|
||||
| `request_failed` | error | a request failed unexpectedly, with its traceback |
|
||||
| `worker_exited`, `worker_failed` | error | a worker stopped; the others stop too, so the container restarts |
|
||||
| `stopping` | info | stopped by a signal |
|
||||
|
||||
| Variable | Default | Meaning |
|
||||
|---|---|---|
|
||||
| `SNRC_LOG_FORMAT` | `text` | `text` (`key=value`) or `json`, one object per line |
|
||||
| `SNRC_LOG_COLOR` | `auto` | `auto` colours only a terminal, `always`, `never`. Docker output is no terminal, so set `always` for coloured `docker compose logs` |
|
||||
| `SNRC_LOG_LEVEL` | `info` | `debug` also logs health checks and idle connections closing |
|
||||
| `SNRC_TRUSTED_PROXIES` | none | addresses or CIDRs of reverse proxies whose `X-Forwarded-For` names the client |
|
||||
|
||||
With docker compose, set these, and those in "Load and scaling", in `.env`
|
||||
(`.env.example` lists them).
|
||||
|
||||
Behind a reverse proxy such as Caddy on the host, the resolver sees the Docker
|
||||
network's gateway rather than the client. Trust that gateway, and the logged
|
||||
client is the last `X-Forwarded-For` address no trusted proxy added, so a caller
|
||||
through the proxy cannot claim another address. Everything else on the host that
|
||||
connects to `127.0.0.1:8000`, the smp-servers included, arrives through the same
|
||||
gateway and could set the logged address too; run the proxy in the compose network
|
||||
and trust only its address to avoid that.
|
||||
|
||||
The gateway is
|
||||
`docker network inspect resolver_default --format '{{(index .IPAM.Config 0).Gateway}}'`,
|
||||
with the network named after the compose project. It changes when the network is
|
||||
recreated, as `docker compose down` does, so trust one of, in `.env`:
|
||||
|
||||
- `SNRC_TRUSTED_PROXIES=172.16.0.0/12`: Docker's default pools for compose
|
||||
networks, whatever subnet it picks. This also trusts every other container on the
|
||||
host. Add the gateway too if Docker ever gives the network a `192.168.x.x` one.
|
||||
- a pinned subnet, so the gateway stays fixed, and only that gateway:
|
||||
|
||||
```yaml
|
||||
networks:
|
||||
default:
|
||||
ipam:
|
||||
config:
|
||||
- subnet: 172.30.0.0/24
|
||||
gateway: 172.30.0.1
|
||||
```
|
||||
|
||||
in `docker-compose.yml`, with `SNRC_TRUSTED_PROXIES=172.30.0.1`, on a subnet no
|
||||
other network on the host uses.
|
||||
|
||||
@@ -38,14 +38,14 @@ services:
|
||||
- sh
|
||||
- -c
|
||||
- |
|
||||
if [ -d /home/user/nimbus-eth2/build/data/${NETWORK}/db ]; then
|
||||
if [ -d /home/user/nimbus-eth2/build/data/${NETWORK:-mainnet}/db ]; then
|
||||
echo "Nimbus data dir already initialised — skipping checkpoint sync";
|
||||
exit 0;
|
||||
fi;
|
||||
/home/user/nimbus-eth2/build/nimbus_beacon_node trustedNodeSync \
|
||||
--network=${NETWORK} \
|
||||
--data-dir=/home/user/nimbus-eth2/build/data/${NETWORK} \
|
||||
--trusted-node-url=${TRUSTED_NODE_URL} \
|
||||
--network=${NETWORK:-mainnet} \
|
||||
--data-dir=/home/user/nimbus-eth2/build/data/${NETWORK:-mainnet} \
|
||||
--trusted-node-url=${TRUSTED_NODE_URL:-https://mainnet-checkpoint-sync.attestant.io} \
|
||||
--backfill=false
|
||||
restart: "no"
|
||||
|
||||
@@ -69,8 +69,8 @@ services:
|
||||
echo "Reth data already initialised — skipping snapshot download";
|
||||
exit 0;
|
||||
fi;
|
||||
echo "Downloading Reth ${NETWORK} --minimal snapshot...";
|
||||
reth download --datadir /data --chain ${NETWORK} --minimal && \
|
||||
echo "Downloading Reth ${NETWORK:-mainnet} --minimal snapshot...";
|
||||
reth download --datadir /data --chain ${NETWORK:-mainnet} --minimal && \
|
||||
touch /data/.snapshot-done && \
|
||||
echo "Snapshot download complete"
|
||||
restart: "no"
|
||||
@@ -92,7 +92,7 @@ services:
|
||||
command: >
|
||||
node
|
||||
--datadir /data
|
||||
--chain ${NETWORK}
|
||||
--chain ${NETWORK:-mainnet}
|
||||
--minimal
|
||||
--authrpc.jwtsecret /jwt/jwt.hex
|
||||
--authrpc.addr 0.0.0.0 --authrpc.port 8551
|
||||
@@ -117,8 +117,8 @@ services:
|
||||
- "9000:9000/udp"
|
||||
- "127.0.0.1:5052:5052"
|
||||
command: >
|
||||
--network=${NETWORK}
|
||||
--data-dir=/home/user/nimbus-eth2/build/data/${NETWORK}
|
||||
--network=${NETWORK:-mainnet}
|
||||
--data-dir=/home/user/nimbus-eth2/build/data/${NETWORK:-mainnet}
|
||||
--el=http://reth:8551
|
||||
--jwt-secret=/jwt/jwt.hex
|
||||
--non-interactive
|
||||
@@ -131,6 +131,7 @@ services:
|
||||
# smp-server points its [NAMES] resolver_endpoint at this URL.
|
||||
# To change the host port, edit the LEFT side of the port mapping below.
|
||||
resolver:
|
||||
image: snrc-resolve:local
|
||||
build:
|
||||
context: ./service
|
||||
dockerfile: Dockerfile
|
||||
@@ -156,8 +157,18 @@ services:
|
||||
# SNRC_REGISTRAR_SIMPLEX: 0x...
|
||||
# SNRC_CONTROLLER_TESTING: 0x...
|
||||
# SNRC_CONTROLLER_SIMPLEX: 0x...
|
||||
# Set in .env (see .env.example); unset or empty means the resolver's default.
|
||||
SNRC_WORKERS: ${SNRC_WORKERS:-}
|
||||
SNRC_RPC_TIMEOUT: ${SNRC_RPC_TIMEOUT:-}
|
||||
SNRC_LOG_FORMAT: ${SNRC_LOG_FORMAT:-}
|
||||
SNRC_LOG_COLOR: ${SNRC_LOG_COLOR:-}
|
||||
SNRC_LOG_LEVEL: ${SNRC_LOG_LEVEL:-}
|
||||
SNRC_TRUSTED_PROXIES: ${SNRC_TRUSTED_PROXIES:-}
|
||||
ports:
|
||||
- "127.0.0.1:8000:8000"
|
||||
# a line per lookup; the local driver keeps 5 rotated files of 20 MB
|
||||
logging:
|
||||
driver: local
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
.venv
|
||||
**/__pycache__
|
||||
.pytest_cache
|
||||
.ruff_cache
|
||||
test_snrc_resolve.py
|
||||
@@ -0,0 +1 @@
|
||||
3.13
|
||||
@@ -1,9 +1,10 @@
|
||||
# syntax=docker/dockerfile:1.7
|
||||
# ---------- builder ----------
|
||||
# Use the official uv image (Astral) on top of a slim Python base.
|
||||
# uv resolves and installs the lockfile-free pyproject.toml in seconds and
|
||||
# produces a portable .venv we can copy into the runtime stage.
|
||||
FROM ghcr.io/astral-sh/uv:python3.13-bookworm-slim AS builder
|
||||
# The official uv image (Astral) on top of a slim Python base installs the
|
||||
# locked dependencies into a portable .venv for the runtime stage.
|
||||
# Pinned to the uv that wrote uv.lock (its bookworm images stopped at 0.9.30), on the
|
||||
# runtime's Debian release, so the .venv's Python path exists there.
|
||||
FROM ghcr.io/astral-sh/uv:0.12.16-python3.13-trixie-slim AS builder
|
||||
|
||||
ENV UV_LINK_MODE=copy \
|
||||
UV_COMPILE_BYTECODE=1 \
|
||||
@@ -12,17 +13,17 @@ ENV UV_LINK_MODE=copy \
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install deps first (separate layer) — script edits won't bust this cache.
|
||||
COPY pyproject.toml ./
|
||||
# Dependencies first (separate layer), exactly as locked: script edits don't bust this cache.
|
||||
COPY pyproject.toml uv.lock .python-version ./
|
||||
RUN --mount=type=cache,target=/root/.cache/uv \
|
||||
uv sync --no-dev --no-install-project
|
||||
uv sync --frozen --no-dev --no-install-project
|
||||
|
||||
# Script is added after the dep layer for cache friendliness.
|
||||
COPY snrc-resolve.py ./
|
||||
|
||||
# ---------- runtime ----------
|
||||
# Slim runtime — only the venv + script. No uv, no apt.
|
||||
FROM python:3.13-slim AS runtime
|
||||
FROM python:3.13-slim-trixie AS runtime
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
PYTHONDONTWRITEBYTECODE=1 \
|
||||
@@ -33,15 +34,15 @@ RUN groupadd --system --gid 10001 snrc && \
|
||||
useradd --system --uid 10001 --gid snrc --no-create-home --shell /usr/sbin/nologin snrc
|
||||
|
||||
WORKDIR /app
|
||||
COPY --from=builder --chown=snrc:snrc /app /app
|
||||
# owned by root, so the service cannot rewrite its own code
|
||||
COPY --from=builder /app /app
|
||||
|
||||
USER snrc:snrc
|
||||
|
||||
EXPOSE 8000
|
||||
|
||||
# Liveness check hits the script's own /health route. ThreadingHTTPServer is
|
||||
# fast enough that 3s is generous for a localhost probe; restart if it stops
|
||||
# responding entirely.
|
||||
# Liveness check hits the script's own /health route. Three failures mark the
|
||||
# container unhealthy; compose does not restart it for that, but `docker ps` shows it.
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||
CMD ["python", "-c", "import urllib.request, sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=3).status == 200 else 1)"]
|
||||
|
||||
|
||||
@@ -9,5 +9,14 @@ dependencies = [
|
||||
"eth-hash[pycryptodome]>=0.7",
|
||||
]
|
||||
|
||||
[dependency-groups]
|
||||
dev = [
|
||||
"pytest>=8",
|
||||
"ruff>=0.6",
|
||||
]
|
||||
|
||||
[tool.uv]
|
||||
package = false
|
||||
|
||||
[tool.ruff.lint]
|
||||
select = ["E9", "F", "B"]
|
||||
|
||||
@@ -46,6 +46,15 @@ Environment:
|
||||
(default: mainnet for .testing, empty for .simplex)
|
||||
SNRC_PORT Listen port (default: 8000)
|
||||
SNRC_BIND Bind address (default: 0.0.0.0)
|
||||
SNRC_WORKERS Worker processes sharing the port (default: CPU count, at most 4)
|
||||
SNRC_RPC_TIMEOUT Seconds to wait for each RPC request (default: 5)
|
||||
SNRC_MULTICALL Multicall3 contract that runs a round of reads as one call
|
||||
(default: 0xcA11bde05977b3631167028862bE2a173976CA11)
|
||||
SNRC_LOG_FORMAT text (key=value) or json (default: text)
|
||||
SNRC_LOG_COLOR auto (on a terminal), always or never (default: auto)
|
||||
SNRC_LOG_LEVEL debug, info, warning or error (default: info)
|
||||
SNRC_TRUSTED_PROXIES Comma-separated addresses or CIDRs of reverse proxies whose
|
||||
X-Forwarded-For names the client (default: none)
|
||||
|
||||
Each TLD is a separate SNRC deployment with its own ENSRegistry; the
|
||||
resolver dispatches by the queried name's rightmost label.
|
||||
@@ -64,19 +73,45 @@ Unrecognised payloads fall back to `0x`-prefixed raw hex.
|
||||
"""
|
||||
|
||||
import hashlib
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import signal
|
||||
import socket
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from functools import lru_cache
|
||||
from http.client import BadStatusLine, HTTPConnection, HTTPSConnection
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.error import HTTPError
|
||||
from urllib.parse import unquote, urlparse
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
from eth_hash.auto import keccak
|
||||
|
||||
RPC = os.environ.get("SNRC_RPC", "http://127.0.0.1:8545")
|
||||
BIND = os.environ.get("SNRC_BIND", "0.0.0.0")
|
||||
PORT = int(os.environ.get("SNRC_PORT", "8000"))
|
||||
# The smp-server gives up after 3 s, so a slower call only holds a thread.
|
||||
RPC_TIMEOUT_S = float(os.environ.get("SNRC_RPC_TIMEOUT", "") or 5)
|
||||
# The node and the beacon client usually share the host, so not every core.
|
||||
MAX_DEFAULT_WORKERS = 4
|
||||
WORKERS = int(os.environ.get("SNRC_WORKERS", "") or min(MAX_DEFAULT_WORKERS, os.cpu_count() or 1))
|
||||
# Multicall3, at this address on mainnet and most chains. A node runs a JSON-RPC
|
||||
# batch one call after another, so a round of reads is sent as one eth_call.
|
||||
MULTICALL = os.environ.get("SNRC_MULTICALL", "") or "0xcA11bde05977b3631167028862bE2a173976CA11"
|
||||
LOG_FORMAT = os.environ.get("SNRC_LOG_FORMAT", "") or "text"
|
||||
LOG_COLOR = os.environ.get("SNRC_LOG_COLOR", "") or "auto"
|
||||
LOG_LEVEL = os.environ.get("SNRC_LOG_LEVEL", "") or "info"
|
||||
# Peers whose X-Forwarded-For is believed, such as the Docker gateway a reverse proxy on the host
|
||||
# connects through. Anyone else could put any address there.
|
||||
TRUSTED_PROXIES = tuple(
|
||||
ipaddress.ip_network(p.strip(), strict=False) for p in os.environ.get("SNRC_TRUSTED_PROXIES", "").split(",") if p.strip()
|
||||
)
|
||||
|
||||
# Each TLD is its own SNRC deployment with its own ENSRegistry. Dispatch
|
||||
# happens on the rightmost label of the queried name. Empty / unset means
|
||||
@@ -121,34 +156,291 @@ COIN_ETH = 60
|
||||
COIN_BTC = 0
|
||||
COIN_XMR = 128
|
||||
COIN_DOT = 354
|
||||
RECORD_COINS = (COIN_ETH, COIN_BTC, COIN_XMR, COIN_DOT)
|
||||
|
||||
ZERO_ADDR = "0x0000000000000000000000000000000000000000"
|
||||
|
||||
# The registry prices in attoUSD (1e-18 USD); the protocol carries US cents.
|
||||
|
||||
|
||||
# ---------- Logging ----------
|
||||
|
||||
LOGGER = logging.getLogger("snrc_resolve")
|
||||
LEVEL_NAMES = {logging.DEBUG: "DEBUG", logging.INFO: "INFO", logging.WARNING: "WARN", logging.ERROR: "ERROR"}
|
||||
# ANSI SGR codes
|
||||
DIM, BOLD, GREEN, YELLOW, RED = "2", "1", "32", "33", "31"
|
||||
LEVEL_COLORS = {"DEBUG": DIM, "INFO": GREEN, "WARN": YELLOW, "ERROR": RED}
|
||||
STATUS_COLORS = {2: GREEN, 3: GREEN, 4: YELLOW, 5: RED}
|
||||
|
||||
|
||||
def log_event(level: int, name: str, /, exc_info=None, **fields):
|
||||
LOGGER.log(level, name, exc_info=exc_info, extra={"fields": fields})
|
||||
|
||||
|
||||
def _log_time(record: logging.LogRecord) -> str:
|
||||
return datetime.fromtimestamp(record.created, timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z")
|
||||
|
||||
|
||||
def _log_value(value) -> str:
|
||||
"""A logfmt value: bare when it cannot be misread, JSON-quoted otherwise."""
|
||||
if value is None:
|
||||
return "-"
|
||||
s = str(value)
|
||||
if not s or any(c in s for c in ' ="\\') or not s.isprintable():
|
||||
return json.dumps(s)
|
||||
return s
|
||||
|
||||
|
||||
class TextFormatter(logging.Formatter):
|
||||
def __init__(self, color: bool):
|
||||
super().__init__()
|
||||
self.color = color
|
||||
|
||||
def _paint(self, code: str, text: str) -> str:
|
||||
return f"\033[{code}m{text}\033[0m" if self.color and code else text
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
level = LEVEL_NAMES.get(record.levelno, record.levelname)
|
||||
fields = " ".join(
|
||||
self._paint(DIM, f"{k}=") + self._paint(STATUS_COLORS.get(v // 100, "") if k == "status" and isinstance(v, int) else "", _log_value(v))
|
||||
for k, v in getattr(record, "fields", {}).items()
|
||||
)
|
||||
line = f"{self._paint(DIM, _log_time(record))} {self._paint(LEVEL_COLORS.get(level, ''), f'{level:<5}')} {self._paint(BOLD, record.getMessage())}"
|
||||
if fields:
|
||||
line += " " + fields
|
||||
if record.exc_info:
|
||||
line += "\n" + self.formatException(record.exc_info)
|
||||
return line
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
out = {
|
||||
"time": _log_time(record),
|
||||
"level": LEVEL_NAMES.get(record.levelno, record.levelname).lower(),
|
||||
"event": record.getMessage(),
|
||||
**getattr(record, "fields", {}),
|
||||
}
|
||||
if record.exc_info:
|
||||
out["exception"] = self.formatException(record.exc_info)
|
||||
return json.dumps(out, default=str)
|
||||
|
||||
|
||||
def setup_logging(stream=None):
|
||||
"""Sends the resolver's events to stderr in the configured format."""
|
||||
stream = stream or sys.stderr
|
||||
if LOG_FORMAT not in ("text", "json"):
|
||||
raise ValueError(f"SNRC_LOG_FORMAT must be text or json, not {LOG_FORMAT!r}")
|
||||
if LOG_COLOR not in ("auto", "always", "never"):
|
||||
raise ValueError(f"SNRC_LOG_COLOR must be auto, always or never, not {LOG_COLOR!r}")
|
||||
level = logging.getLevelName(LOG_LEVEL.upper())
|
||||
if not isinstance(level, int):
|
||||
raise ValueError(f"SNRC_LOG_LEVEL must be debug, info, warning or error, not {LOG_LEVEL!r}")
|
||||
color = LOG_COLOR == "always" or (LOG_COLOR == "auto" and stream.isatty())
|
||||
handler = logging.StreamHandler(stream)
|
||||
handler.setFormatter(JsonFormatter() if LOG_FORMAT == "json" else TextFormatter(color))
|
||||
LOGGER.handlers[:] = [handler]
|
||||
LOGGER.setLevel(level)
|
||||
LOGGER.propagate = False
|
||||
|
||||
|
||||
# ---------- RPC + ABI helpers (mirrors ens-lookup.py shape) ----------
|
||||
|
||||
RPC_URL = urlparse(RPC)
|
||||
# Set a non-default User-Agent; Cloudflare-fronted public RPCs (drpc,
|
||||
# publicnode, etc.) reject `Python-urllib/3.x` with 403.
|
||||
RPC_HEADERS = {"Content-Type": "application/json", "User-Agent": "snrc-resolve/1.0"}
|
||||
|
||||
# Idle keep-alive connections to SNRC_RPC. A connection per call costs most of
|
||||
# a lookup's CPU and leaves a TIME_WAIT socket per call, which exhausts local
|
||||
# ports at a few dozen lookups per second.
|
||||
# Idle connections kept per worker; more are closed after use. 4 workers stay well
|
||||
# under reth's default limit of 500 connections.
|
||||
RPC_POOL_SIZE = 32
|
||||
_rpc_pool = queue.LifoQueue(maxsize=RPC_POOL_SIZE)
|
||||
|
||||
|
||||
def _new_rpc_connection():
|
||||
conn_class = HTTPSConnection if RPC_URL.scheme == "https" else HTTPConnection
|
||||
return conn_class(RPC_URL.hostname, RPC_URL.port, timeout=RPC_TIMEOUT_S)
|
||||
|
||||
|
||||
def _post_rpc(conn, body: bytes) -> bytes:
|
||||
path = (RPC_URL.path or "/") + (f"?{RPC_URL.query}" if RPC_URL.query else "")
|
||||
conn.request("POST", path, body, RPC_HEADERS)
|
||||
res = conn.getresponse()
|
||||
data = res.read()
|
||||
# HTTPError, as urlopen raised: RuntimeError means the call itself failed
|
||||
if not 200 <= res.status < 300:
|
||||
raise HTTPError(RPC, res.status, res.reason, res.headers, None)
|
||||
return data
|
||||
|
||||
|
||||
def _post_pooled(conn, body: bytes) -> bytes:
|
||||
try:
|
||||
data = _post_rpc(conn, body)
|
||||
except BaseException:
|
||||
conn.close()
|
||||
raise
|
||||
try:
|
||||
_rpc_pool.put_nowait(conn)
|
||||
except queue.Full:
|
||||
conn.close()
|
||||
return data
|
||||
|
||||
|
||||
def _send_rpc(payload) -> object:
|
||||
body = json.dumps(payload).encode()
|
||||
try:
|
||||
idle = _rpc_pool.get_nowait()
|
||||
except queue.Empty:
|
||||
return json.loads(_post_pooled(_new_rpc_connection(), body))
|
||||
try:
|
||||
return json.loads(_post_pooled(idle, body))
|
||||
except (ConnectionError, BadStatusLine):
|
||||
# the node closed the idle connection; every call is a read, so resending is safe
|
||||
return json.loads(_post_pooled(_new_rpc_connection(), body))
|
||||
|
||||
|
||||
# Reads prefetched for the request being answered, keyed by _read_key. Only set
|
||||
# inside request_reads(), so code called outside a request reads one call at a time.
|
||||
_request = threading.local()
|
||||
_UNREAD = object()
|
||||
|
||||
|
||||
def _read_key(method, params) -> str:
|
||||
return method + json.dumps(params, sort_keys=True)
|
||||
|
||||
|
||||
@contextmanager
|
||||
def request_reads():
|
||||
_request.reads = {}
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
del _request.reads
|
||||
|
||||
|
||||
def rpc(method, params):
|
||||
body = json.dumps(
|
||||
{"jsonrpc": "2.0", "method": method, "params": params, "id": 1}
|
||||
).encode()
|
||||
# Set a non-default User-Agent; Cloudflare-fronted public RPCs (drpc,
|
||||
# publicnode, etc.) reject `Python-urllib/3.x` with 403.
|
||||
req = Request(
|
||||
RPC,
|
||||
data=body,
|
||||
headers={
|
||||
"Content-Type": "application/json",
|
||||
"User-Agent": "snrc-resolve/1.0",
|
||||
},
|
||||
)
|
||||
res = json.loads(urlopen(req, timeout=15).read())
|
||||
reads = getattr(_request, "reads", None)
|
||||
if reads is not None:
|
||||
read = reads.get(_read_key(method, params), _UNREAD)
|
||||
if isinstance(read, RuntimeError):
|
||||
raise read
|
||||
if read is not _UNREAD:
|
||||
return read
|
||||
res = _send_rpc({"jsonrpc": "2.0", "method": method, "params": params, "id": 1})
|
||||
if "error" in res:
|
||||
raise RuntimeError(res["error"])
|
||||
return res["result"]
|
||||
|
||||
|
||||
def _send_batch(requests):
|
||||
"""Answers of a JSON-RPC batch in request order, None for a request left
|
||||
unanswered; all None when the node does not batch."""
|
||||
res = _send_rpc([{"jsonrpc": "2.0", "id": i, "method": m, "params": p} for i, (m, p) in enumerate(requests)])
|
||||
# a node that does not batch answers with a single error object
|
||||
by_id = {r.get("id"): r for r in res if isinstance(r, dict)} if isinstance(res, list) else {}
|
||||
return [by_id.get(i) for i in range(len(requests))]
|
||||
|
||||
|
||||
def _is_latest_call(method, params) -> bool:
|
||||
return method == "eth_call" and params[1] == "latest"
|
||||
|
||||
|
||||
_multicall_failed_logged = False
|
||||
|
||||
|
||||
def _remember(reads, requests, answers):
|
||||
for (m, p), r in zip(requests, answers, strict=True):
|
||||
if r is not None and ("result" in r or "error" in r):
|
||||
reads[_read_key(m, p)] = RuntimeError(r["error"]) if "error" in r else r["result"]
|
||||
|
||||
|
||||
def prefetch(requests):
|
||||
"""Sends reads the request will make in one round trip, with its contract
|
||||
reads as one multicall. A read left unanswered is made on its own when the
|
||||
code gets to it."""
|
||||
global _multicall_failed_logged
|
||||
reads = getattr(_request, "reads", None)
|
||||
if reads is None:
|
||||
return
|
||||
todo = [(m, p) for m, p in requests if _read_key(m, p) not in reads]
|
||||
if len(todo) < 2:
|
||||
return
|
||||
calls = [(m, p) for m, p in todo if _is_latest_call(m, p)]
|
||||
if len(calls) < 2:
|
||||
_remember(reads, todo, _send_batch(todo))
|
||||
return
|
||||
others = [(m, p) for m, p in todo if not _is_latest_call(m, p)]
|
||||
multicall = eth_call_read(MULTICALL, encode_aggregate3([(p[0]["to"], p[0]["data"]) for _, p in calls]))
|
||||
answers = _send_batch(others + [multicall])
|
||||
if all(r is None for r in answers):
|
||||
return
|
||||
_remember(reads, others, answers[:-1])
|
||||
try:
|
||||
result = (answers[-1] or {}).get("result")
|
||||
if not isinstance(result, str):
|
||||
raise ValueError(f"multicall answered {answers[-1]!r}")
|
||||
results = decode_aggregate3(result)
|
||||
if len(results) != len(calls):
|
||||
raise ValueError("multicall answered a different number of calls")
|
||||
except (KeyError, TypeError, ValueError) as e:
|
||||
if not _multicall_failed_logged:
|
||||
_multicall_failed_logged = True
|
||||
log_event(logging.WARNING, "multicall_unavailable", multicall=MULTICALL, error=repr(e), fallback="batch")
|
||||
_remember(reads, calls, _send_batch(calls))
|
||||
return
|
||||
for (m, p), (success, data) in zip(calls, results, strict=True):
|
||||
reads[_read_key(m, p)] = "0x" + data.hex() if success else RuntimeError("execution reverted")
|
||||
|
||||
|
||||
AGGREGATE3 = "0x82ad56cb" # aggregate3((address,bool,bytes)[])
|
||||
|
||||
|
||||
def encode_aggregate3(calls) -> str:
|
||||
"""Calldata for Multicall3.aggregate3 with every call allowed to fail."""
|
||||
tuples = []
|
||||
for to, data in calls:
|
||||
b = bytes.fromhex(data[2:])
|
||||
tuples.append(
|
||||
int(to, 16).to_bytes(32, "big")
|
||||
+ (1).to_bytes(32, "big")
|
||||
+ (0x60).to_bytes(32, "big")
|
||||
+ len(b).to_bytes(32, "big")
|
||||
+ b
|
||||
+ b"\x00" * ((-len(b)) % 32)
|
||||
)
|
||||
offsets, at = [], 32 * len(tuples)
|
||||
for t in tuples:
|
||||
offsets.append(at.to_bytes(32, "big"))
|
||||
at += len(t)
|
||||
body = (0x20).to_bytes(32, "big") + len(tuples).to_bytes(32, "big") + b"".join(offsets) + b"".join(tuples)
|
||||
return AGGREGATE3 + body.hex()
|
||||
|
||||
|
||||
def decode_aggregate3(hex_data: str):
|
||||
"""Multicall3.aggregate3's (bool success, bytes returnData)[]."""
|
||||
raw = bytes.fromhex(hex_data[2:] if hex_data.startswith("0x") else hex_data)
|
||||
|
||||
def word(at: int) -> int:
|
||||
if at + 32 > len(raw):
|
||||
raise ValueError("multicall answer is truncated")
|
||||
return int.from_bytes(raw[at:at + 32], "big")
|
||||
|
||||
array = word(0)
|
||||
base = array + 32
|
||||
out = []
|
||||
for i in range(word(array)):
|
||||
item = base + word(base + 32 * i)
|
||||
data = item + word(item + 32)
|
||||
length = word(data)
|
||||
if data + 32 + length > len(raw):
|
||||
raise ValueError("multicall answer is truncated")
|
||||
out.append((word(item) != 0, raw[data + 32:data + 32 + length]))
|
||||
return out
|
||||
|
||||
|
||||
def namehash(name: str) -> bytes:
|
||||
node = b"\x00" * 32
|
||||
if name:
|
||||
@@ -184,11 +476,14 @@ def node_of(name: str) -> bytes:
|
||||
# ---------- Registration status ----------
|
||||
|
||||
|
||||
BLOCK_READ = ("eth_getBlockByNumber", ["latest", False])
|
||||
|
||||
|
||||
def head_block():
|
||||
"""How far behind the node is. Unlike expiry, this is the one thing that has
|
||||
to be measured against the host clock: a node that stops still has a block."""
|
||||
try:
|
||||
block = rpc("eth_getBlockByNumber", ["latest", False])
|
||||
block = rpc(*BLOCK_READ)
|
||||
return {
|
||||
"blockNumber": decode_uint(block["number"]),
|
||||
"chainLagSeconds": int(time.time()) - decode_uint(block["timestamp"]),
|
||||
@@ -199,13 +494,13 @@ def head_block():
|
||||
|
||||
def chain_now() -> int:
|
||||
"""Expiry is compared against the block timestamp, never the host clock."""
|
||||
block = rpc("eth_getBlockByNumber", ["latest", False])
|
||||
block = rpc(*BLOCK_READ)
|
||||
return decode_uint(block["timestamp"])
|
||||
|
||||
|
||||
def grace_period(registrar: str) -> int:
|
||||
"""A deployment can configure a different window, so it is read on chain."""
|
||||
return decode_uint(eth_call(registrar, selector("GRACE_PERIOD()")))
|
||||
return decode_uint(eth_call(*grace_call(registrar)))
|
||||
|
||||
|
||||
def expiry_status(expires: int, grace: int, now: int) -> str:
|
||||
@@ -225,8 +520,7 @@ def reservation_reason(tld: str, token: int) -> int:
|
||||
controller = CONTROLLERS.get(tld)
|
||||
if not controller:
|
||||
return 0
|
||||
raw = eth_call(controller, selector("reservedNames(bytes32)") + encode_uint(token))
|
||||
return decode_uint(raw)
|
||||
return decode_uint(eth_call(*reserved_call(controller, token)))
|
||||
|
||||
|
||||
def pricing_params(tld: str):
|
||||
@@ -235,7 +529,8 @@ def pricing_params(tld: str):
|
||||
controller = CONTROLLERS.get(tld)
|
||||
if not controller:
|
||||
return None
|
||||
oracle = decode_address(eth_call(controller, selector("prices()")))
|
||||
prefetch([eth_call_read(*prices_call(controller)), eth_call_read(*min_length_call(controller))])
|
||||
oracle = decode_address(eth_call(*prices_call(controller)))
|
||||
if oracle == ZERO_ADDR:
|
||||
return None
|
||||
try:
|
||||
@@ -247,6 +542,8 @@ def pricing_params(tld: str):
|
||||
|
||||
|
||||
SECONDS_PER_YEAR = 31536000
|
||||
# an ENS-shaped oracle prices names by length up to six letters
|
||||
LETTER_TIERS = range(1, 7)
|
||||
ATTO_PER_CENT = 10**16
|
||||
|
||||
|
||||
@@ -255,13 +552,15 @@ def read_oracle_prices(controller: str, oracle: str):
|
||||
protocol carries. An ENS-shaped oracle prices in attoUSD per second and
|
||||
charges a premium on lapsed names that it does not expose, so a quote from
|
||||
it is only safe for a name that was never registered."""
|
||||
# either oracle shape is answered in the same round trip
|
||||
prefetch([eth_call_read(*prices_call(oracle))] + [eth_call_read(*letter_price_call(oracle, n)) for n in LETTER_TIERS])
|
||||
try:
|
||||
base, tiers = decode_prices(eth_call(oracle, selector("prices()")))
|
||||
base, tiers = decode_prices(eth_call(*prices_call(oracle)))
|
||||
premium_unknown = False
|
||||
except RuntimeError:
|
||||
base, tiers = decode_letter_prices(oracle)
|
||||
premium_unknown = True
|
||||
min_len = decode_uint(eth_call(controller, selector("minCharLength()")))
|
||||
min_len = decode_uint(eth_call(*min_length_call(controller)))
|
||||
return {
|
||||
# lengths the registry refuses are left out rather than priced at zero
|
||||
"registrationPrices": {n: c for n, c in tiers.items() if n >= min_len},
|
||||
@@ -277,9 +576,9 @@ def decode_letter_prices(oracle: str):
|
||||
the six-letter tier stops at five, and charges its highest tier for anything
|
||||
longer, which is what basePrice means here."""
|
||||
tiers = {}
|
||||
for n in range(1, 7):
|
||||
for n in LETTER_TIERS:
|
||||
try:
|
||||
rate = decode_uint(eth_call(oracle, selector(f"price{n}Letter()")))
|
||||
rate = decode_uint(eth_call(*letter_price_call(oracle, n)))
|
||||
except RuntimeError:
|
||||
if n <= 5:
|
||||
raise
|
||||
@@ -301,6 +600,9 @@ def decode_prices(hex_data: str):
|
||||
base = int.from_bytes(raw[:32], "big")
|
||||
at = int.from_bytes(raw[32:64], "big")
|
||||
count = int.from_bytes(raw[at:at + 32], "big")
|
||||
# the count comes from the answer, so it is checked against the answer's length
|
||||
if at + 32 + count * 64 > len(raw):
|
||||
raise RuntimeError("prices(): short response")
|
||||
tiers = {}
|
||||
for i in range(count):
|
||||
item = at + 32 + i * 64
|
||||
@@ -328,9 +630,7 @@ def name_status(name: str):
|
||||
# The 2LD's label is that key at any depth. node_of decodes a bracket only
|
||||
# in a two-label name, so a bracket subname gets a status but no record.
|
||||
token = label_token(labels[-2])
|
||||
expires = decode_uint(
|
||||
eth_call(registrar, selector("nameExpires(uint256)") + encode_uint(token))
|
||||
)
|
||||
expires = decode_uint(eth_call(*expires_call(registrar, token)))
|
||||
grace = grace_period(registrar) if expires else 0
|
||||
now = chain_now()
|
||||
status = expiry_status(expires, grace, now)
|
||||
@@ -358,12 +658,17 @@ def name_status(name: str):
|
||||
return out
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def selector(signature: str) -> str:
|
||||
return "0x" + keccak(signature.encode())[:4].hex()
|
||||
|
||||
|
||||
def eth_call_read(to: str, data: str):
|
||||
return "eth_call", [{"to": to, "data": data}, "latest"]
|
||||
|
||||
|
||||
def eth_call(to: str, data: str) -> str:
|
||||
result = rpc("eth_call", [{"to": to, "data": data}, "latest"])
|
||||
result = rpc(*eth_call_read(to, data))
|
||||
if result == "0x":
|
||||
raise RuntimeError(f"empty return from {to}: no contract at that address?")
|
||||
return result
|
||||
@@ -385,7 +690,7 @@ def registered_label(registrar: str, token: int):
|
||||
"""The plaintext label the registrar recorded at registration, keyed by the
|
||||
hash of that label. None when the name was registered without
|
||||
registerWithLabel, so the registrar cannot name it."""
|
||||
raw = decode_bytes(eth_call(registrar, selector("labelOf(uint256)") + encode_uint(token)))
|
||||
raw = decode_bytes(eth_call(*label_call(registrar, token)))
|
||||
return raw.decode("utf-8", errors="replace") if raw else None
|
||||
|
||||
|
||||
@@ -427,7 +732,7 @@ def encode_text_call(node: bytes, key: str) -> str:
|
||||
|
||||
|
||||
def text(resolver: str, node: bytes, key: str) -> str:
|
||||
raw = decode_bytes(eth_call(resolver, encode_text_call(node, key)))
|
||||
raw = decode_bytes(eth_call(*text_call(resolver, node, key)))
|
||||
return raw.decode("utf-8", errors="replace") if raw else ""
|
||||
|
||||
|
||||
@@ -446,7 +751,7 @@ def addr_multicoin(resolver: str, node: bytes, coin_type: int):
|
||||
payload doesn't match any recognised on-chain shape. Returns None when
|
||||
the record is unset."""
|
||||
try:
|
||||
raw = decode_bytes(eth_call(resolver, encode_addr_multicoin_call(node, coin_type)))
|
||||
raw = decode_bytes(eth_call(*addr_call(resolver, node, coin_type)))
|
||||
except RuntimeError:
|
||||
return None
|
||||
if not raw:
|
||||
@@ -465,6 +770,81 @@ def addr_multicoin(resolver: str, node: bytes, coin_type: int):
|
||||
return "0x" + raw.hex()
|
||||
|
||||
|
||||
# ---------- Contract calls, as (to, data) ----------
|
||||
# Shared by the reads and prefetch, so a prefetched read is found by the code that makes it.
|
||||
|
||||
|
||||
def expires_call(registrar: str, token: int):
|
||||
return registrar, selector("nameExpires(uint256)") + encode_uint(token)
|
||||
|
||||
|
||||
def grace_call(registrar: str):
|
||||
return registrar, selector("GRACE_PERIOD()")
|
||||
|
||||
|
||||
def label_call(registrar: str, token: int):
|
||||
return registrar, selector("labelOf(uint256)") + encode_uint(token)
|
||||
|
||||
|
||||
def reserved_call(controller: str, token: int):
|
||||
return controller, selector("reservedNames(bytes32)") + encode_uint(token)
|
||||
|
||||
|
||||
def prices_call(contract: str):
|
||||
return contract, selector("prices()")
|
||||
|
||||
|
||||
def letter_price_call(oracle: str, letters: int):
|
||||
return oracle, selector(f"price{letters}Letter()")
|
||||
|
||||
|
||||
def min_length_call(controller: str):
|
||||
return controller, selector("minCharLength()")
|
||||
|
||||
|
||||
def resolver_call(registry: str, node: bytes):
|
||||
return registry, selector("resolver(bytes32)") + node.hex()
|
||||
|
||||
|
||||
def owner_call(registry: str, node: bytes):
|
||||
return registry, selector("owner(bytes32)") + node.hex()
|
||||
|
||||
|
||||
def text_call(resolver: str, node: bytes, key: str):
|
||||
return resolver, encode_text_call(node, key)
|
||||
|
||||
|
||||
def addr_call(resolver: str, node: bytes, coin_type: int):
|
||||
return resolver, encode_addr_multicoin_call(node, coin_type)
|
||||
|
||||
|
||||
def lookup_reads(name: str):
|
||||
"""The reads a lookup makes before it knows the name's resolver."""
|
||||
labels = name.split(".")
|
||||
tld = labels[-1]
|
||||
reads = []
|
||||
registrar = REGISTRARS.get(tld)
|
||||
if registrar and len(labels) >= 2:
|
||||
token = label_token(labels[-2])
|
||||
reads += [BLOCK_READ, eth_call_read(*expires_call(registrar, token)), eth_call_read(*grace_call(registrar))]
|
||||
if len(labels) == 2 and is_encoded_labelhash(labels[0]):
|
||||
reads.append(eth_call_read(*label_call(registrar, token)))
|
||||
if CONTROLLERS.get(tld):
|
||||
reads.append(eth_call_read(*reserved_call(CONTROLLERS[tld], token)))
|
||||
registry = REGISTRIES.get(tld)
|
||||
if registry:
|
||||
node = node_of(name)
|
||||
reads += [eth_call_read(*resolver_call(registry, node)), eth_call_read(*owner_call(registry, node))]
|
||||
return reads
|
||||
|
||||
|
||||
def record_reads(resolver: str, node: bytes):
|
||||
"""The reads of a name's record from its resolver."""
|
||||
return [eth_call_read(*text_call(resolver, node, k)) for k in TEXT_KEYS] + [
|
||||
eth_call_read(*addr_call(resolver, node, coin)) for coin in RECORD_COINS
|
||||
]
|
||||
|
||||
|
||||
# ---------- Coin-specific address encoders ----------
|
||||
# Each takes raw bytes as stored under ENSIP-9 and returns the canonical
|
||||
# user-facing string for that chain (EIP-55 for ETH, bech32/base58check
|
||||
@@ -665,9 +1045,9 @@ def split_links(value: str) -> list:
|
||||
|
||||
|
||||
def upstream_error(subject: dict, e: Exception) -> dict:
|
||||
"""urlopen puts the failing URL into its message and SNRC_RPC can carry a
|
||||
provider key, so the text goes to the log and only the type to the caller."""
|
||||
print(f"upstream error: {type(e).__name__}: {e}", file=sys.stderr)
|
||||
"""The exception can carry the failing URL and SNRC_RPC can carry a provider
|
||||
key, so the text goes to the log and only the type to the caller."""
|
||||
log_event(logging.WARNING, "upstream_error", **subject, error=type(e).__name__, message=str(e))
|
||||
return {
|
||||
**subject,
|
||||
"error": "upstreamError",
|
||||
@@ -680,9 +1060,8 @@ def name_record(name: str):
|
||||
has one, with every field unset."""
|
||||
registry = REGISTRIES[name.rsplit(".", 1)[-1]]
|
||||
node = node_of(name)
|
||||
node_hex = node.hex()
|
||||
resolver_addr = decode_address(eth_call(registry, selector("resolver(bytes32)") + node_hex))
|
||||
owner = decode_address(eth_call(registry, selector("owner(bytes32)") + node_hex))
|
||||
resolver_addr = decode_address(eth_call(*resolver_call(registry, node)))
|
||||
owner = decode_address(eth_call(*owner_call(registry, node)))
|
||||
rec = {
|
||||
"name": canonical_name(name),
|
||||
"nickname": "",
|
||||
@@ -699,6 +1078,7 @@ def name_record(name: str):
|
||||
}
|
||||
if resolver_addr == ZERO_ADDR:
|
||||
return rec
|
||||
prefetch(record_reads(resolver_addr, node))
|
||||
texts = {}
|
||||
for k in TEXT_KEYS:
|
||||
try:
|
||||
@@ -734,6 +1114,7 @@ def registration(name: str):
|
||||
tld = name.rsplit(".", 1)[-1]
|
||||
if not REGISTRIES.get(tld):
|
||||
return 400, {"name": name, "error": "tldNotConfigured"}
|
||||
prefetch(lookup_reads(name))
|
||||
reg = name_status(name)
|
||||
status = reg["status"]
|
||||
if status in ("registered", "grace"):
|
||||
@@ -788,7 +1169,7 @@ def resolve(name: str):
|
||||
}
|
||||
|
||||
node = node_of(name)
|
||||
node_hex = node.hex()
|
||||
prefetch(lookup_reads(name))
|
||||
|
||||
# Before the resolver lookup, so a lapsed name is not reported as noResolver.
|
||||
reg = name_status(name)
|
||||
@@ -806,13 +1187,12 @@ def resolve(name: str):
|
||||
}
|
||||
return (404 if reg["status"] == "unregistered" else 410), body
|
||||
|
||||
resolver_raw = eth_call(registry, selector("resolver(bytes32)") + node_hex)
|
||||
resolver_addr = decode_address(resolver_raw)
|
||||
resolver_addr = decode_address(eth_call(*resolver_call(registry, node)))
|
||||
if resolver_addr == ZERO_ADDR:
|
||||
# A registered name always resolves: with no resolver set the record is
|
||||
# still returned with every field unset, so "taken until <date>" stays
|
||||
# answerable. For a subname, no owner means nobody created it.
|
||||
owner = decode_address(eth_call(registry, selector("owner(bytes32)") + node_hex))
|
||||
owner = decode_address(eth_call(*owner_call(registry, node)))
|
||||
if len(name.split(".")) > 2 and owner == ZERO_ADDR:
|
||||
return 404, {
|
||||
"name": name,
|
||||
@@ -837,9 +1217,9 @@ def resolve(name: str):
|
||||
**reg,
|
||||
}
|
||||
|
||||
owner_raw = eth_call(registry, selector("owner(bytes32)") + node_hex)
|
||||
owner = decode_address(owner_raw)
|
||||
owner = decode_address(eth_call(*owner_call(registry, node)))
|
||||
|
||||
prefetch(record_reads(resolver_addr, node))
|
||||
texts = {}
|
||||
for k in TEXT_KEYS:
|
||||
try:
|
||||
@@ -878,12 +1258,34 @@ def resolve(name: str):
|
||||
# ---------- HTTP layer ----------
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
# the smp-server reuses a connection only after an HTTP/1.1 response
|
||||
protocol_version = "HTTP/1.1"
|
||||
# idle seconds before a kept-alive connection is closed; the smp-server closes
|
||||
# its idle ones after 30-35 s, so it is the one to close them
|
||||
timeout = 60
|
||||
|
||||
def setup(self):
|
||||
super().setup()
|
||||
# headers and body go out in separate writes, which Nagle holds for the client's delayed ACK
|
||||
self.request.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
|
||||
def handle_one_request(self):
|
||||
# on a kept-alive connection these still hold the previous request's values
|
||||
self._started = self._sent = self.path = self.headers = None
|
||||
super().handle_one_request()
|
||||
|
||||
def do_GET(self): # noqa: N802 - http.server contract
|
||||
self._started = time.monotonic()
|
||||
if self._has_body():
|
||||
# an unread body would be taken for the next request on this connection, which a
|
||||
# reverse proxy reusing it would pass to another client (request smuggling)
|
||||
self._respond(400, {"error": "unexpectedBody"}, close=True)
|
||||
return
|
||||
path = urlparse(self.path).path
|
||||
parts = [unquote(p) for p in path.split("/") if p]
|
||||
|
||||
if parts == ["health"]:
|
||||
self._respond(200, {"ok": True, "rpc": RPC, "registries": REGISTRIES, **head_block()})
|
||||
self._respond(200, {"ok": True, "rpc": public_endpoint(RPC), "registries": REGISTRIES, **head_block()})
|
||||
return
|
||||
|
||||
if len(parts) == 3 and parts[0] == "v2" and parts[1] == "resolve":
|
||||
@@ -892,7 +1294,8 @@ class Handler(BaseHTTPRequestHandler):
|
||||
self._respond(400, {"name": name, "error": "notFullyQualified"})
|
||||
return
|
||||
try:
|
||||
status, body = registration(name)
|
||||
with request_reads():
|
||||
status, body = registration(name)
|
||||
except Exception as e: # surface upstream errors as 502
|
||||
status, body = 502, upstream_error({"name": name}, e)
|
||||
self._respond(status, body)
|
||||
@@ -915,7 +1318,8 @@ class Handler(BaseHTTPRequestHandler):
|
||||
)
|
||||
return
|
||||
try:
|
||||
status, body = resolve(name)
|
||||
with request_reads():
|
||||
status, body = resolve(name)
|
||||
except Exception as e: # surface upstream errors as 502
|
||||
status, body = 502, upstream_error({"name": name}, e)
|
||||
self._respond(status, body)
|
||||
@@ -930,34 +1334,176 @@ class Handler(BaseHTTPRequestHandler):
|
||||
},
|
||||
)
|
||||
|
||||
def _respond(self, status: int, body: dict):
|
||||
def _has_body(self) -> bool:
|
||||
# a malformed header line ends parsing, leaving the rest, Content-Length included, as payload
|
||||
if self.headers.defects or self.headers.get_payload():
|
||||
return True
|
||||
if self.headers.get_all("Transfer-Encoding"):
|
||||
return True
|
||||
return any(length.strip() != "0" for length in self.headers.get_all("Content-Length") or [])
|
||||
|
||||
def _respond(self, status: int, body: dict, close: bool = False):
|
||||
data = json.dumps(body, indent=2).encode()
|
||||
# send_response logs the request, so the size is known before it
|
||||
self._sent = len(data)
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
if close:
|
||||
# also sets close_connection, so the server closes it after this response
|
||||
self.send_header("Connection", "close")
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def address_string(self) -> str:
|
||||
forwarded = self.headers.get_all("X-Forwarded-For") if self.headers else None
|
||||
return client_address(self.client_address[0], ",".join(forwarded) if forwarded else None)
|
||||
|
||||
def log_request(self, code="-", size="-"):
|
||||
started = self._started
|
||||
path = urlparse(self.path).path if self.path else None
|
||||
log_event(
|
||||
# the container's health check runs every 30 s
|
||||
logging.DEBUG if path == "/health" else logging.INFO,
|
||||
"request",
|
||||
client=self.address_string(),
|
||||
worker=os.getpid(),
|
||||
method=getattr(self, "command", None),
|
||||
path=unquote(self.path) if self.path else None,
|
||||
status=getattr(code, "value", code),
|
||||
bytes=self._sent,
|
||||
ms=round((time.monotonic() - started) * 1000) if started else None,
|
||||
)
|
||||
|
||||
def log_error(self, fmt, *args):
|
||||
message = fmt % args
|
||||
# a kept-alive connection left idle is closed on purpose
|
||||
level = logging.DEBUG if message.startswith("Request timed out") else logging.WARNING
|
||||
log_event(level, "http_error", client=self.address_string(), worker=os.getpid(), message=message)
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
# Quiet the default per-request access log; route to stderr in one line.
|
||||
sys.stderr.write(f"{self.address_string()} - {fmt % args}\n")
|
||||
log_event(logging.INFO, "http", client=self.address_string(), message=fmt % args)
|
||||
|
||||
|
||||
class ResolverServer(ThreadingHTTPServer):
|
||||
# socketserver's default of 5 drops simultaneous connections, each retried by
|
||||
# TCP after 1 s, and the smp-server gives up after 3 s. The kernel caps it at somaxconn.
|
||||
request_queue_size = 128
|
||||
|
||||
def handle_error(self, request, client_address):
|
||||
error = sys.exc_info()[1]
|
||||
if isinstance(error, ConnectionError):
|
||||
# the smp-server hung up, usually after its own timeout
|
||||
log_event(logging.WARNING, "client_gone", client=client_address[0], worker=os.getpid(), error=type(error).__name__)
|
||||
else:
|
||||
log_event(logging.ERROR, "request_failed", client=client_address[0], worker=os.getpid(), exc_info=True)
|
||||
|
||||
|
||||
def public_endpoint(url: str) -> str:
|
||||
"""The RPC endpoint without credentials, path or query, where a provider key would be."""
|
||||
parsed = urlparse(url)
|
||||
return f"{parsed.scheme}://{parsed.netloc.rpartition('@')[2]}"
|
||||
|
||||
|
||||
def client_address(peer: str, forwarded_for: str | None) -> str:
|
||||
"""The peer, or behind trusted proxies the last X-Forwarded-For address none of them added."""
|
||||
if not forwarded_for or not _trusted(peer):
|
||||
return peer
|
||||
hops = [hop.strip() for hop in forwarded_for.split(",")]
|
||||
for hop in reversed(hops):
|
||||
# an address it cannot read, and anything claimed before it, is not believed
|
||||
if _address(hop) is None:
|
||||
return peer
|
||||
if not _trusted(hop):
|
||||
return hop
|
||||
return hops[0]
|
||||
|
||||
|
||||
def _address(text: str):
|
||||
try:
|
||||
ip = ipaddress.ip_address(text)
|
||||
except ValueError:
|
||||
return None
|
||||
return ip.ipv4_mapped or ip if ip.version == 6 else ip
|
||||
|
||||
|
||||
def _trusted(text: str) -> bool:
|
||||
ip = _address(text)
|
||||
return ip is not None and any(ip in net for net in TRUSTED_PROXIES)
|
||||
|
||||
|
||||
def serve(reuse_port: bool):
|
||||
server = ResolverServer((BIND, PORT), Handler, bind_and_activate=False)
|
||||
server.allow_reuse_port = reuse_port
|
||||
try:
|
||||
server.server_bind()
|
||||
server.server_activate()
|
||||
server.serve_forever()
|
||||
finally:
|
||||
server.server_close()
|
||||
|
||||
|
||||
def stop_on(signum, _frame):
|
||||
log_event(logging.INFO, "stopping", signal=signal.Signals(signum).name)
|
||||
sys.exit(0)
|
||||
|
||||
|
||||
def supervise(workers: int):
|
||||
"""Runs the workers, each with its own socket on the shared port. One worker
|
||||
exiting stops the others, so the container restarts instead of running short."""
|
||||
children = []
|
||||
|
||||
def stop(*_):
|
||||
for pid in children:
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
|
||||
def stop_and_exit(signum, frame):
|
||||
stop()
|
||||
stop_on(signum, frame)
|
||||
|
||||
# before forking, so a signal during startup cannot orphan the workers
|
||||
signal.signal(signal.SIGTERM, stop_and_exit)
|
||||
signal.signal(signal.SIGINT, stop_and_exit)
|
||||
for _ in range(workers):
|
||||
pid = os.fork()
|
||||
if pid == 0:
|
||||
signal.signal(signal.SIGTERM, signal.SIG_DFL)
|
||||
signal.signal(signal.SIGINT, signal.SIG_DFL)
|
||||
code = 0
|
||||
try:
|
||||
serve(reuse_port=True)
|
||||
except BaseException:
|
||||
log_event(logging.ERROR, "worker_failed", worker=os.getpid(), exc_info=True)
|
||||
code = 1
|
||||
os._exit(code)
|
||||
children.append(pid)
|
||||
pid, status = os.wait()
|
||||
log_event(logging.ERROR, "worker_exited", worker=pid, status=status, action="stopping")
|
||||
stop()
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
server = ThreadingHTTPServer((BIND, PORT), Handler)
|
||||
sys.stderr.write(
|
||||
f"snrc-resolve listening on {BIND}:{PORT}\n"
|
||||
f" RPC = {RPC}\n"
|
||||
f" Registries:\n"
|
||||
setup_logging()
|
||||
log_event(
|
||||
logging.INFO,
|
||||
"listening",
|
||||
bind=BIND,
|
||||
port=PORT,
|
||||
workers=WORKERS,
|
||||
rpc=public_endpoint(RPC),
|
||||
registries=",".join(f"{tld}={addr or '-'}" for tld, addr in REGISTRIES.items()),
|
||||
trusted_proxies=",".join(map(str, TRUSTED_PROXIES)) or None,
|
||||
)
|
||||
for tld, addr in REGISTRIES.items():
|
||||
sys.stderr.write(f" .{tld:<8s} = {addr or '(not configured)'}\n")
|
||||
sys.stderr.write(" GET /v2/resolve/<name> GET /v1/resolve/<name> GET /health\n")
|
||||
try:
|
||||
server.serve_forever()
|
||||
except KeyboardInterrupt:
|
||||
sys.stderr.write("\nshutting down\n")
|
||||
server.server_close()
|
||||
if WORKERS > 1:
|
||||
supervise(WORKERS)
|
||||
else:
|
||||
signal.signal(signal.SIGTERM, stop_on)
|
||||
signal.signal(signal.SIGINT, stop_on)
|
||||
serve(reuse_port=False)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,15 +1,30 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Unit tests for snrc-resolve helpers.
|
||||
|
||||
Run with `python3 -m unittest scripts/resolver/service/test_snrc_resolve.py`.
|
||||
Run with `uv run pytest` from scripts/resolver/service.
|
||||
"""
|
||||
|
||||
import contextlib
|
||||
import http.client
|
||||
import importlib.util
|
||||
import io
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import queue
|
||||
import re
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
import unittest
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.error import HTTPError
|
||||
from urllib.parse import urlparse
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
# snrc-resolve.py has a hyphen, so import it via importlib instead of `import`.
|
||||
_HERE = os.path.dirname(os.path.abspath(__file__))
|
||||
@@ -636,6 +651,25 @@ class PricingTests(unittest.TestCase):
|
||||
self.assertEqual(body["basePrice"], self.BASE)
|
||||
|
||||
|
||||
class DecodePricesTests(unittest.TestCase):
|
||||
def test_a_count_longer_than_the_answer_is_refused(self):
|
||||
"""The count comes from the oracle; decoding it unchecked could loop for ever."""
|
||||
huge = "0x" + snrc.encode_uint(200) + snrc.encode_uint(0x40) + snrc.encode_uint(2**64)
|
||||
outcome = []
|
||||
|
||||
def decode():
|
||||
try:
|
||||
snrc.decode_prices(huge)
|
||||
except RuntimeError as e:
|
||||
outcome.append(e)
|
||||
|
||||
worker = threading.Thread(target=decode, daemon=True)
|
||||
worker.start()
|
||||
worker.join(5)
|
||||
self.assertFalse(worker.is_alive(), "decoding did not stop")
|
||||
self.assertRegex(str(outcome[0]), "short response")
|
||||
|
||||
|
||||
class EnsOracleTests(unittest.TestCase):
|
||||
""".testing runs an ENS-shaped oracle: it prices in attoUSD per second and
|
||||
charges a premium on lapsed names that it does not expose."""
|
||||
@@ -777,13 +811,17 @@ class ErrorCodeTests(unittest.TestCase):
|
||||
self.assertNotEqual(body["error"], body["message"])
|
||||
|
||||
def test_an_upstream_failure_does_not_echo_the_exception(self):
|
||||
with contextlib.redirect_stderr(io.StringIO()) as log:
|
||||
with self.assertLogs("snrc_resolve", "WARNING") as logs:
|
||||
body = snrc.upstream_error(
|
||||
{"name": "alice.testing"},
|
||||
RuntimeError("http://user:secret@rpc.example/kEy8 refused"),
|
||||
)
|
||||
# the operator still sees the detail in the log
|
||||
self.assertIn("secret", log.getvalue())
|
||||
[record] = logs.records
|
||||
self.assertEqual(record.getMessage(), "upstream_error")
|
||||
self.assertEqual(record.fields["name"], "alice.testing")
|
||||
self.assertEqual(record.fields["error"], "RuntimeError")
|
||||
self.assertIn("secret", record.fields["message"])
|
||||
self.assertEqual(body["error"], "upstreamError")
|
||||
self.assertIn("RuntimeError", body["message"])
|
||||
self.assertNotIn("secret", body["message"])
|
||||
@@ -1035,5 +1073,797 @@ class RegistrationV2Tests(unittest.TestCase):
|
||||
self.assertEqual(res["lastBlockTs"], self.now)
|
||||
self.assertEqual(res["registration"]["type"], "available")
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@unittest.skipUnless(sys.platform.startswith("linux"), "Linux drops SYNs on a full accept queue")
|
||||
class ListenBacklogTests(unittest.TestCase):
|
||||
"""The smp-server opens a connection per lookup and gives up after 3 s, so a
|
||||
burst the accept queue cannot hold fails: TCP retries a dropped SYN after 1 s."""
|
||||
|
||||
BURST = 50
|
||||
|
||||
def test_a_burst_of_connections_is_queued_while_the_server_is_busy(self):
|
||||
# never accepts, so every connection must wait in the queue
|
||||
server = snrc.ResolverServer(("127.0.0.1", 0), snrc.Handler)
|
||||
clients = []
|
||||
try:
|
||||
for i in range(self.BURST):
|
||||
c = socket.socket()
|
||||
clients.append(c)
|
||||
c.settimeout(0.5)
|
||||
try:
|
||||
c.connect(server.server_address)
|
||||
except TimeoutError:
|
||||
self.fail(f"connection {i + 1} of {self.BURST} was not queued")
|
||||
finally:
|
||||
for c in clients:
|
||||
c.close()
|
||||
server.server_close()
|
||||
|
||||
|
||||
def _word(value: int) -> bytes:
|
||||
return value.to_bytes(32, "big")
|
||||
|
||||
|
||||
def _decode_aggregate3_calls(data: str):
|
||||
"""Multicall3.aggregate3 calldata back to (to, data) pairs, written apart
|
||||
from the resolver's encoder so the two check each other."""
|
||||
raw = bytes.fromhex(data[len(snrc.AGGREGATE3):])
|
||||
|
||||
def word(at):
|
||||
return int.from_bytes(raw[at:at + 32], "big")
|
||||
|
||||
array = word(0)
|
||||
base = array + 32
|
||||
calls = []
|
||||
for i in range(word(array)):
|
||||
item = base + word(base + 32 * i)
|
||||
call = item + word(item + 64)
|
||||
calls.append(("0x" + raw[item + 12:item + 32].hex(), "0x" + raw[call + 32:call + 32 + word(call)].hex()))
|
||||
return calls
|
||||
|
||||
|
||||
def _encode_aggregate3_results(results) -> str:
|
||||
tuples = [
|
||||
_word(int(ok)) + _word(0x40) + _word(len(data)) + data + b"\x00" * ((-len(data)) % 32)
|
||||
for ok, data in results
|
||||
]
|
||||
offsets, at = b"", 32 * len(tuples)
|
||||
for t in tuples:
|
||||
offsets += _word(at)
|
||||
at += len(t)
|
||||
return "0x" + (_word(0x20) + _word(len(tuples)) + offsets + b"".join(tuples)).hex()
|
||||
|
||||
|
||||
class FakeChain:
|
||||
"""Contract state for one registered name and one free name. Any other
|
||||
call reverts, as a view function asked for something unset does."""
|
||||
|
||||
REGISTRY = "0x58fc46996d975c57883564648bda5206d1a0102b"
|
||||
REGISTRAR = "0xef47eb4384b46c89e4482a677c2cbcbd2a6fd85a"
|
||||
CONTROLLER = "0x281ca41311c2aa808c917c4674639d7567b75714"
|
||||
ORACLE = "0x1e0c9a2b9d1a4c8f7b3e5d6a9c2f4b8e1d7a3c50"
|
||||
OWNER = "0xd83bd7e0e6b8a4c1f2593a7b0c4e8d1a6f9b2c37"
|
||||
RESOLVER = "0x80fa2b1c3d4e5f60718293a4b5c6d7e8f9012345"
|
||||
GRACE = 90 * 86400
|
||||
TEXTS = {"nickname": "Acme", "url": "https://acme.example", "simplex.channel": "https://a.example/c#1;https://b.example/c#2"}
|
||||
|
||||
def __init__(self):
|
||||
self.now = int(time.time())
|
||||
acme, free = snrc.label_token("acme"), snrc.label_token("free")
|
||||
node = snrc.node_of("acme.testing")
|
||||
abi_bytes = RegistrationV2Tests._abi_bytes
|
||||
prices = RegistrationV2Tests._prices_return(RegistrationV2Tests())
|
||||
self.answers = {
|
||||
snrc.expires_call(self.REGISTRAR, acme): "0x" + snrc.encode_uint(self.now + 3600),
|
||||
snrc.expires_call(self.REGISTRAR, free): "0x" + snrc.encode_uint(0),
|
||||
snrc.grace_call(self.REGISTRAR): "0x" + snrc.encode_uint(self.GRACE),
|
||||
snrc.label_call(self.REGISTRAR, acme): abi_bytes(b"acme"),
|
||||
snrc.reserved_call(self.CONTROLLER, acme): "0x" + snrc.encode_uint(0),
|
||||
snrc.reserved_call(self.CONTROLLER, free): "0x" + snrc.encode_uint(0),
|
||||
snrc.resolver_call(self.REGISTRY, node): "0x" + snrc.encode_uint(int(self.RESOLVER, 16)),
|
||||
snrc.owner_call(self.REGISTRY, node): "0x" + snrc.encode_uint(int(self.OWNER, 16)),
|
||||
snrc.addr_call(self.RESOLVER, node, snrc.COIN_ETH): abi_bytes(bytes.fromhex(self.OWNER[2:])),
|
||||
snrc.prices_call(self.CONTROLLER): "0x" + snrc.encode_uint(int(self.ORACLE, 16)),
|
||||
snrc.prices_call(self.ORACLE): prices,
|
||||
snrc.min_length_call(self.CONTROLLER): "0x" + snrc.encode_uint(3),
|
||||
}
|
||||
for key, value in self.TEXTS.items():
|
||||
self.answers[snrc.text_call(self.RESOLVER, node, key)] = abi_bytes(value.encode())
|
||||
|
||||
def call(self, to, data):
|
||||
answer = self.answers.get((to.lower(), data))
|
||||
if answer is None:
|
||||
raise RuntimeError("execution reverted")
|
||||
return answer
|
||||
|
||||
|
||||
class FakeNode(ThreadingHTTPServer):
|
||||
"""A JSON-RPC node over HTTP/1.1 keep-alive, serving FakeChain, with
|
||||
batches and Multicall3, each of which a test can take away."""
|
||||
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self):
|
||||
super().__init__(("127.0.0.1", 0), _FakeNodeHandler)
|
||||
self.chain = FakeChain()
|
||||
self.block = 100
|
||||
self.requests = 0
|
||||
self.connections = 0
|
||||
self.batch = True
|
||||
self.multicall = True
|
||||
self.multicall_null = False
|
||||
self.status = 200
|
||||
self.hang_up = False
|
||||
self.drop_after_reply = False
|
||||
threading.Thread(target=self.serve_forever, args=(0.05,), daemon=True).start()
|
||||
|
||||
@property
|
||||
def url(self):
|
||||
return f"http://127.0.0.1:{self.server_address[1]}/"
|
||||
|
||||
def stop(self):
|
||||
self.shutdown()
|
||||
self.server_close()
|
||||
|
||||
def answer(self, req):
|
||||
out = {"jsonrpc": "2.0", "id": req.get("id")}
|
||||
method, params = req["method"], req["params"]
|
||||
if method == "eth_blockNumber":
|
||||
out["result"] = hex(self.block)
|
||||
elif method == "eth_getBlockByNumber":
|
||||
out["result"] = {"number": hex(self.block), "timestamp": hex(self.chain.now)}
|
||||
elif method == "eth_call" and params[0]["to"].lower() == snrc.MULTICALL.lower():
|
||||
if not self.multicall:
|
||||
out["error"] = {"code": -32000, "message": "no contract code"}
|
||||
elif self.multicall_null:
|
||||
out["result"] = None
|
||||
else:
|
||||
results = []
|
||||
for to, data in _decode_aggregate3_calls(params[0]["data"]):
|
||||
try:
|
||||
results.append((True, bytes.fromhex(self.chain.call(to, data)[2:])))
|
||||
except RuntimeError:
|
||||
results.append((False, b""))
|
||||
out["result"] = _encode_aggregate3_results(results)
|
||||
elif method == "eth_call":
|
||||
try:
|
||||
out["result"] = self.chain.call(params[0]["to"], params[0]["data"])
|
||||
except RuntimeError:
|
||||
out["error"] = {"code": 3, "message": "execution reverted"}
|
||||
else:
|
||||
out["error"] = {"code": -32601, "message": "method not found"}
|
||||
return out
|
||||
|
||||
|
||||
class _FakeNodeHandler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def setup(self):
|
||||
super().setup()
|
||||
# headers and body go out in separate writes, which Nagle holds for the client's delayed ACK
|
||||
self.request.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1)
|
||||
self.server.connections += 1
|
||||
|
||||
def do_POST(self): # noqa: N802 - http.server contract
|
||||
request = json.loads(self.rfile.read(int(self.headers["Content-Length"])))
|
||||
node = self.server
|
||||
node.requests += 1
|
||||
if node.hang_up:
|
||||
self.close_connection = True
|
||||
return
|
||||
if node.status != 200:
|
||||
reply = {"error": "unavailable"}
|
||||
elif isinstance(request, list):
|
||||
reply = [node.answer(r) for r in request] if node.batch else {"jsonrpc": "2.0", "id": None, "error": {"code": -32600, "message": "batch not supported"}}
|
||||
else:
|
||||
reply = node.answer(request)
|
||||
data = json.dumps(reply).encode()
|
||||
self.send_response(node.status)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
# closes without `Connection: close`, as a node dropping an idle connection does
|
||||
self.close_connection = node.drop_after_reply
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
pass
|
||||
|
||||
|
||||
class FakeNodeTestCase(unittest.TestCase):
|
||||
"""Points the resolver at a FakeNode and at FakeChain's contracts."""
|
||||
|
||||
def setUp(self):
|
||||
self.node = FakeNode()
|
||||
self._saved = (snrc.RPC, snrc.RPC_URL, snrc.REGISTRIES, snrc.REGISTRARS, snrc.CONTROLLERS)
|
||||
snrc.RPC = self.node.url
|
||||
snrc.RPC_URL = urlparse(snrc.RPC)
|
||||
snrc.REGISTRIES = {"testing": FakeChain.REGISTRY, "simplex": ""}
|
||||
snrc.REGISTRARS = {"testing": FakeChain.REGISTRAR}
|
||||
snrc.CONTROLLERS = {"testing": FakeChain.CONTROLLER}
|
||||
snrc._multicall_failed_logged = False
|
||||
self._drain_pool()
|
||||
|
||||
def tearDown(self):
|
||||
self._drain_pool()
|
||||
snrc.RPC, snrc.RPC_URL, snrc.REGISTRIES, snrc.REGISTRARS, snrc.CONTROLLERS = self._saved
|
||||
self.node.stop()
|
||||
|
||||
def _drain_pool(self):
|
||||
while not snrc._rpc_pool.empty():
|
||||
snrc._rpc_pool.get_nowait().close()
|
||||
|
||||
def requests_made(self, action):
|
||||
before = self.node.requests
|
||||
result = action()
|
||||
return result, self.node.requests - before
|
||||
|
||||
|
||||
class RpcTransportTests(FakeNodeTestCase):
|
||||
"""A lookup makes several reads, and a new connection per read costs CPU
|
||||
and leaves a TIME_WAIT socket each, which exhausts local ports under load."""
|
||||
|
||||
def test_reads_share_one_connection(self):
|
||||
for _ in range(18):
|
||||
self.assertEqual(snrc.rpc("eth_blockNumber", []), hex(self.node.block))
|
||||
self.assertEqual(self.node.connections, 1)
|
||||
|
||||
def test_a_connection_the_node_closed_is_replaced(self):
|
||||
self.node.drop_after_reply = True
|
||||
for _ in range(3):
|
||||
self.assertEqual(snrc.rpc("eth_blockNumber", []), hex(self.node.block))
|
||||
self.assertEqual(self.node.connections, 3)
|
||||
|
||||
def test_a_fresh_connection_that_fails_is_not_retried(self):
|
||||
"""Only a pooled connection can be stale; a new one failing means the
|
||||
node is down, and resending would only double the wait."""
|
||||
self.node.hang_up = True
|
||||
with self.assertRaises(ConnectionError):
|
||||
snrc.rpc("eth_blockNumber", [])
|
||||
self.assertEqual(self.node.requests, 1)
|
||||
|
||||
def test_a_node_failure_is_not_a_reverted_call(self):
|
||||
"""Callers read RuntimeError as the call reverting and fall back to an
|
||||
empty value, so a node failure must not look like one."""
|
||||
self.node.status = 502
|
||||
with self.assertRaises(HTTPError) as cm:
|
||||
snrc.rpc("eth_blockNumber", [])
|
||||
self.assertNotIsInstance(cm.exception, RuntimeError)
|
||||
self.assertEqual(cm.exception.code, 502)
|
||||
|
||||
def test_a_reverted_call_is_a_runtime_error_and_keeps_the_connection(self):
|
||||
with self.assertRaises(RuntimeError):
|
||||
snrc.eth_call("0x" + "11" * 20, "0xdeadbeef")
|
||||
snrc.rpc("eth_blockNumber", [])
|
||||
self.assertEqual(self.node.connections, 1)
|
||||
|
||||
|
||||
class RpcPoolTests(FakeNodeTestCase):
|
||||
def test_connections_beyond_the_pool_are_closed_after_use(self):
|
||||
"""Every worker would otherwise keep its peak concurrency open to the node."""
|
||||
saved, snrc._rpc_pool = snrc._rpc_pool, queue.LifoQueue(maxsize=1)
|
||||
try:
|
||||
body = json.dumps({"jsonrpc": "2.0", "id": 1, "method": "eth_blockNumber", "params": []}).encode()
|
||||
kept, extra = snrc._new_rpc_connection(), snrc._new_rpc_connection()
|
||||
snrc._post_pooled(kept, body)
|
||||
# returning a connection to a full pool must not wait for a free slot
|
||||
done = threading.Thread(target=snrc._post_pooled, args=(extra, body), daemon=True)
|
||||
done.start()
|
||||
done.join(5)
|
||||
self.assertFalse(done.is_alive(), "returning a connection to a full pool blocked")
|
||||
self.assertEqual(snrc._rpc_pool.qsize(), 1)
|
||||
self.assertIsNotNone(kept.sock)
|
||||
self.assertIsNone(extra.sock)
|
||||
finally:
|
||||
while not snrc._rpc_pool.empty():
|
||||
snrc._rpc_pool.get_nowait().close()
|
||||
snrc._rpc_pool = saved
|
||||
|
||||
|
||||
class BatchedReadsTests(FakeNodeTestCase):
|
||||
"""Inside a request a round of reads is one round trip, and its contract
|
||||
reads one multicall, because a node runs the calls of a JSON-RPC batch one
|
||||
after another. Answers must be exactly those of reading one call at a time."""
|
||||
|
||||
def batched(self, answer, name):
|
||||
def action():
|
||||
with snrc.request_reads():
|
||||
return answer(name)
|
||||
return self.requests_made(action)
|
||||
|
||||
def assert_same_answer(self, answer, name, round_trips):
|
||||
one_by_one, one_by_one_trips = self.requests_made(lambda: answer(name))
|
||||
batched, batched_trips = self.batched(answer, name)
|
||||
self.assertEqual(batched, one_by_one)
|
||||
self.assertEqual(batched_trips, round_trips)
|
||||
self.assertGreater(one_by_one_trips, round_trips)
|
||||
return batched
|
||||
|
||||
def test_a_registered_name_takes_two_round_trips(self):
|
||||
status, body = self.assert_same_answer(snrc.registration, "acme.testing", 2)
|
||||
record = body["registration"]["nameRecord"]
|
||||
self.assertEqual(record["nickname"], "Acme")
|
||||
self.assertEqual(record["simplexChannel"], ["https://a.example/c#1", "https://b.example/c#2"])
|
||||
self.assertIsNone(record["btc"])
|
||||
|
||||
def test_a_hashed_query_is_named_from_the_same_round_trip(self):
|
||||
hashed = "[" + snrc.keccak(b"acme").hex() + "].testing"
|
||||
status, body = self.assert_same_answer(snrc.registration, hashed, 2)
|
||||
self.assertEqual(body["registration"]["nameRecord"]["name"], "acme.testing")
|
||||
|
||||
def test_an_available_name_is_priced_in_three_round_trips(self):
|
||||
status, body = self.assert_same_answer(snrc.registration, "free.testing", 3)
|
||||
self.assertEqual(body["registration"]["type"], "available")
|
||||
|
||||
def test_v1_answers_the_same(self):
|
||||
self.assert_same_answer(snrc.resolve, "acme.testing", 2)
|
||||
|
||||
def test_without_multicall_a_round_is_still_one_batch(self):
|
||||
self.node.multicall = False
|
||||
with self.assertLogs("snrc_resolve", "WARNING") as logs:
|
||||
self.assert_same_answer(snrc.registration, "acme.testing", 4)
|
||||
[record] = logs.records
|
||||
self.assertEqual((record.getMessage(), record.fields["fallback"]), ("multicall_unavailable", "batch"))
|
||||
|
||||
def test_a_multicall_without_a_result_falls_back_to_a_batch(self):
|
||||
self.node.multicall_null = True
|
||||
with self.assertLogs("snrc_resolve", "WARNING") as logs:
|
||||
self.assert_same_answer(snrc.registration, "acme.testing", 4)
|
||||
self.assertEqual([r.getMessage() for r in logs.records], ["multicall_unavailable"])
|
||||
|
||||
def test_an_answer_with_neither_result_nor_error_is_not_taken_as_one(self):
|
||||
reads = {}
|
||||
snrc._remember(reads, [("eth_blockNumber", [])], [{"jsonrpc": "2.0", "id": 0}])
|
||||
self.assertEqual(reads, {})
|
||||
|
||||
def test_a_node_that_does_not_batch_is_read_one_call_at_a_time(self):
|
||||
self.node.batch = False
|
||||
one_by_one, one_by_one_trips = self.requests_made(lambda: snrc.registration("acme.testing"))
|
||||
with self.assertNoLogs("snrc_resolve"):
|
||||
batched, batched_trips = self.batched(snrc.registration, "acme.testing")
|
||||
self.assertEqual(batched, one_by_one)
|
||||
# one refused batch per round, then the reads the batch would have made
|
||||
self.assertEqual(batched_trips, one_by_one_trips + 2)
|
||||
|
||||
def test_a_read_reverted_in_the_multicall_is_a_reverted_call(self):
|
||||
with snrc.request_reads():
|
||||
snrc.prefetch([snrc.eth_call_read(*snrc.grace_call(FakeChain.REGISTRAR)), snrc.eth_call_read(FakeChain.REGISTRY, "0xdeadbeef")])
|
||||
_, trips = self.requests_made(lambda: self.assertRaises(RuntimeError, snrc.eth_call, FakeChain.REGISTRY, "0xdeadbeef"))
|
||||
self.assertEqual(trips, 0)
|
||||
|
||||
def test_outside_a_request_nothing_is_prefetched(self):
|
||||
_, trips = self.requests_made(lambda: snrc.prefetch(snrc.lookup_reads("acme.testing")))
|
||||
self.assertEqual(trips, 0)
|
||||
|
||||
|
||||
class Aggregate3Tests(unittest.TestCase):
|
||||
def test_calls_are_encoded_as_multicall3_reads_them(self):
|
||||
calls = [(FakeChain.REGISTRY, "0x0178b8bf" + "11" * 32), (FakeChain.RESOLVER, "0x59d1d43c" + "22" * 100)]
|
||||
data = snrc.encode_aggregate3(calls)
|
||||
self.assertTrue(data.startswith(snrc.AGGREGATE3))
|
||||
self.assertEqual(_decode_aggregate3_calls(data), calls)
|
||||
|
||||
def test_results_are_decoded_with_their_success_flags(self):
|
||||
results = [(True, b"\x01" * 40), (False, b""), (True, b"")]
|
||||
self.assertEqual(snrc.decode_aggregate3(_encode_aggregate3_results(results)), results)
|
||||
|
||||
def test_a_truncated_answer_is_refused(self):
|
||||
whole = _encode_aggregate3_results([(True, b"\x01" * 40)])
|
||||
with self.assertRaises(ValueError):
|
||||
snrc.decode_aggregate3(whole[:-64])
|
||||
with self.assertRaises(ValueError):
|
||||
snrc.decode_aggregate3("0x")
|
||||
|
||||
|
||||
class RequestLogTests(unittest.TestCase):
|
||||
"""Each request is one event with the client it came from, behind a reverse
|
||||
proxy the client the proxy names rather than the proxy itself."""
|
||||
|
||||
def setUp(self):
|
||||
self.server = snrc.ResolverServer(("127.0.0.1", 0), snrc.Handler)
|
||||
threading.Thread(target=self.server.serve_forever, args=(0.05,), daemon=True).start()
|
||||
self._saved = snrc.TRUSTED_PROXIES
|
||||
|
||||
def tearDown(self):
|
||||
snrc.TRUSTED_PROXIES = self._saved
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
|
||||
def request(self, path, headers=None, level="INFO"):
|
||||
url = f"http://127.0.0.1:{self.server.server_address[1]}{path}"
|
||||
with self.assertLogs("snrc_resolve", level) as logs:
|
||||
try:
|
||||
urlopen(Request(url, headers=headers or {}), timeout=5).read()
|
||||
except HTTPError:
|
||||
pass
|
||||
deadline = time.monotonic() + 2
|
||||
while not any(r.getMessage() == "request" for r in logs.records) and time.monotonic() < deadline:
|
||||
time.sleep(0.01)
|
||||
[record] = [r for r in logs.records if r.getMessage() == "request"]
|
||||
return record
|
||||
|
||||
def test_a_request_is_logged_with_its_outcome(self):
|
||||
record = self.request("/v2/resolve/x.simplex")
|
||||
fields = record.fields
|
||||
self.assertEqual(record.levelname, "INFO")
|
||||
self.assertEqual(
|
||||
(fields["client"], fields["method"], fields["path"], fields["status"], fields["worker"]),
|
||||
("127.0.0.1", "GET", "/v2/resolve/x.simplex", 400, os.getpid()),
|
||||
)
|
||||
self.assertGreater(fields["bytes"], 0)
|
||||
self.assertGreaterEqual(fields["ms"], 0)
|
||||
|
||||
def test_a_trusted_proxy_names_the_client(self):
|
||||
snrc.TRUSTED_PROXIES = (ipaddress.ip_network("127.0.0.1/32"),)
|
||||
record = self.request("/v2/resolve/x.simplex", {"X-Forwarded-For": "203.0.113.7"})
|
||||
self.assertEqual(record.fields["client"], "203.0.113.7")
|
||||
|
||||
def test_any_other_peer_cannot_name_itself(self):
|
||||
record = self.request("/v2/resolve/x.simplex", {"X-Forwarded-For": "203.0.113.7"})
|
||||
self.assertEqual(record.fields["client"], "127.0.0.1")
|
||||
|
||||
def test_a_bad_request_is_not_logged_with_the_previous_ones_fields(self):
|
||||
"""On a kept-alive connection the handler still held the last request's path and size."""
|
||||
with self.assertLogs("snrc_resolve", "INFO") as logs:
|
||||
with socket.create_connection(self.server.server_address, timeout=5) as sock:
|
||||
sock.sendall(b"GET /v2/resolve/x.simplex HTTP/1.1\r\nHost: x\r\n\r\n")
|
||||
sock.recv(65536)
|
||||
sock.sendall(b"BOGUS\r\n\r\n")
|
||||
while sock.recv(65536):
|
||||
pass
|
||||
first, second = [r for r in logs.records if r.getMessage() == "request"]
|
||||
self.assertEqual(first.fields["path"], "/v2/resolve/x.simplex")
|
||||
self.assertEqual((second.fields["status"], second.fields["path"], second.fields["bytes"], second.fields["ms"]), (400, None, None, None))
|
||||
|
||||
def test_health_checks_are_logged_only_at_debug(self):
|
||||
"""The container checks /health every 30 s."""
|
||||
record = self.request("/health", level="DEBUG")
|
||||
self.assertEqual(record.levelname, "DEBUG")
|
||||
|
||||
|
||||
class RequestBodyTests(unittest.TestCase):
|
||||
"""Behind a reverse proxy that reuses connections, a body left unread would be
|
||||
answered as another request, and that answer given to the proxy's next client."""
|
||||
|
||||
def setUp(self):
|
||||
self.server = snrc.ResolverServer(("127.0.0.1", 0), snrc.Handler)
|
||||
threading.Thread(target=self.server.serve_forever, args=(0.05,), daemon=True).start()
|
||||
self._log = contextlib.redirect_stderr(io.StringIO())
|
||||
self._log.__enter__()
|
||||
|
||||
def tearDown(self):
|
||||
self._log.__exit__(None, None, None)
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
|
||||
def exchange(self, head: bytes, body: bytes = b"") -> bytes:
|
||||
with socket.create_connection(self.server.server_address, timeout=5) as sock:
|
||||
sock.sendall(head + body)
|
||||
data = b""
|
||||
while chunk := sock.recv(65536):
|
||||
data += chunk
|
||||
if data.count(b"HTTP/1.1 ") > 1:
|
||||
break
|
||||
return data
|
||||
|
||||
SMUGGLED = b"GET /v2/resolve/y.simplex HTTP/1.1\r\nHost: x\r\n\r\n"
|
||||
|
||||
def test_a_body_is_refused_and_the_connection_closed(self):
|
||||
data = self.exchange(b"GET /v2/resolve/x.simplex HTTP/1.1\r\nHost: x\r\nContent-Length: %d\r\n\r\n" % len(self.SMUGGLED), self.SMUGGLED)
|
||||
self.assertEqual(data.count(b"HTTP/1.1 "), 1)
|
||||
self.assertIn(b"HTTP/1.1 400 ", data)
|
||||
self.assertIn(b"Connection: close", data)
|
||||
self.assertNotIn(b"y.simplex", data)
|
||||
|
||||
def test_a_length_after_a_malformed_header_line_is_still_seen(self):
|
||||
"""Python stops parsing headers at a malformed line and keeps the rest as payload."""
|
||||
data = self.exchange(b"GET /v2/resolve/x.simplex HTTP/1.1\r\nX-Junk : 1\r\nContent-Length: %d\r\n\r\n" % len(self.SMUGGLED), self.SMUGGLED)
|
||||
self.assertEqual(data.count(b"HTTP/1.1 "), 1)
|
||||
self.assertIn(b"unexpectedBody", data)
|
||||
self.assertNotIn(b"y.simplex", data)
|
||||
|
||||
def test_a_chunked_body_is_refused_and_the_connection_closed(self):
|
||||
chunked = b"%x\r\n" % len(self.SMUGGLED) + self.SMUGGLED + b"\r\n0\r\n\r\n"
|
||||
data = self.exchange(b"GET /v2/resolve/x.simplex HTTP/1.1\r\nHost: x\r\nTransfer-Encoding: chunked\r\n\r\n", chunked)
|
||||
self.assertEqual(data.count(b"HTTP/1.1 "), 1)
|
||||
self.assertNotIn(b"y.simplex", data)
|
||||
|
||||
def test_an_empty_body_keeps_the_connection(self):
|
||||
conn = http.client.HTTPConnection(*self.server.server_address, timeout=5)
|
||||
answers = []
|
||||
for _ in range(2):
|
||||
conn.request("GET", "/v2/resolve/x.simplex", headers={"Content-Length": "0"})
|
||||
res = conn.getresponse()
|
||||
answers.append((res.status, json.loads(res.read())["error"], conn.sock))
|
||||
conn.close()
|
||||
self.assertEqual([a[:2] for a in answers], [(400, "tldNotConfigured")] * 2)
|
||||
self.assertIs(answers[0][2], answers[1][2])
|
||||
|
||||
|
||||
class PublicEndpointTests(unittest.TestCase):
|
||||
def test_credentials_path_and_query_are_not_shown(self):
|
||||
self.assertEqual(snrc.public_endpoint("https://user:pw@rpc.example:8443/v2/KEY?apikey=K"), "https://rpc.example:8443")
|
||||
|
||||
def test_a_plain_endpoint_is_unchanged(self):
|
||||
self.assertEqual(snrc.public_endpoint("http://reth:8545"), "http://reth:8545")
|
||||
|
||||
def test_an_ipv6_host_keeps_its_brackets(self):
|
||||
self.assertEqual(snrc.public_endpoint("http://u:p@[::1]:8545/key"), "http://[::1]:8545")
|
||||
|
||||
def test_a_malformed_port_does_not_break_health(self):
|
||||
self.assertEqual(snrc.public_endpoint("http://reth:port"), "http://reth:port")
|
||||
|
||||
|
||||
class ClientAddressTests(unittest.TestCase):
|
||||
PROXY = "172.18.0.1"
|
||||
|
||||
def setUp(self):
|
||||
self._saved = snrc.TRUSTED_PROXIES
|
||||
snrc.TRUSTED_PROXIES = (ipaddress.ip_network("172.16.0.0/12"),)
|
||||
|
||||
def tearDown(self):
|
||||
snrc.TRUSTED_PROXIES = self._saved
|
||||
|
||||
def test_a_peer_that_is_no_proxy_is_the_client(self):
|
||||
self.assertEqual(snrc.client_address("198.51.100.9", "203.0.113.7"), "198.51.100.9")
|
||||
|
||||
def test_without_the_header_the_proxy_is_the_client(self):
|
||||
self.assertEqual(snrc.client_address(self.PROXY, None), self.PROXY)
|
||||
|
||||
def test_the_address_the_proxy_added_is_the_client(self):
|
||||
self.assertEqual(snrc.client_address(self.PROXY, "203.0.113.7"), "203.0.113.7")
|
||||
|
||||
def test_addresses_a_client_sent_itself_are_not_believed(self):
|
||||
"""A proxy appends the address it saw, so only the last untrusted one is known."""
|
||||
self.assertEqual(snrc.client_address(self.PROXY, "6.6.6.6, 203.0.113.7"), "203.0.113.7")
|
||||
|
||||
def test_trusted_proxies_in_a_chain_are_skipped(self):
|
||||
self.assertEqual(snrc.client_address(self.PROXY, "203.0.113.7, 172.18.0.5"), "203.0.113.7")
|
||||
|
||||
def test_an_address_that_does_not_parse_is_not_believed(self):
|
||||
self.assertEqual(snrc.client_address(self.PROXY, "203.0.113.7, not-an-ip"), self.PROXY)
|
||||
|
||||
def test_an_ipv4_mapped_peer_is_matched_as_ipv4(self):
|
||||
self.assertEqual(snrc.client_address("::ffff:172.18.0.1", "203.0.113.7"), "203.0.113.7")
|
||||
|
||||
def test_an_ipv6_client_is_kept(self):
|
||||
self.assertEqual(snrc.client_address(self.PROXY, "2001:db8::7"), "2001:db8::7")
|
||||
|
||||
|
||||
class RequestErrorTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.server = snrc.ResolverServer(("127.0.0.1", 0), snrc.Handler)
|
||||
|
||||
def tearDown(self):
|
||||
self.server.server_close()
|
||||
|
||||
def test_a_client_that_hung_up_is_a_warning_without_a_traceback(self):
|
||||
"""What an smp-server that gave up after its timeout leaves behind."""
|
||||
with self.assertLogs("snrc_resolve", "WARNING") as logs:
|
||||
try:
|
||||
raise BrokenPipeError
|
||||
except BrokenPipeError:
|
||||
self.server.handle_error(None, ("198.51.100.9", 4000))
|
||||
[record] = logs.records
|
||||
self.assertEqual((record.levelname, record.getMessage(), record.fields["error"]), ("WARNING", "client_gone", "BrokenPipeError"))
|
||||
self.assertIsNone(record.exc_info)
|
||||
|
||||
def test_a_failure_is_an_error_with_its_traceback(self):
|
||||
with self.assertLogs("snrc_resolve", "ERROR") as logs:
|
||||
try:
|
||||
raise KeyError("boom")
|
||||
except KeyError:
|
||||
self.server.handle_error(None, ("198.51.100.9", 4000))
|
||||
[record] = logs.records
|
||||
self.assertEqual((record.getMessage(), record.fields["client"]), ("request_failed", "198.51.100.9"))
|
||||
self.assertIsNotNone(record.exc_info)
|
||||
|
||||
|
||||
class KeepAliveTests(unittest.TestCase):
|
||||
"""The smp-server keeps a resolver connection only after an HTTP/1.1
|
||||
response, and otherwise connects for every lookup."""
|
||||
|
||||
def setUp(self):
|
||||
self.server = snrc.ResolverServer(("127.0.0.1", 0), snrc.Handler)
|
||||
threading.Thread(target=self.server.serve_forever, args=(0.05,), daemon=True).start()
|
||||
self._saved_timeout = snrc.Handler.timeout
|
||||
self._log = contextlib.redirect_stderr(io.StringIO())
|
||||
self._log.__enter__()
|
||||
|
||||
def tearDown(self):
|
||||
self._log.__exit__(None, None, None)
|
||||
snrc.Handler.timeout = self._saved_timeout
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
|
||||
def test_requests_share_one_connection_without_delay(self):
|
||||
conn = http.client.HTTPConnection(*self.server.server_address, timeout=5)
|
||||
start = time.monotonic()
|
||||
for i in range(20):
|
||||
conn.request("GET", "/v2/resolve/x.simplex")
|
||||
res = conn.getresponse()
|
||||
res.read()
|
||||
self.assertEqual((res.version, res.status), (11, 400))
|
||||
if i == 0:
|
||||
sock = conn.sock
|
||||
self.assertIs(conn.sock, sock)
|
||||
conn.close()
|
||||
# a response held by Nagle for the delayed ACK takes ~40 ms, 20 of them over 0.8 s
|
||||
self.assertLess(time.monotonic() - start, 0.5)
|
||||
|
||||
def test_idle_connections_outlast_the_smp_servers(self):
|
||||
"""http-client drops a connection idle for 30 s, checked every 5 s. A
|
||||
resolver that closed sooner would race the client reusing it, and one
|
||||
that never closed would keep a thread per dead connection."""
|
||||
self.assertIsNotNone(self._saved_timeout)
|
||||
self.assertGreater(self._saved_timeout, 35)
|
||||
|
||||
def test_an_idle_connection_is_closed(self):
|
||||
snrc.Handler.timeout = 0.2
|
||||
with socket.create_connection(self.server.server_address, timeout=5) as sock:
|
||||
time.sleep(0.5)
|
||||
self.assertEqual(sock.recv(1), b"")
|
||||
|
||||
|
||||
@unittest.skipUnless(sys.platform.startswith("linux"), "reads worker processes from /proc")
|
||||
class WorkerProcessesTests(unittest.TestCase):
|
||||
"""Workers share the port, and the service stops as a whole, so the
|
||||
container restarts rather than serving on fewer workers."""
|
||||
|
||||
def setUp(self):
|
||||
self.node = FakeNode()
|
||||
with socket.socket() as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
self.port = s.getsockname()[1]
|
||||
env = dict(os.environ, SNRC_RPC=self.node.url, SNRC_BIND="127.0.0.1", SNRC_PORT=str(self.port), SNRC_WORKERS="2",
|
||||
SNRC_REGISTRY_TESTING=FakeChain.REGISTRY, SNRC_REGISTRAR_TESTING=FakeChain.REGISTRAR, SNRC_CONTROLLER_TESTING=FakeChain.CONTROLLER)
|
||||
self.service = subprocess.Popen([sys.executable, os.path.join(_HERE, "snrc-resolve.py")], env=env, stderr=subprocess.PIPE, text=True)
|
||||
self.workers = self._wait_for_workers(2)
|
||||
|
||||
def tearDown(self):
|
||||
if self.service.poll() is None:
|
||||
self.service.kill()
|
||||
self.service.wait()
|
||||
self.service.stderr.close()
|
||||
for pid in self.workers:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
os.kill(pid, signal.SIGKILL)
|
||||
self.node.stop()
|
||||
|
||||
def _wait_for_workers(self, count):
|
||||
deadline = time.monotonic() + 10
|
||||
while time.monotonic() < deadline:
|
||||
workers = [int(p) for p in os.listdir("/proc") if p.isdigit() and self._parent(p) == self.service.pid]
|
||||
if len(workers) == count and self._serving():
|
||||
return workers
|
||||
time.sleep(0.05)
|
||||
self.fail("workers did not start")
|
||||
|
||||
@staticmethod
|
||||
def _parent(pid):
|
||||
try:
|
||||
with open(f"/proc/{pid}/stat") as f:
|
||||
return int(f.read().rsplit(")", 1)[1].split()[1])
|
||||
except (FileNotFoundError, ProcessLookupError):
|
||||
return None
|
||||
|
||||
def _serving(self):
|
||||
try:
|
||||
with socket.create_connection(("127.0.0.1", self.port), timeout=1):
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
def _gone(self, pid):
|
||||
deadline = time.monotonic() + 5
|
||||
while time.monotonic() < deadline:
|
||||
if self._parent(pid) != self.service.pid:
|
||||
return True
|
||||
time.sleep(0.05)
|
||||
return False
|
||||
|
||||
def test_workers_answer_on_the_shared_port(self):
|
||||
for _ in range(20):
|
||||
with urlopen(f"http://127.0.0.1:{self.port}/v2/resolve/acme.testing", timeout=5) as res:
|
||||
self.assertEqual(json.loads(res.read())["registration"]["type"], "registered")
|
||||
|
||||
def test_stopping_the_service_stops_every_worker(self):
|
||||
self.service.send_signal(signal.SIGTERM)
|
||||
self.assertEqual(self.service.wait(timeout=5), 0)
|
||||
self.assertTrue(all(self._gone(pid) for pid in self.workers))
|
||||
self.assertRegex(self.service.stderr.read(), r"INFO stopping signal=SIGTERM")
|
||||
|
||||
def test_a_worker_exiting_stops_the_service(self):
|
||||
os.kill(self.workers[0], signal.SIGKILL)
|
||||
self.assertEqual(self.service.wait(timeout=5), 1)
|
||||
self.assertTrue(self._gone(self.workers[1]))
|
||||
|
||||
|
||||
TIME = r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d{3}Z"
|
||||
|
||||
|
||||
class LogTestCase(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self._saved = (snrc.LOG_FORMAT, snrc.LOG_COLOR, snrc.LOG_LEVEL)
|
||||
snrc.LOG_FORMAT, snrc.LOG_COLOR, snrc.LOG_LEVEL = "text", "never", "info"
|
||||
self.out = io.StringIO()
|
||||
|
||||
def tearDown(self):
|
||||
snrc.LOG_FORMAT, snrc.LOG_COLOR, snrc.LOG_LEVEL = self._saved
|
||||
snrc.LOGGER.handlers[:] = []
|
||||
snrc.LOGGER.setLevel(logging.NOTSET)
|
||||
snrc.LOGGER.propagate = True
|
||||
|
||||
def emit(self, level=logging.INFO, name="request", **fields):
|
||||
snrc.setup_logging(self.out)
|
||||
snrc.log_event(level, name, **fields)
|
||||
return self.out.getvalue()
|
||||
|
||||
|
||||
class TextFormatTests(LogTestCase):
|
||||
def test_a_line_is_utc_time_level_event_and_fields(self):
|
||||
line = self.emit(client="203.0.113.7", path="/v2/resolve/[4fdd].testing", status=200, ms=7)
|
||||
self.assertRegex(line, rf"^{TIME} INFO request client=203\.0\.113\.7 path=/v2/resolve/\[4fdd\]\.testing status=200 ms=7\n$")
|
||||
|
||||
def test_values_that_would_be_misread_are_quoted(self):
|
||||
line = self.emit(message='says "hi" here', empty="", missing=None, eq="a=b")
|
||||
self.assertIn('message="says \\"hi\\" here" empty="" missing=- eq="a=b"', line)
|
||||
|
||||
def test_levels_are_named_in_five_columns(self):
|
||||
self.assertRegex(self.emit(logging.WARNING, "upstream_error"), rf"^{TIME} WARN upstream_error\n$")
|
||||
|
||||
def test_below_the_configured_level_nothing_is_written(self):
|
||||
self.assertEqual(self.emit(logging.DEBUG), "")
|
||||
|
||||
def test_an_exception_follows_its_line(self):
|
||||
snrc.setup_logging(self.out)
|
||||
try:
|
||||
raise KeyError("boom")
|
||||
except KeyError:
|
||||
snrc.log_event(logging.ERROR, "request_failed", exc_info=True)
|
||||
self.assertRegex(self.out.getvalue(), rf"(?s)^{TIME} ERROR request_failed\nTraceback .*KeyError: 'boom'\n$")
|
||||
|
||||
|
||||
class ColorTests(LogTestCase):
|
||||
def test_colours_mark_the_level_and_the_status_class(self):
|
||||
snrc.LOG_COLOR = "always"
|
||||
line = self.emit(status=503)
|
||||
self.assertIn("\033[32mINFO ", line)
|
||||
self.assertIn("\033[31m503\033[0m", line)
|
||||
|
||||
def test_auto_leaves_output_that_is_no_terminal_plain(self):
|
||||
snrc.LOG_COLOR = "auto"
|
||||
self.assertNotIn("\033[", self.emit(status=200))
|
||||
|
||||
def test_never_is_plain(self):
|
||||
self.assertNotIn("\033[", self.emit(status=200))
|
||||
|
||||
|
||||
class JsonFormatTests(LogTestCase):
|
||||
def test_a_line_is_one_json_object(self):
|
||||
snrc.LOG_FORMAT = "json"
|
||||
record = json.loads(self.emit(client="203.0.113.7", status=200, missing=None))
|
||||
self.assertRegex(record.pop("time"), rf"^{TIME}$")
|
||||
self.assertEqual(record, {"level": "info", "event": "request", "client": "203.0.113.7", "status": 200, "missing": None})
|
||||
|
||||
def test_an_exception_is_a_field(self):
|
||||
snrc.LOG_FORMAT = "json"
|
||||
snrc.setup_logging(self.out)
|
||||
try:
|
||||
raise KeyError("boom")
|
||||
except KeyError:
|
||||
snrc.log_event(logging.ERROR, "request_failed", exc_info=True)
|
||||
self.assertIn("KeyError: 'boom'", json.loads(self.out.getvalue())["exception"])
|
||||
|
||||
|
||||
class SetupTests(LogTestCase):
|
||||
def test_unknown_settings_are_refused_at_start(self):
|
||||
for setting, value in (("LOG_FORMAT", "yaml"), ("LOG_COLOR", "sometimes"), ("LOG_LEVEL", "loud")):
|
||||
with self.subTest(setting=setting):
|
||||
saved = getattr(snrc, setting)
|
||||
setattr(snrc, setting, value)
|
||||
try:
|
||||
with self.assertRaisesRegex(ValueError, re.escape(f"SNRC_{setting}")):
|
||||
snrc.setup_logging(self.out)
|
||||
finally:
|
||||
setattr(snrc, setting, saved)
|
||||
|
||||
Generated
+156
@@ -0,0 +1,156 @@
|
||||
version = 1
|
||||
revision = 3
|
||||
requires-python = ">=3.11"
|
||||
|
||||
[[package]]
|
||||
name = "colorama"
|
||||
version = "0.4.6"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "eth-hash"
|
||||
version = "0.8.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/3c/f5/c67fc24f2f676aa9b7ab29679d44f113f314c817207cd4319353356f62da/eth_hash-0.8.0.tar.gz", hash = "sha256:b009752b620da2e9c7668014849d1f5fadbe4f138603f1871cc5d4ca706896b1", size = 12225, upload-time = "2026-03-25T16:36:55.099Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/87/87/b36792150ca0b28e4df683a34be15a61461ca0e349e5b5cf3ec8f694edb9/eth_hash-0.8.0-py3-none-any.whl", hash = "sha256:523718a51b369ab89866b929a5c93c52978cd866ea309192ad980dd8271f9fac", size = 7965, upload-time = "2026-03-25T16:36:54.205Z" },
|
||||
]
|
||||
|
||||
[package.optional-dependencies]
|
||||
pycryptodome = [
|
||||
{ name = "pycryptodome" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iniconfig"
|
||||
version = "2.3.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/72/34/14ca021ce8e5dfedc35312d08ba8bf51fdd999c576889fc2c24cb97f4f10/iniconfig-2.3.0.tar.gz", hash = "sha256:c76315c77db068650d49c5b56314774a7804df16fee4402c1f19d6d15d8c4730", size = 20503, upload-time = "2025-10-18T21:55:43.219Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "packaging"
|
||||
version = "26.3"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pluggy"
|
||||
version = "1.6.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/f9/e2/3e91f31a7d2b083fe6ef3fa267035b518369d9511ffab804f839851d2779/pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3", size = 69412, upload-time = "2025-05-15T12:30:07.975Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pycryptodome"
|
||||
version = "3.23.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/8e/a6/8452177684d5e906854776276ddd34eca30d1b1e15aa1ee9cefc289a33f5/pycryptodome-3.23.0.tar.gz", hash = "sha256:447700a657182d60338bab09fdb27518f8856aecd80ae4c6bdddb67ff5da44ef", size = 4921276, upload-time = "2025-05-17T17:21:45.242Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/04/5d/bdb09489b63cd34a976cc9e2a8d938114f7a53a74d3dd4f125ffa49dce82/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:0011f7f00cdb74879142011f95133274741778abba114ceca229adbf8e62c3e4", size = 2495152, upload-time = "2025-05-17T17:20:20.833Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a7/ce/7840250ed4cc0039c433cd41715536f926d6e86ce84e904068eb3244b6a6/pycryptodome-3.23.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:90460fc9e088ce095f9ee8356722d4f10f86e5be06e2354230a9880b9c549aae", size = 1639348, upload-time = "2025-05-17T17:20:23.171Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ee/f0/991da24c55c1f688d6a3b5a11940567353f74590734ee4a64294834ae472/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4764e64b269fc83b00f682c47443c2e6e85b18273712b98aa43bcb77f8570477", size = 2184033, upload-time = "2025-05-17T17:20:25.424Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/54/16/0e11882deddf00f68b68dd4e8e442ddc30641f31afeb2bc25588124ac8de/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:eb8f24adb74984aa0e5d07a2368ad95276cf38051fe2dc6605cbcf482e04f2a7", size = 2270142, upload-time = "2025-05-17T17:20:27.808Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d5/fc/4347fea23a3f95ffb931f383ff28b3f7b1fe868739182cb76718c0da86a1/pycryptodome-3.23.0-cp313-cp313t-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d97618c9c6684a97ef7637ba43bdf6663a2e2e77efe0f863cce97a76af396446", size = 2309384, upload-time = "2025-05-17T17:20:30.765Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/d9/c5261780b69ce66d8cfab25d2797bd6e82ba0241804694cd48be41add5eb/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9a53a4fe5cb075075d515797d6ce2f56772ea7e6a1e5e4b96cf78a14bac3d265", size = 2183237, upload-time = "2025-05-17T17:20:33.736Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5a/6f/3af2ffedd5cfa08c631f89452c6648c4d779e7772dfc388c77c920ca6bbf/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_i686.whl", hash = "sha256:763d1d74f56f031788e5d307029caef067febf890cd1f8bf61183ae142f1a77b", size = 2343898, upload-time = "2025-05-17T17:20:36.086Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/9a/dc/9060d807039ee5de6e2f260f72f3d70ac213993a804f5e67e0a73a56dd2f/pycryptodome-3.23.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:954af0e2bd7cea83ce72243b14e4fb518b18f0c1649b576d114973e2073b273d", size = 2269197, upload-time = "2025-05-17T17:20:38.414Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/34/e6c8ca177cb29dcc4967fef73f5de445912f93bd0343c9c33c8e5bf8cde8/pycryptodome-3.23.0-cp313-cp313t-win32.whl", hash = "sha256:257bb3572c63ad8ba40b89f6fc9d63a2a628e9f9708d31ee26560925ebe0210a", size = 1768600, upload-time = "2025-05-17T17:20:40.688Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/e4/1d/89756b8d7ff623ad0160f4539da571d1f594d21ee6d68be130a6eccb39a4/pycryptodome-3.23.0-cp313-cp313t-win_amd64.whl", hash = "sha256:6501790c5b62a29fcb227bd6b62012181d886a767ce9ed03b303d1f22eb5c625", size = 1799740, upload-time = "2025-05-17T17:20:42.413Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5d/61/35a64f0feaea9fd07f0d91209e7be91726eb48c0f1bfc6720647194071e4/pycryptodome-3.23.0-cp313-cp313t-win_arm64.whl", hash = "sha256:9a77627a330ab23ca43b48b130e202582e91cc69619947840ea4d2d1be21eb39", size = 1703685, upload-time = "2025-05-17T17:20:44.388Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/db/6c/a1f71542c969912bb0e106f64f60a56cc1f0fabecf9396f45accbe63fa68/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:187058ab80b3281b1de11c2e6842a357a1f71b42cb1e15bce373f3d238135c27", size = 2495627, upload-time = "2025-05-17T17:20:47.139Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/6e/4e/a066527e079fc5002390c8acdd3aca431e6ea0a50ffd7201551175b47323/pycryptodome-3.23.0-cp37-abi3-macosx_10_9_x86_64.whl", hash = "sha256:cfb5cd445280c5b0a4e6187a7ce8de5a07b5f3f897f235caa11f1f435f182843", size = 1640362, upload-time = "2025-05-17T17:20:50.392Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/50/52/adaf4c8c100a8c49d2bd058e5b551f73dfd8cb89eb4911e25a0c469b6b4e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:67bd81fcbe34f43ad9422ee8fd4843c8e7198dd88dd3d40e6de42ee65fbe1490", size = 2182625, upload-time = "2025-05-17T17:20:52.866Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/5f/e9/a09476d436d0ff1402ac3867d933c61805ec2326c6ea557aeeac3825604e/pycryptodome-3.23.0-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c8987bd3307a39bc03df5c8e0e3d8be0c4c3518b7f044b0f4c15d1aa78f52575", size = 2268954, upload-time = "2025-05-17T17:20:55.027Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/f9/c5/ffe6474e0c551d54cab931918127c46d70cab8f114e0c2b5a3c071c2f484/pycryptodome-3.23.0-cp37-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aa0698f65e5b570426fc31b8162ed4603b0c2841cbb9088e2b01641e3065915b", size = 2308534, upload-time = "2025-05-17T17:20:57.279Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/18/28/e199677fc15ecf43010f2463fde4c1a53015d1fe95fb03bca2890836603a/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:53ecbafc2b55353edcebd64bf5da94a2a2cdf5090a6915bcca6eca6cc452585a", size = 2181853, upload-time = "2025-05-17T17:20:59.322Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ce/ea/4fdb09f2165ce1365c9eaefef36625583371ee514db58dc9b65d3a255c4c/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_i686.whl", hash = "sha256:156df9667ad9f2ad26255926524e1c136d6664b741547deb0a86a9acf5ea631f", size = 2342465, upload-time = "2025-05-17T17:21:03.83Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/22/82/6edc3fc42fe9284aead511394bac167693fb2b0e0395b28b8bedaa07ef04/pycryptodome-3.23.0-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:dea827b4d55ee390dc89b2afe5927d4308a8b538ae91d9c6f7a5090f397af1aa", size = 2267414, upload-time = "2025-05-17T17:21:06.72Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/59/fe/aae679b64363eb78326c7fdc9d06ec3de18bac68be4b612fc1fe8902693c/pycryptodome-3.23.0-cp37-abi3-win32.whl", hash = "sha256:507dbead45474b62b2bbe318eb1c4c8ee641077532067fec9c1aa82c31f84886", size = 1768484, upload-time = "2025-05-17T17:21:08.535Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/54/2f/e97a1b8294db0daaa87012c24a7bb714147c7ade7656973fd6c736b484ff/pycryptodome-3.23.0-cp37-abi3-win_amd64.whl", hash = "sha256:c75b52aacc6c0c260f204cbdd834f76edc9fb0d8e0da9fbf8352ef58202564e2", size = 1799636, upload-time = "2025-05-17T17:21:10.393Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/18/3d/f9441a0d798bf2b1e645adc3265e55706aead1255ccdad3856dbdcffec14/pycryptodome-3.23.0-cp37-abi3-win_arm64.whl", hash = "sha256:11eeeb6917903876f134b56ba11abe95c0b0fd5e3330def218083c7d98bbcb3c", size = 1703675, upload-time = "2025-05-17T17:21:13.146Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pygments"
|
||||
version = "2.21.0"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "pytest"
|
||||
version = "9.1.1"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
dependencies = [
|
||||
{ name = "colorama", marker = "sys_platform == 'win32'" },
|
||||
{ name = "iniconfig" },
|
||||
{ name = "packaging" },
|
||||
{ name = "pluggy" },
|
||||
{ name = "pygments" },
|
||||
]
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ruff"
|
||||
version = "0.16.9"
|
||||
source = { registry = "https://pypi.org/simple" }
|
||||
sdist = { url = "https://files.pythonhosted.org/packages/96/bf/c935ca98e73fe8ce65b87ef08a280c0c1e85295d569228c15e87d8fdfaf1/ruff-0.16.9.tar.gz", hash = "sha256:12b625c6cfba78d285d9f48eda5f053374f1e53cb10ef17342a383750db99161", size = 4948764, upload-time = "2026-09-24T20:37:49.416Z" }
|
||||
wheels = [
|
||||
{ url = "https://files.pythonhosted.org/packages/0d/26/df51322b52ee1ada7eff2d071ea09d11d5c2d1dcc9f02594f5785c4d1635/ruff-0.16.9-py3-none-linux_armv6l.whl", hash = "sha256:95e6f022090368ab3b824c36276839c53b2adf1a3f4c09fefc33dfc400f6da96", size = 10082922, upload-time = "2026-09-24T20:37:13.045Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/a5/27/7bf51f5a7aa375e9f339280a303aab44ca75dc1525f1cdc5991761685b0f/ruff-0.16.9-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:a5f27be168556594a86d2f415db0cf43f5291917849318f873c7e2791f7a8c67", size = 10236360, upload-time = "2026-09-24T20:37:16.049Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/b6/63/09659283f92f02dff45809da194a70da2f688d87c55d8875c4fae3536072/ruff-0.16.9-py3-none-macosx_11_0_arm64.whl", hash = "sha256:1632eb1d6197f33bd00b1acbc5b71009e89a8895c158e2d2b03a834fac964ab6", size = 9892940, upload-time = "2026-09-24T20:37:17.957Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/24/58/98de1b72ec172f5f8f1731236fe21585b3998bf7dfe9fcc44ae9ba626012/ruff-0.16.9-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b3f951b14d865d5952c89d40a5ca07e87abe24fa5453299878411e127748fb1c", size = 10032114, upload-time = "2026-09-24T20:37:19.942Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/7d/f1e17c54ab59d4bad1dce8ee3e22a7a1d0ef4745240decacdcf3832b5bb2/ruff-0.16.9-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:447fc07e1573afff7cb02803462b12b6c8ece7cf10e2cd78565fa6d7a1c0bf8d", size = 9910227, upload-time = "2026-09-24T20:37:21.872Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/34/19/436f647a65075bbd3bab2668b3bdaa5120559b294694018cdcefabbbf30b/ruff-0.16.9-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8a3e039a6a40ed976c491722b60e0ae4a4aa1a86057f540ee7a37a5d19ae9120", size = 10547484, upload-time = "2026-09-24T20:37:24.229Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/03/59/38430a6bc2f6d8095447ac39625cf8b6e9344a47e6c26225c8ba1bff3ffb/ruff-0.16.9-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:4684dded7db60aa57cb118fa158630f5feade4af5782903b6053484bdf9bd129", size = 11412367, upload-time = "2026-09-24T20:37:26.307Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/c8/bd/bbb6d7fc7f208c8b8c50dd5dc8206e4cfdb1e7a8fb852606a3adf370c880/ruff-0.16.9-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d29c934357e45642fda2f34c0b1f4025b4a6c01e15e4bf0016879d60078a142c", size = 10869787, upload-time = "2026-09-24T20:37:28.35Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/bc/b8/9c543074918061abbefc3bd139bee22de35abedb00dde0f2d27288838962/ruff-0.16.9-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a21713e629d3e5bdb2f5c2def1cc7f04f47fa8e1a7eb0571b4a28e1da64bc728", size = 10406494, upload-time = "2026-09-24T20:37:30.624Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/35/7a/5a8851bd146e7ccf8fd4b003f6c75c11f8fbdb0e60673b45097986b6bf41/ruff-0.16.9-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:7baa24ef5fc8e77aa93879e1d3f43754a01ae488e869f1ae30cf431afd4d2452", size = 10590083, upload-time = "2026-09-24T20:37:32.439Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/87/f0/4c3467188f23f806960b46fa76575a7cd0514c9ba90562650b71efc96980/ruff-0.16.9-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:a41aac6230aadfaa133bdfa1614488531ffa3e0837567ae04c0da2058a9c0f9e", size = 10119151, upload-time = "2026-09-24T20:37:34.581Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/15/34/5a4def5adea572ce6aea0bb64f21f928ee317b80e0db747d7979b01d7261/ruff-0.16.9-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:c2529fb5896d49115b0e9aa8f887490b34bbe76baf879ec2264ac59406869ce7", size = 9911544, upload-time = "2026-09-24T20:37:36.796Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/62/5d/d15ebea7499eef9373318c0ee6ca127832927c6529731f6d48e18dce7ca9/ruff-0.16.9-py3-none-musllinux_1_2_i686.whl", hash = "sha256:41e3870277694177429b56406d65dfbdb2c2802c52b715edaf6a0b829c69d4ee", size = 10269884, upload-time = "2026-09-24T20:37:38.857Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/d1/56/c5d3cd119ded7a3c7aba0e961b69cb3df701c91662c98ad694467d060ce1/ruff-0.16.9-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:8adbe4e58af167f767d7b2ba5e83c42e878350796cf78c2f5e14ab9903a92588", size = 10749366, upload-time = "2026-09-24T20:37:41.042Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/ac/fe/734ec7527029ac757ecf821f143c9f3fcf69149c21f53a044a899b430f5a/ruff-0.16.9-py3-none-win32.whl", hash = "sha256:0e1dbc2073624dee6618d41d0098690a7244654af746704b64759e12b6b6b385", size = 10152355, upload-time = "2026-09-24T20:37:43.025Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/14/21/26e4643629b3ebb44f0a06f9c9a53058d63d989415f63a9a3c28e2ee7f22/ruff-0.16.9-py3-none-win_amd64.whl", hash = "sha256:6bd40fec8cd4c8a3d4dd589bd8ad4e6320c13c29234159bfd959a40d529d597b", size = 10592965, upload-time = "2026-09-24T20:37:44.944Z" },
|
||||
{ url = "https://files.pythonhosted.org/packages/51/60/5fb1a39dbb5ae314d5f59bc7348a63c1d5c20f3cd83914c4b5cb0be31d2d/ruff-0.16.9-py3-none-win_arm64.whl", hash = "sha256:ed1a252039200f57a59eebc063b54beabea67bfbaaca0eeaa7f54b5fbcda2284", size = 10458649, upload-time = "2026-09-24T20:37:46.882Z" },
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "snrc-resolve"
|
||||
version = "0.1.0"
|
||||
source = { virtual = "." }
|
||||
dependencies = [
|
||||
{ name = "eth-hash", extra = ["pycryptodome"] },
|
||||
]
|
||||
|
||||
[package.dev-dependencies]
|
||||
dev = [
|
||||
{ name = "pytest" },
|
||||
{ name = "ruff" },
|
||||
]
|
||||
|
||||
[package.metadata]
|
||||
requires-dist = [{ name = "eth-hash", extras = ["pycryptodome"], specifier = ">=0.7" }]
|
||||
|
||||
[package.metadata.requires-dev]
|
||||
dev = [
|
||||
{ name = "pytest", specifier = ">=8" },
|
||||
{ name = "ruff", specifier = ">=0.6" },
|
||||
]
|
||||
+1
-1
@@ -1,7 +1,7 @@
|
||||
cabal-version: 3.0
|
||||
|
||||
name: simplexmq
|
||||
version: 7.1.0.9
|
||||
version: 7.1.0.10
|
||||
synopsis: SimpleXMQ message broker
|
||||
description: This package includes <./docs/Simplex-Messaging-Server.html server>,
|
||||
<./docs/Simplex-Messaging-Client.html client> and
|
||||
|
||||
@@ -25,7 +25,6 @@ import Control.Monad.Reader
|
||||
import Control.Monad.Trans.Except
|
||||
import qualified Data.Aeson as J
|
||||
import Data.Bifunctor (first)
|
||||
import qualified Data.ByteString.Base64.URL as B64
|
||||
import Data.ByteString.Builder (Builder, byteString)
|
||||
import Data.ByteString.Char8 (ByteString)
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
@@ -594,7 +593,7 @@ processXFTPRequest ent HTTP2Body {bodyPart} = \case
|
||||
\used -> let used' = used + fromIntegral size in if used' <= quota then (True, used') else (False, used)
|
||||
receive = do
|
||||
path <- asks $ filesPath . config
|
||||
let fPath = path </> B.unpack (B64.encode $ unEntityId senderId)
|
||||
let fPath = path </> serverFileName senderId
|
||||
receiveChunk (XFTPRcvChunkSpec fPath size digest) >>= \case
|
||||
Right () -> do
|
||||
stats <- asks serverStats
|
||||
|
||||
@@ -17,6 +17,7 @@ module Simplex.FileTransfer.Server.StoreLog
|
||||
logDeleteFile,
|
||||
logBlockFile,
|
||||
logAckFile,
|
||||
serverFileName,
|
||||
)
|
||||
where
|
||||
|
||||
@@ -24,6 +25,7 @@ import Control.Applicative (optional, (<|>))
|
||||
import Control.Concurrent.STM
|
||||
import Control.Monad.Except
|
||||
import qualified Data.Attoparsec.ByteString.Char8 as A
|
||||
import qualified Data.ByteString.Base64.URL as B64
|
||||
import qualified Data.ByteString.Char8 as B
|
||||
import qualified Data.ByteString.Lazy.Char8 as LB
|
||||
import Data.Composition ((.:), (.::.))
|
||||
@@ -35,10 +37,11 @@ import Simplex.FileTransfer.Protocol (FileInfo (..))
|
||||
import Simplex.FileTransfer.Server.Store
|
||||
import Simplex.FileTransfer.Transport (XFTPErrorType (..))
|
||||
import Simplex.Messaging.Encoding.String
|
||||
import Simplex.Messaging.Protocol (BlockingInfo, RcvPublicAuthKey, RecipientId, SenderId)
|
||||
import Simplex.Messaging.Protocol (BlockingInfo, EntityId (..), RcvPublicAuthKey, RecipientId, SenderId)
|
||||
import Simplex.Messaging.Server.QueueStore (ServerEntityStatus (..))
|
||||
import Simplex.Messaging.Server.StoreLog
|
||||
import Simplex.Messaging.Util (bshow)
|
||||
import System.FilePath (takeFileName)
|
||||
import System.IO
|
||||
|
||||
data FileStoreLogRecord
|
||||
@@ -99,6 +102,9 @@ logBlockFile s fId = logFileStoreRecord s . BlockFile fId
|
||||
logAckFile :: StoreLog 'WriteMode -> RecipientId -> IO ()
|
||||
logAckFile s = logFileStoreRecord s . AckFile
|
||||
|
||||
serverFileName :: SenderId -> FilePath
|
||||
serverFileName = B.unpack . B64.encode . unEntityId
|
||||
|
||||
readWriteFileStore :: FilePath -> STMFileStore -> IO (StoreLog 'WriteMode)
|
||||
readWriteFileStore = readWriteStoreLog readFileStore writeFileStore
|
||||
|
||||
@@ -115,7 +121,9 @@ readFileStore f st = mapM_ (addFileLogRecord . LB.toStrict) . LB.lines =<< LB.re
|
||||
AddFile sId file createdAt expiresAt status
|
||||
| size file > 0 -> addFile st sId file createdAt expiresAt status
|
||||
| otherwise -> pure $ Left SIZE
|
||||
PutFile qId path -> setFilePath st qId path
|
||||
PutFile qId path
|
||||
| takeFileName path == serverFileName qId -> setFilePath st qId path
|
||||
| otherwise -> pure $ Left AUTH
|
||||
AddRecipients sId rcps -> runExceptT $ addRecipients sId rcps
|
||||
DeleteFile sId -> deleteFile st sId
|
||||
BlockFile sId info -> blockFile st sId info True
|
||||
|
||||
@@ -4388,8 +4388,9 @@ agentRatchetDecrypt' :: TVar ChaChaDRG -> DB.Connection -> ConnId -> CR.RatchetX
|
||||
agentRatchetDecrypt' g db connId rc encAgentMsg = do
|
||||
skipped <- liftIO $ getSkippedMsgKeys db connId
|
||||
(agentMsgBody_, rc', skippedDiff) <- withExceptT (SEAgentError . cryptoError) $ CR.rcDecrypt g rc skipped encAgentMsg
|
||||
agentMsgBody <- liftEither $ first (SEAgentError . cryptoError) agentMsgBody_
|
||||
liftIO $ updateRatchet db connId rc' skippedDiff
|
||||
liftEither $ bimap (SEAgentError . cryptoError) (,CR.rcRcvKEM rc') agentMsgBody_
|
||||
pure (agentMsgBody, CR.rcRcvKEM rc')
|
||||
|
||||
newSndQueue :: UserId -> ConnId -> Compatible SMPQueueInfo -> Maybe (C.APrivateAuthKey) -> AM' (NewSndQueue, C.PublicKeyX25519)
|
||||
newSndQueue userId connId (Compatible (SMPQueueInfo smpClientVersion SMPQueueAddress {smpServer, senderId, queueMode, dhPublicKey = rcvE2ePubDhKey})) sndKey_ = do
|
||||
|
||||
@@ -196,6 +196,7 @@ data PClient v err msg = PClient
|
||||
transportHost :: TransportHost,
|
||||
tcpConnectTimeout :: NetworkTimeout,
|
||||
tcpTimeout :: NetworkTimeout,
|
||||
proxiedRelayVRange :: VersionRange v,
|
||||
sendPings :: TVar Bool,
|
||||
lastReceived :: TVar UTCTime,
|
||||
timeoutErrorCount :: TVar Int,
|
||||
@@ -241,6 +242,7 @@ smpClientStub g sessionId thVersion thAuth = do
|
||||
transportHost = "localhost",
|
||||
tcpConnectTimeout,
|
||||
tcpTimeout,
|
||||
proxiedRelayVRange = supportedClientSMPRelayVRange,
|
||||
sendPings,
|
||||
lastReceived,
|
||||
timeoutErrorCount,
|
||||
@@ -482,6 +484,7 @@ data ProtocolClientConfig v = ProtocolClientConfig
|
||||
serviceCredentials :: Maybe ServiceCredentials,
|
||||
-- | client-server protocol version range
|
||||
serverVRange :: VersionRange v,
|
||||
proxiedRelayVRange :: VersionRange v,
|
||||
-- | agree shared session secret (used in SMP proxy for additional encryption layer)
|
||||
agreeSecret :: Bool,
|
||||
-- | Whether connecting client is a proxy server. See comment in ClientHandshake
|
||||
@@ -500,6 +503,7 @@ defaultClientConfig clientALPN useSNI serverVRange =
|
||||
clientALPN,
|
||||
serviceCredentials = Nothing,
|
||||
serverVRange,
|
||||
proxiedRelayVRange = serverVRange,
|
||||
agreeSecret = False,
|
||||
proxyServer = False,
|
||||
useSNI
|
||||
@@ -510,6 +514,7 @@ defaultSMPClientConfig :: ProtocolClientConfig SMPVersion
|
||||
defaultSMPClientConfig =
|
||||
(defaultClientConfig (Just alpnSupportedSMPHandshakes) False supportedClientSMPRelayVRange)
|
||||
{ defaultTransport = (show defaultSMPPort, transport @TLS),
|
||||
proxiedRelayVRange = supportedClientSMPRelayVRange,
|
||||
agreeSecret = True
|
||||
}
|
||||
{-# INLINE defaultSMPClientConfig #-}
|
||||
@@ -573,7 +578,7 @@ type SMPTransportSession = TransportSession BrokerMsg
|
||||
-- A single queue can be used for multiple 'SMPClient' instances,
|
||||
-- as 'SMPServerTransmission' includes server information.
|
||||
getProtocolClient :: forall v err msg. Protocol v err msg => TVar ChaChaDRG -> NetworkRequestMode -> TransportSession msg -> ProtocolClientConfig v -> [HostName] -> Maybe (TBQueue (ServerTransmissionBatch v err msg)) -> UTCTime -> (ProtocolClient v err msg -> IO ()) -> IO (Either (ProtocolClientError err) (ProtocolClient v err msg))
|
||||
getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qSize, networkConfig, clientALPN, serviceCredentials, serverVRange, agreeSecret, proxyServer, useSNI} presetDomains msgQ proxySessTs disconnected = do
|
||||
getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qSize, networkConfig, clientALPN, serviceCredentials, serverVRange, proxiedRelayVRange, agreeSecret, proxyServer, useSNI} presetDomains msgQ proxySessTs disconnected = do
|
||||
case chooseTransportHost networkConfig (host srv) of
|
||||
Right useHost ->
|
||||
(getCurrentTime >>= mkProtocolClient useHost >>= runClient useTransport useHost)
|
||||
@@ -598,6 +603,7 @@ getProtocolClient g nm transportSession@(_, srv, _) cfg@ProtocolClientConfig {qS
|
||||
transportHost,
|
||||
tcpConnectTimeout,
|
||||
tcpTimeout,
|
||||
proxiedRelayVRange,
|
||||
sendPings,
|
||||
lastReceived,
|
||||
timeoutErrorCount,
|
||||
@@ -1120,10 +1126,10 @@ deleteSMPQueues = okSMPCommands DEL
|
||||
-- send PRXY :: SMPServer -> Maybe BasicAuth -> Command Sender
|
||||
-- receives PKEY :: SessionId -> X.CertificateChain -> X.SignedExact X.PubKey -> BrokerMsg
|
||||
connectSMPProxiedRelay :: SMPClient -> NetworkRequestMode -> SMPServer -> Maybe BasicAuth -> ExceptT SMPClientError IO ProxiedRelay
|
||||
connectSMPProxiedRelay c@ProtocolClient {client_ = PClient {tcpConnectTimeout, tcpTimeout}} nm relayServ@ProtocolServer {port = relayPort, keyHash = C.KeyHash kh} proxyAuth =
|
||||
connectSMPProxiedRelay c@ProtocolClient {client_ = PClient {tcpConnectTimeout, tcpTimeout, proxiedRelayVRange}} nm relayServ@ProtocolServer {port = relayPort, keyHash = C.KeyHash kh} proxyAuth =
|
||||
sendProtocolCommand_ c nm Nothing tOut Nothing NoEntity (Cmd SProxiedClient (PRXY relayServ proxyAuth)) >>= \case
|
||||
PKEY sId vr (CertChainPubKey chain key) ->
|
||||
case supportedClientSMPRelayVRange `compatibleVersion` vr of
|
||||
case proxiedRelayVRange `compatibleVersion` vr of
|
||||
Nothing -> throwE $ transportErr TEVersion
|
||||
Just (Compatible v) -> do
|
||||
relayKey <- liftEitherWith (const $ transportErr $ TEHandshake IDENTITY) =<< liftIO (runExceptT $ validateRelay chain key)
|
||||
@@ -1174,7 +1180,7 @@ instance StrEncoding ProxyClientError where
|
||||
-- consider how to process slow responses - is it handled somehow locally or delegated to the caller
|
||||
-- this method is used in the client
|
||||
-- sends PFWD :: C.PublicKeyX25519 -> EncTransmission -> Command Sender
|
||||
-- receives PRES :: EncResponse -> BrokerMsg -- proxy to client
|
||||
-- receives PRES :: Maybe C.CbNonce -> EncResponse -> BrokerMsg -- proxy to client
|
||||
|
||||
-- When client sends message via proxy, there may be one successful scenario and 9 error scenarios
|
||||
-- as shown below (WTF stands for unexpected response, ??? for response that failed to parse).
|
||||
@@ -1233,14 +1239,14 @@ proxySMPCommand c@ProtocolClient {thParams = proxyThParams, client_ = PClient {c
|
||||
TBError e _ : _ -> throwE $ PCETransportError e
|
||||
TBTransmission s _ : _ -> pure s
|
||||
TBTransmissions s _ _ : _ -> pure s
|
||||
et <- liftEitherWith PCECryptoError $ EncTransmission <$> C.cbEncrypt cmdSecret nonce b paddedProxiedTLength
|
||||
et <- liftEitherWith PCECryptoError $ EncTransmission <$> C.cbEncrypt cmdSecret (encTransmissionNonce v nonce) b paddedProxiedTLength
|
||||
-- proxy interaction errors are wrapped
|
||||
let tOut = Just $ 2 * netTimeoutInt tcpTimeout nm
|
||||
tryE (sendProtocolCommand_ c nm (Just nonce) tOut Nothing (EntityId sessionId) (Cmd SProxiedClient (PFWD v cmdPubKey et))) >>= \case
|
||||
Right r -> case r of
|
||||
PRES (EncResponse er) -> do
|
||||
PRES nonce_ (EncResponse er) -> do
|
||||
-- server interaction errors are thrown directly
|
||||
t' <- liftEitherWith PCECryptoError $ C.cbDecrypt cmdSecret (C.reverseNonce nonce) er
|
||||
t' <- liftEitherWith PCECryptoError $ C.cbDecrypt cmdSecret (fromMaybe (C.reverseNonce nonce) nonce_) er
|
||||
case tParse serverThParams t' of
|
||||
t'' :| [] -> case tDecodeClient serverThParams t'' of
|
||||
(_, _, cmd) -> case cmd of
|
||||
@@ -1258,10 +1264,10 @@ proxySMPCommand c@ProtocolClient {thParams = proxyThParams, client_ = PClient {c
|
||||
|
||||
-- this method is used in the proxy
|
||||
-- sends RFWD :: EncFwdTransmission -> Command Sender
|
||||
-- receives RRES :: EncFwdResponse -> BrokerMsg
|
||||
-- receives RRES :: Maybe C.CbNonce -> EncFwdResponse -> BrokerMsg
|
||||
-- proxy should send PRES to the client with EncResponse
|
||||
-- Always uses background timeout mode
|
||||
forwardSMPTransmission :: SMPClient -> CorrId -> VersionSMP -> C.PublicKeyX25519 -> EncTransmission -> ExceptT SMPClientError IO EncResponse
|
||||
forwardSMPTransmission :: SMPClient -> CorrId -> VersionSMP -> C.PublicKeyX25519 -> EncTransmission -> ExceptT SMPClientError IO (Maybe C.CbNonce, EncResponse)
|
||||
forwardSMPTransmission c@ProtocolClient {thParams, client_ = PClient {clientCorrId = g}} fwdCorrId fwdVersion fwdKey fwdTransmission = do
|
||||
-- prepare params
|
||||
sessSecret <- case thAuth thParams of
|
||||
@@ -1273,11 +1279,11 @@ forwardSMPTransmission c@ProtocolClient {thParams, client_ = PClient {clientCorr
|
||||
eft = EncFwdTransmission $ C.cbEncryptNoPad sessSecret nonce (smpEncode fwdT)
|
||||
-- send
|
||||
sendProtocolCommand_ c NRMBackground (Just nonce) Nothing Nothing NoEntity (Cmd SProxyService (RFWD eft)) >>= \case
|
||||
RRES (EncFwdResponse efr) -> do
|
||||
RRES nonce_ (EncFwdResponse efr) -> do
|
||||
-- unwrap
|
||||
r' <- liftEitherWith PCECryptoError $ C.cbDecryptNoPad sessSecret (C.reverseNonce nonce) efr
|
||||
FwdResponse {fwdCorrId = _, fwdResponse} <- liftEitherWith (const $ PCEResponseError BLOCK) $ smpDecode r'
|
||||
pure fwdResponse
|
||||
pure (nonce_, fwdResponse)
|
||||
r -> throwE $ unexpectedResponse r
|
||||
|
||||
-- get queue information - always sent interactively
|
||||
|
||||
@@ -168,6 +168,7 @@ module Simplex.Messaging.Protocol
|
||||
EncFwdTransmission (..),
|
||||
EncResponse (..),
|
||||
EncTransmission (..),
|
||||
encTransmissionNonce,
|
||||
FwdResponse (..),
|
||||
FwdTransmission (..),
|
||||
NameRecord (..),
|
||||
@@ -241,6 +242,7 @@ import Data.Attoparsec.ByteString.Char8 (Parser, (<?>))
|
||||
import qualified Data.Attoparsec.ByteString.Char8 as A
|
||||
import Data.Bifunctor (bimap, first)
|
||||
import Data.Bits (xor)
|
||||
import qualified Data.ByteArray as BA
|
||||
import qualified Data.ByteString as BS
|
||||
import qualified Data.ByteString.Base64 as B64
|
||||
import Data.ByteString.Char8 (ByteString)
|
||||
@@ -279,7 +281,7 @@ import Simplex.Messaging.ServiceScheme
|
||||
import Simplex.Messaging.SimplexName (LabelHash, SimplexDomain (..), SimplexTLD (..), fullDomainName, labelHash)
|
||||
import Simplex.Messaging.Transport
|
||||
import Simplex.Messaging.Transport.Client (TransportHost, TransportHosts (..))
|
||||
import Simplex.Messaging.Util (bshow, eitherToMaybe, safeDecodeUtf8, (<$?>))
|
||||
import Simplex.Messaging.Util (bshow, eitherToMaybe, packZipWith, safeDecodeUtf8, (<$?>))
|
||||
import Simplex.Messaging.Version
|
||||
import Simplex.Messaging.Version.Internal
|
||||
|
||||
@@ -701,6 +703,11 @@ instance Encoding NewNtfCreds where
|
||||
newtype EncTransmission = EncTransmission ByteString
|
||||
deriving (Show)
|
||||
|
||||
encTransmissionNonce :: VersionSMP -> C.CbNonce -> C.CbNonce
|
||||
encTransmissionNonce v nonce@(C.CbNonce s)
|
||||
| v >= fwdNoncesSMPVersion = C.cbNonce $ packZipWith xor (smpEncode v) s <> BS.drop 2 s
|
||||
| otherwise = nonce
|
||||
|
||||
data FwdTransmission = FwdTransmission
|
||||
{ fwdCorrId :: CorrId,
|
||||
fwdVersion :: VersionSMP,
|
||||
@@ -736,8 +743,8 @@ data BrokerMsg where
|
||||
NMSG :: C.CbNonce -> EncNMsgMeta -> BrokerMsg
|
||||
-- Should include certificate chain
|
||||
PKEY :: SessionId -> VersionRangeSMP -> CertChainPubKey -> BrokerMsg -- TLS-signed server key for proxy shared secret and initial sender key
|
||||
RRES :: EncFwdResponse -> BrokerMsg -- relay to proxy
|
||||
PRES :: EncResponse -> BrokerMsg -- proxy to client
|
||||
RRES :: Maybe C.CbNonce -> EncFwdResponse -> BrokerMsg -- relay to proxy
|
||||
PRES :: Maybe C.CbNonce -> EncResponse -> BrokerMsg -- proxy to client
|
||||
END :: BrokerMsg
|
||||
ENDS :: Int64 -> IdsHash -> BrokerMsg
|
||||
DELD :: BrokerMsg
|
||||
@@ -1323,7 +1330,10 @@ instance ProtocolTypeI p => FromJSON (ProtocolServer p) where
|
||||
parseJSON = strParseJSON "ProtocolServer"
|
||||
|
||||
newtype BasicAuth = BasicAuth {unBasicAuth :: ByteString}
|
||||
deriving (Eq, Ord, Show)
|
||||
deriving (Ord, Show)
|
||||
|
||||
instance Eq BasicAuth where
|
||||
BasicAuth s == BasicAuth s' = BA.constEq s s'
|
||||
|
||||
instance IsString BasicAuth where fromString = BasicAuth . B.pack
|
||||
|
||||
@@ -1981,8 +1991,8 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
|
||||
NID nId srvNtfDh -> e (NID_, ' ', nId, srvNtfDh)
|
||||
NMSG nmsgNonce encNMsgMeta -> e (NMSG_, ' ', nmsgNonce, encNMsgMeta)
|
||||
PKEY sid vr certKey -> e (PKEY_, ' ', sid, vr, certKey)
|
||||
RRES (EncFwdResponse encBlock) -> e (RRES_, ' ', Tail encBlock)
|
||||
PRES (EncResponse encBlock) -> e (PRES_, ' ', Tail encBlock)
|
||||
RRES nonce_ (EncFwdResponse encBlock) -> fwdResp RRES_ nonce_ encBlock
|
||||
PRES nonce_ (EncResponse encBlock) -> fwdResp PRES_ nonce_ encBlock
|
||||
END -> e END_
|
||||
ENDS n idsHash -> serviceResp ENDS_ n idsHash
|
||||
DELD -> e DELD_
|
||||
@@ -2006,6 +2016,9 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
|
||||
serviceResp tag n idsHash
|
||||
| v >= rcvServiceSMPVersion = e (tag, ' ', n, idsHash)
|
||||
| otherwise = e (tag, ' ', n)
|
||||
fwdResp tag nonce_ encBlock
|
||||
| v >= fwdNoncesSMPVersion = e (tag, ' ', nonce_, Tail encBlock)
|
||||
| otherwise = e (tag, ' ', Tail encBlock)
|
||||
|
||||
protocolP v = \case
|
||||
MSG_ -> do
|
||||
@@ -2037,8 +2050,8 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
|
||||
NID_ -> NID <$> _smpP <*> smpP
|
||||
NMSG_ -> NMSG <$> _smpP <*> smpP
|
||||
PKEY_ -> PKEY <$> _smpP <*> smpP <*> smpP
|
||||
RRES_ -> RRES <$> (EncFwdResponse . unTail <$> _smpP)
|
||||
PRES_ -> PRES <$> (EncResponse . unTail <$> _smpP)
|
||||
RRES_ -> fwdRespP RRES EncFwdResponse
|
||||
PRES_ -> fwdRespP PRES EncResponse
|
||||
END_ -> pure END
|
||||
ENDS_ -> serviceRespP ENDS
|
||||
DELD_ -> pure DELD
|
||||
@@ -2055,6 +2068,9 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
|
||||
serviceRespP resp
|
||||
| v >= rcvServiceSMPVersion = resp <$> _smpP <*> smpP
|
||||
| otherwise = resp <$> _smpP <*> pure mempty
|
||||
fwdRespP :: (Maybe C.CbNonce -> a -> BrokerMsg) -> (ByteString -> a) -> Parser BrokerMsg
|
||||
fwdRespP resp enc = resp <$> (A.space *> nonceP) <*> (enc <$> A.takeByteString)
|
||||
nonceP = if v >= fwdNoncesSMPVersion then smpP else pure Nothing
|
||||
|
||||
fromProtocolError = \case
|
||||
PECmdSyntax -> CMD SYNTAX
|
||||
@@ -2071,7 +2087,7 @@ instance ProtocolEncoding SMPVersion ErrorType BrokerMsg where
|
||||
-- PONG response must not have queue ID
|
||||
PONG -> noEntityMsg
|
||||
PKEY {} -> noEntityMsg
|
||||
RRES _ -> noEntityMsg
|
||||
RRES {} -> noEntityMsg
|
||||
ALLS -> noEntityMsg
|
||||
RNAME {} -> noEntityMsg
|
||||
-- other broker responses must have queue ID
|
||||
|
||||
@@ -1092,6 +1092,7 @@ controlPortAuth h user admin role auth = do
|
||||
readTVarIO role >>= \case
|
||||
CPRNone -> do
|
||||
atomically $ writeTVar role $! newRole
|
||||
when (newRole == CPRNone) $ logWarn "ControlPort: failed auth"
|
||||
hPutStrLn h $ currentRole newRole
|
||||
r -> hPutStrLn h $ currentRole r <> if r == newRole then "" else ", start new session to change."
|
||||
where
|
||||
@@ -1461,7 +1462,7 @@ client
|
||||
inc own pRequests
|
||||
forkProxiedCmd $ do
|
||||
liftIO (runExceptT (forwardSMPTransmission smp corrId fwdV pubKey encBlock) `E.catches` clientHandlers) >>= \case
|
||||
Right r -> PRES r <$ inc own pSuccesses
|
||||
Right (nonce_, r) -> PRES nonce_ r <$ inc own pSuccesses
|
||||
Left e -> ERR (smpProxyError e) <$ case e of
|
||||
PCEProtocolError {} -> inc own pSuccesses
|
||||
_ -> inc own pErrorsOther
|
||||
@@ -2130,7 +2131,7 @@ client
|
||||
unless (fwdVersion `isCompatible` thServerVRange thParams') $ throwE $ transportErr TEVersion
|
||||
let clientSecret = C.dh' fwdKey serverPrivKey
|
||||
clientNonce = C.cbNonce $ bs fwdCorrId
|
||||
b <- liftEitherWith (const CRYPTO) $ C.cbDecrypt clientSecret clientNonce et
|
||||
b <- liftEitherWith (const CRYPTO) $ C.cbDecrypt clientSecret (encTransmissionNonce fwdVersion clientNonce) et
|
||||
let clntTHParams = smpTHParamsSetVersion fwdVersion thParams'
|
||||
-- only allowing single forwarded transactions
|
||||
t' <- case tParse clntTHParams b of
|
||||
@@ -2143,9 +2144,13 @@ client
|
||||
TBError _ _ : _ -> throwE BLOCK
|
||||
TBTransmission b' _ : _ -> pure b'
|
||||
TBTransmissions b' _ _ : _ -> pure b'
|
||||
r2 <- liftEitherWith (const BLOCK) $ EncResponse <$> C.cbEncrypt clientSecret (C.reverseNonce clientNonce) r' paddedProxiedTLength
|
||||
nonce_ <-
|
||||
if fwdVersion >= fwdNoncesSMPVersion
|
||||
then Just <$> (atomically . C.randomCbNonce =<< asks random)
|
||||
else pure Nothing
|
||||
r2 <- liftEitherWith (const BLOCK) $ EncResponse <$> C.cbEncrypt clientSecret (fromMaybe (C.reverseNonce clientNonce) nonce_) r' paddedProxiedTLength
|
||||
let fr = FwdResponse {fwdCorrId, fwdResponse = r2}
|
||||
pure $ RRES $ EncFwdResponse $ C.cbEncryptNoPad sessSecret (C.reverseNonce proxyNonce) (smpEncode fr)
|
||||
pure $ RRES nonce_ $ EncFwdResponse $ C.cbEncryptNoPad sessSecret (C.reverseNonce proxyNonce) (smpEncode fr)
|
||||
-- the inner response, or Nothing if forked (RSLV).
|
||||
r_ <- lift (rejectOrVerify clntThAuth t') >>= \case
|
||||
-- rejectOrVerify filters allowed commands, no need to repeat it here.
|
||||
|
||||
@@ -114,7 +114,9 @@ instance MsgStoreClass PostgresMsgStore where
|
||||
where
|
||||
st = dbStore $ queueStore_ ms
|
||||
oldMsg = now - ttl
|
||||
batchSize = 10000 :: Int
|
||||
-- expired messages read per page in expire_old_messages, and the page is one
|
||||
-- transaction: queues in it stay row-locked against SEND and ACK until it commits.
|
||||
batchSize = 100 :: Int
|
||||
toMessageStats (expiredMsgsCount, storedMsgsCount, storedQueues) =
|
||||
MessageStats {expiredMsgsCount, storedMsgsCount, storedQueues}
|
||||
|
||||
@@ -363,8 +365,8 @@ deleteAllMessages ms =
|
||||
db
|
||||
[sql|
|
||||
UPDATE msg_queues
|
||||
SET msg_queue_size = 0, msg_can_write = TRUE, msg_queue_expire = FALSE
|
||||
WHERE msg_queue_size != 0 OR msg_can_write = FALSE OR msg_queue_expire = TRUE
|
||||
SET msg_queue_size = 0, msg_can_write = TRUE
|
||||
WHERE msg_queue_size != 0 OR msg_can_write = FALSE
|
||||
|]
|
||||
|
||||
updateQueueCounts :: PostgresMsgStore -> IO ()
|
||||
@@ -384,16 +386,15 @@ updateQueueCounts ms =
|
||||
db
|
||||
[sql|
|
||||
UPDATE msg_queues
|
||||
SET msg_queue_size = 0, msg_can_write = TRUE, msg_queue_expire = FALSE
|
||||
WHERE msg_queue_size != 0 OR msg_can_write = FALSE OR msg_queue_expire = TRUE
|
||||
SET msg_queue_size = 0, msg_can_write = TRUE
|
||||
WHERE msg_queue_size != 0 OR msg_can_write = FALSE
|
||||
|]
|
||||
void $ DB.execute_
|
||||
db
|
||||
[sql|
|
||||
UPDATE msg_queues q
|
||||
SET msg_queue_size = s.size,
|
||||
msg_can_write = s.quota_count = 0,
|
||||
msg_queue_expire = s.size > s.quota_count
|
||||
msg_can_write = s.quota_count = 0
|
||||
FROM queue_stats s
|
||||
WHERE q.recipient_id = s.recipient_id
|
||||
|]
|
||||
|
||||
@@ -22,7 +22,8 @@ serverSchemaMigrations =
|
||||
("20250903_store_messages", m20250903_store_messages, Just down_m20250903_store_messages),
|
||||
("20250915_queue_ids_hash", m20250915_queue_ids_hash, Just down_m20250915_queue_ids_hash),
|
||||
("20260916_prometheus_indexes", m20260916_prometheus_indexes, Just down_m20260916_prometheus_indexes),
|
||||
("20260917_msg_queues_hot", m20260917_msg_queues_hot, Just down_m20260917_msg_queues_hot)
|
||||
("20260917_msg_queues_hot", m20260917_msg_queues_hot, Just down_m20260917_msg_queues_hot),
|
||||
("20260918_expire_messages", m20260918_expire_messages, Just down_m20260918_expire_messages)
|
||||
]
|
||||
|
||||
-- | The list of migrations in ascending order by date
|
||||
@@ -734,3 +735,504 @@ ALTER TABLE msg_queues RESET (fillfactor, autovacuum_vacuum_scale_factor, autova
|
||||
ALTER TABLE messages RESET (autovacuum_vacuum_scale_factor, autovacuum_analyze_scale_factor, toast.autovacuum_vacuum_scale_factor);
|
||||
ALTER TABLE services RESET (fillfactor, autovacuum_vacuum_threshold, autovacuum_vacuum_scale_factor);
|
||||
|]
|
||||
|
||||
m20260918_expire_messages :: Text
|
||||
m20260918_expire_messages =
|
||||
[r|
|
||||
CREATE INDEX idx_messages_expire ON messages (msg_ts, recipient_id) WHERE NOT msg_quota;
|
||||
|
||||
DROP INDEX idx_messages_recipient_id_msg_ts;
|
||||
|
||||
DROP INDEX idx_msg_queues_expire;
|
||||
|
||||
DROP PROCEDURE expire_old_messages(bigint, integer);
|
||||
|
||||
CREATE PROCEDURE expire_old_messages(IN p_old_ts bigint, IN batch_size integer, OUT r_expired_msgs_count bigint, OUT r_stored_msgs_count bigint, OUT r_stored_queues bigint)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
rids BYTEA[];
|
||||
rid BYTEA;
|
||||
last_ts BIGINT := -1;
|
||||
last_rid BYTEA := '\x';
|
||||
next_ts BIGINT;
|
||||
next_rid BYTEA;
|
||||
del_count BIGINT;
|
||||
total_deleted BIGINT := 0;
|
||||
BEGIN
|
||||
LOOP
|
||||
-- The page is scanned in (msg_ts, recipient_id) order, so its last row is the next
|
||||
-- keyset cursor. Advancing it past every row read, including queues left unexpired
|
||||
-- because delete_expired_msgs skipped a locked row or raised, is what terminates the
|
||||
-- loop; re-reading from the start instead would repeat those queues forever.
|
||||
SELECT array_agg(DISTINCT recipient_id),
|
||||
(array_agg(msg_ts ORDER BY msg_ts DESC, recipient_id DESC))[1],
|
||||
(array_agg(recipient_id ORDER BY msg_ts DESC, recipient_id DESC))[1]
|
||||
INTO rids, next_ts, next_rid
|
||||
FROM (
|
||||
SELECT msg_ts, recipient_id
|
||||
FROM messages
|
||||
WHERE NOT msg_quota
|
||||
AND msg_ts < p_old_ts
|
||||
AND (msg_ts, recipient_id) > (last_ts, last_rid)
|
||||
ORDER BY msg_ts ASC, recipient_id ASC
|
||||
LIMIT batch_size
|
||||
) m;
|
||||
|
||||
EXIT WHEN rids IS NULL;
|
||||
|
||||
FOREACH rid IN ARRAY rids
|
||||
LOOP
|
||||
BEGIN
|
||||
del_count := delete_expired_msgs(rid, p_old_ts);
|
||||
total_deleted := total_deleted + del_count;
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
RAISE WARNING 'STORE, expire_old_messages, error expiring queue %: %', encode(rid, 'base64'), SQLERRM;
|
||||
CONTINUE;
|
||||
END;
|
||||
END LOOP;
|
||||
COMMIT;
|
||||
|
||||
last_ts := next_ts;
|
||||
last_rid := next_rid;
|
||||
END LOOP;
|
||||
|
||||
r_expired_msgs_count := total_deleted;
|
||||
r_stored_msgs_count := (SELECT COUNT(1) FROM messages);
|
||||
r_stored_queues := (SELECT COUNT(1) FROM msg_queues WHERE deleted_at IS NULL);
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION delete_expired_msgs(p_recipient_id bytea, p_old_ts bigint) RETURNS bigint
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
q_size BIGINT;
|
||||
keep_min_id BIGINT;
|
||||
del_count BIGINT;
|
||||
BEGIN
|
||||
SELECT msg_queue_size INTO q_size
|
||||
FROM msg_queues
|
||||
WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
|
||||
FOR UPDATE SKIP LOCKED;
|
||||
|
||||
IF NOT FOUND OR q_size = 0 THEN
|
||||
RETURN 0;
|
||||
END IF;
|
||||
|
||||
SELECT MIN(message_id) INTO keep_min_id
|
||||
FROM messages WHERE recipient_id = p_recipient_id AND msg_ts >= p_old_ts AND msg_quota = FALSE;
|
||||
|
||||
IF keep_min_id IS NULL THEN
|
||||
DELETE FROM messages WHERE recipient_id = p_recipient_id AND msg_quota = FALSE;
|
||||
ELSE
|
||||
DELETE FROM messages WHERE recipient_id = p_recipient_id AND message_id < keep_min_id AND msg_quota = FALSE;
|
||||
END IF;
|
||||
|
||||
GET DIAGNOSTICS del_count = ROW_COUNT;
|
||||
IF del_count > 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= del_count,
|
||||
msg_queue_size = GREATEST(msg_queue_size - del_count, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
RETURN del_count;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION write_message(p_recipient_id bytea, p_msg_id bytea, p_msg_ts bigint, p_msg_quota boolean, p_msg_ntf_flag boolean, p_msg_body bytea, p_quota integer) RETURNS TABLE(quota_written boolean, was_empty boolean)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
q_can_write BOOLEAN;
|
||||
q_size BIGINT;
|
||||
BEGIN
|
||||
SELECT msg_can_write, msg_queue_size INTO q_can_write, q_size
|
||||
FROM msg_queues
|
||||
WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
|
||||
FOR UPDATE;
|
||||
|
||||
IF q_can_write OR q_size = 0 THEN
|
||||
quota_written := p_msg_quota OR q_size >= p_quota;
|
||||
was_empty := q_size = 0;
|
||||
|
||||
INSERT INTO messages(recipient_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body)
|
||||
VALUES (p_recipient_id, p_msg_id, p_msg_ts, quota_written, p_msg_ntf_flag AND NOT quota_written, CASE WHEN quota_written THEN '' :: BYTEA ELSE p_msg_body END);
|
||||
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = NOT quota_written,
|
||||
msg_queue_size = msg_queue_size + 1
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
|
||||
RETURN QUERY VALUES (quota_written, was_empty);
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION try_del_msg(p_recipient_id bytea, p_msg_id bytea) RETURNS TABLE(r_msg_id bytea, r_msg_ts bigint, r_msg_quota boolean, r_msg_ntf_flag boolean, r_msg_body bytea)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
q_size BIGINT;
|
||||
msg RECORD;
|
||||
BEGIN
|
||||
SELECT msg_queue_size INTO q_size
|
||||
FROM msg_queues
|
||||
WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
|
||||
FOR UPDATE;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
SELECT message_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
|
||||
INTO msg
|
||||
FROM messages
|
||||
WHERE recipient_id = p_recipient_id
|
||||
ORDER BY message_id ASC LIMIT 1;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
IF q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF msg.msg_id = p_msg_id THEN
|
||||
DELETE FROM messages WHERE message_id = msg.message_id;
|
||||
IF FOUND THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
|
||||
msg_queue_size = GREATEST(msg_queue_size - 1, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
END IF;
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION try_del_peek_msg(p_recipient_id bytea, p_msg_id bytea) RETURNS TABLE(r_msg_id bytea, r_msg_ts bigint, r_msg_quota boolean, r_msg_ntf_flag boolean, r_msg_body bytea)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
q_size BIGINT;
|
||||
msg RECORD;
|
||||
msg_deleted BOOLEAN;
|
||||
BEGIN
|
||||
SELECT msg_queue_size INTO q_size
|
||||
FROM msg_queues
|
||||
WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
|
||||
FOR UPDATE;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
SELECT message_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
|
||||
INTO msg
|
||||
FROM messages
|
||||
WHERE recipient_id = p_recipient_id
|
||||
ORDER BY message_id ASC LIMIT 1;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
IF q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF msg.msg_id = p_msg_id THEN
|
||||
DELETE FROM messages WHERE message_id = msg.message_id;
|
||||
|
||||
msg_deleted := FOUND;
|
||||
IF msg_deleted THEN
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
END IF;
|
||||
|
||||
SELECT msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
|
||||
INTO msg
|
||||
FROM messages
|
||||
WHERE recipient_id = p_recipient_id
|
||||
ORDER BY message_id ASC LIMIT 1;
|
||||
|
||||
IF FOUND THEN
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
IF msg_deleted THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
|
||||
msg_queue_size = GREATEST(msg_queue_size - 1, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
ELSIF msg_deleted OR q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
ELSE
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
ALTER TABLE msg_queues DROP COLUMN msg_queue_expire;
|
||||
|]
|
||||
|
||||
down_m20260918_expire_messages :: Text
|
||||
down_m20260918_expire_messages =
|
||||
[r|
|
||||
ALTER TABLE msg_queues ADD COLUMN msg_queue_expire boolean NOT NULL DEFAULT FALSE;
|
||||
|
||||
-- ADD COLUMN already defaulted every row to FALSE, so the backfill only needs the rows
|
||||
-- that become TRUE, which avoids rewriting the whole table. The flag is only read to pick
|
||||
-- queues for delete_expired_msgs, and that returns 0 without deleting when msg_queue_size
|
||||
-- is 0, so excluding empty queues here cannot change what expires.
|
||||
UPDATE msg_queues q
|
||||
SET msg_queue_expire = TRUE
|
||||
WHERE msg_queue_size > 0
|
||||
AND EXISTS (SELECT 1 FROM messages m WHERE m.recipient_id = q.recipient_id AND NOT m.msg_quota);
|
||||
|
||||
CREATE INDEX idx_msg_queues_expire ON msg_queues (recipient_id) WHERE deleted_at IS NULL AND msg_queue_expire;
|
||||
|
||||
CREATE INDEX idx_messages_recipient_id_msg_ts ON messages (recipient_id, msg_ts);
|
||||
|
||||
DROP INDEX idx_messages_expire;
|
||||
|
||||
CREATE OR REPLACE FUNCTION delete_expired_msgs(p_recipient_id bytea, p_old_ts bigint) RETURNS bigint
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
q_size BIGINT;
|
||||
keep_min_id BIGINT;
|
||||
del_count BIGINT;
|
||||
BEGIN
|
||||
SELECT msg_queue_size INTO q_size
|
||||
FROM msg_queues
|
||||
WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
|
||||
FOR UPDATE SKIP LOCKED;
|
||||
|
||||
IF NOT FOUND OR q_size = 0 THEN
|
||||
RETURN 0;
|
||||
END IF;
|
||||
|
||||
SELECT MIN(message_id) INTO keep_min_id
|
||||
FROM messages WHERE recipient_id = p_recipient_id AND msg_ts >= p_old_ts AND msg_quota = FALSE;
|
||||
|
||||
IF keep_min_id IS NULL THEN
|
||||
DELETE FROM messages WHERE recipient_id = p_recipient_id AND msg_quota = FALSE;
|
||||
ELSE
|
||||
DELETE FROM messages WHERE recipient_id = p_recipient_id AND message_id < keep_min_id AND msg_quota = FALSE;
|
||||
END IF;
|
||||
|
||||
GET DIAGNOSTICS del_count = ROW_COUNT;
|
||||
IF del_count > 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= del_count,
|
||||
msg_queue_expire = msg_queue_size > del_count AND keep_min_id IS NOT NULL,
|
||||
msg_queue_size = GREATEST(msg_queue_size - del_count, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
RETURN del_count;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION write_message(p_recipient_id bytea, p_msg_id bytea, p_msg_ts bigint, p_msg_quota boolean, p_msg_ntf_flag boolean, p_msg_body bytea, p_quota integer) RETURNS TABLE(quota_written boolean, was_empty boolean)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
q_can_write BOOLEAN;
|
||||
q_size BIGINT;
|
||||
BEGIN
|
||||
SELECT msg_can_write, msg_queue_size INTO q_can_write, q_size
|
||||
FROM msg_queues
|
||||
WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
|
||||
FOR UPDATE;
|
||||
|
||||
IF q_can_write OR q_size = 0 THEN
|
||||
quota_written := p_msg_quota OR q_size >= p_quota;
|
||||
was_empty := q_size = 0;
|
||||
|
||||
INSERT INTO messages(recipient_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body)
|
||||
VALUES (p_recipient_id, p_msg_id, p_msg_ts, quota_written, p_msg_ntf_flag AND NOT quota_written, CASE WHEN quota_written THEN '' :: BYTEA ELSE p_msg_body END);
|
||||
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = NOT quota_written,
|
||||
msg_queue_expire = TRUE,
|
||||
msg_queue_size = msg_queue_size + 1
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
|
||||
RETURN QUERY VALUES (quota_written, was_empty);
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION try_del_msg(p_recipient_id bytea, p_msg_id bytea) RETURNS TABLE(r_msg_id bytea, r_msg_ts bigint, r_msg_quota boolean, r_msg_ntf_flag boolean, r_msg_body bytea)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
q_size BIGINT;
|
||||
msg RECORD;
|
||||
BEGIN
|
||||
SELECT msg_queue_size INTO q_size
|
||||
FROM msg_queues
|
||||
WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
|
||||
FOR UPDATE;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
SELECT message_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
|
||||
INTO msg
|
||||
FROM messages
|
||||
WHERE recipient_id = p_recipient_id
|
||||
ORDER BY message_id ASC LIMIT 1;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
IF q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_expire = FALSE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF msg.msg_id = p_msg_id THEN
|
||||
DELETE FROM messages WHERE message_id = msg.message_id;
|
||||
IF FOUND THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
|
||||
msg_queue_expire = msg_queue_size > 1,
|
||||
msg_queue_size = GREATEST(msg_queue_size - 1, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
END IF;
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
CREATE OR REPLACE FUNCTION try_del_peek_msg(p_recipient_id bytea, p_msg_id bytea) RETURNS TABLE(r_msg_id bytea, r_msg_ts bigint, r_msg_quota boolean, r_msg_ntf_flag boolean, r_msg_body bytea)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
q_size BIGINT;
|
||||
msg RECORD;
|
||||
msg_deleted BOOLEAN;
|
||||
BEGIN
|
||||
SELECT msg_queue_size INTO q_size
|
||||
FROM msg_queues
|
||||
WHERE recipient_id = p_recipient_id AND deleted_at IS NULL
|
||||
FOR UPDATE;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
SELECT message_id, msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
|
||||
INTO msg
|
||||
FROM messages
|
||||
WHERE recipient_id = p_recipient_id
|
||||
ORDER BY message_id ASC LIMIT 1;
|
||||
|
||||
IF NOT FOUND THEN
|
||||
IF q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_expire = FALSE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
RETURN;
|
||||
END IF;
|
||||
|
||||
IF msg.msg_id = p_msg_id THEN
|
||||
DELETE FROM messages WHERE message_id = msg.message_id;
|
||||
|
||||
msg_deleted := FOUND;
|
||||
IF msg_deleted THEN
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
END IF;
|
||||
|
||||
SELECT msg_id, msg_ts, msg_quota, msg_ntf_flag, msg_body
|
||||
INTO msg
|
||||
FROM messages
|
||||
WHERE recipient_id = p_recipient_id
|
||||
ORDER BY message_id ASC LIMIT 1;
|
||||
|
||||
IF FOUND THEN
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
IF msg_deleted THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
|
||||
msg_queue_expire = msg_queue_size > 1,
|
||||
msg_queue_size = GREATEST(msg_queue_size - 1, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
ELSIF msg_deleted OR q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_expire = FALSE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
ELSE
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
END IF;
|
||||
END;
|
||||
$$;
|
||||
|
||||
DROP PROCEDURE expire_old_messages(bigint, integer);
|
||||
|
||||
CREATE PROCEDURE expire_old_messages(IN p_old_ts bigint, IN batch_size integer, OUT r_expired_msgs_count bigint, OUT r_stored_msgs_count bigint, OUT r_stored_queues bigint)
|
||||
LANGUAGE plpgsql
|
||||
AS $$
|
||||
DECLARE
|
||||
rids BYTEA[];
|
||||
rid BYTEA;
|
||||
last_rid BYTEA := '\x';
|
||||
del_count BIGINT;
|
||||
total_deleted BIGINT := 0;
|
||||
BEGIN
|
||||
LOOP
|
||||
SELECT array_agg(recipient_id)
|
||||
INTO rids
|
||||
FROM (
|
||||
SELECT recipient_id
|
||||
FROM msg_queues
|
||||
WHERE deleted_at IS NULL
|
||||
AND msg_queue_expire = TRUE
|
||||
AND recipient_id > last_rid
|
||||
ORDER BY recipient_id ASC
|
||||
LIMIT batch_size
|
||||
) qs;
|
||||
|
||||
EXIT WHEN rids IS NULL OR cardinality(rids) = 0;
|
||||
|
||||
FOREACH rid IN ARRAY rids
|
||||
LOOP
|
||||
BEGIN
|
||||
del_count := delete_expired_msgs(rid, p_old_ts);
|
||||
total_deleted := total_deleted + del_count;
|
||||
EXCEPTION WHEN OTHERS THEN
|
||||
RAISE WARNING 'STORE, expire_old_messages, error expiring queue %: %', encode(rid, 'base64'), SQLERRM;
|
||||
CONTINUE;
|
||||
END;
|
||||
COMMIT;
|
||||
END LOOP;
|
||||
last_rid := rids[cardinality(rids)];
|
||||
END LOOP;
|
||||
|
||||
r_expired_msgs_count := total_deleted;
|
||||
r_stored_msgs_count := (SELECT COUNT(1) FROM messages);
|
||||
r_stored_queues := (SELECT COUNT(1) FROM msg_queues WHERE deleted_at IS NULL);
|
||||
END;
|
||||
$$;
|
||||
|]
|
||||
|
||||
@@ -47,7 +47,6 @@ BEGIN
|
||||
IF del_count > 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= del_count,
|
||||
msg_queue_expire = msg_queue_size > del_count AND keep_min_id IS NOT NULL,
|
||||
msg_queue_size = GREATEST(msg_queue_size - del_count, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
@@ -63,24 +62,33 @@ CREATE PROCEDURE smp_server.expire_old_messages(IN p_old_ts bigint, IN batch_siz
|
||||
DECLARE
|
||||
rids BYTEA[];
|
||||
rid BYTEA;
|
||||
last_ts BIGINT := -1;
|
||||
last_rid BYTEA := '\x';
|
||||
next_ts BIGINT;
|
||||
next_rid BYTEA;
|
||||
del_count BIGINT;
|
||||
total_deleted BIGINT := 0;
|
||||
BEGIN
|
||||
LOOP
|
||||
SELECT array_agg(recipient_id)
|
||||
INTO rids
|
||||
-- The page is scanned in (msg_ts, recipient_id) order, so its last row is the next
|
||||
-- keyset cursor. Advancing it past every row read, including queues left unexpired
|
||||
-- because delete_expired_msgs skipped a locked row or raised, is what terminates the
|
||||
-- loop; re-reading from the start instead would repeat those queues forever.
|
||||
SELECT array_agg(DISTINCT recipient_id),
|
||||
(array_agg(msg_ts ORDER BY msg_ts DESC, recipient_id DESC))[1],
|
||||
(array_agg(recipient_id ORDER BY msg_ts DESC, recipient_id DESC))[1]
|
||||
INTO rids, next_ts, next_rid
|
||||
FROM (
|
||||
SELECT recipient_id
|
||||
FROM msg_queues
|
||||
WHERE deleted_at IS NULL
|
||||
AND msg_queue_expire = TRUE
|
||||
AND recipient_id > last_rid
|
||||
ORDER BY recipient_id ASC
|
||||
SELECT msg_ts, recipient_id
|
||||
FROM messages
|
||||
WHERE NOT msg_quota
|
||||
AND msg_ts < p_old_ts
|
||||
AND (msg_ts, recipient_id) > (last_ts, last_rid)
|
||||
ORDER BY msg_ts ASC, recipient_id ASC
|
||||
LIMIT batch_size
|
||||
) qs;
|
||||
) m;
|
||||
|
||||
EXIT WHEN rids IS NULL OR cardinality(rids) = 0;
|
||||
EXIT WHEN rids IS NULL;
|
||||
|
||||
FOREACH rid IN ARRAY rids
|
||||
LOOP
|
||||
@@ -91,9 +99,11 @@ BEGIN
|
||||
RAISE WARNING 'STORE, expire_old_messages, error expiring queue %: %', encode(rid, 'base64'), SQLERRM;
|
||||
CONTINUE;
|
||||
END;
|
||||
COMMIT;
|
||||
END LOOP;
|
||||
last_rid := rids[cardinality(rids)];
|
||||
COMMIT;
|
||||
|
||||
last_ts := next_ts;
|
||||
last_rid := next_rid;
|
||||
END LOOP;
|
||||
|
||||
r_expired_msgs_count := total_deleted;
|
||||
@@ -195,7 +205,6 @@ BEGIN
|
||||
IF q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_expire = FALSE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
@@ -207,7 +216,6 @@ BEGIN
|
||||
IF FOUND THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
|
||||
msg_queue_expire = msg_queue_size > 1,
|
||||
msg_queue_size = GREATEST(msg_queue_size - 1, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
RETURN QUERY VALUES (msg.msg_id, msg.msg_ts, msg.msg_quota, msg.msg_ntf_flag, msg.msg_body);
|
||||
@@ -245,7 +253,6 @@ BEGIN
|
||||
IF q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_expire = FALSE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
@@ -271,14 +278,12 @@ BEGIN
|
||||
IF msg_deleted THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = msg_can_write OR msg_queue_size <= 1,
|
||||
msg_queue_expire = msg_queue_size > 1,
|
||||
msg_queue_size = GREATEST(msg_queue_size - 1, 0)
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
ELSIF msg_deleted OR q_size != 0 THEN
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = TRUE,
|
||||
msg_queue_expire = FALSE,
|
||||
msg_queue_size = 0
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
END IF;
|
||||
@@ -348,7 +353,6 @@ BEGIN
|
||||
|
||||
UPDATE msg_queues
|
||||
SET msg_can_write = NOT quota_written,
|
||||
msg_queue_expire = TRUE,
|
||||
msg_queue_size = msg_queue_size + 1
|
||||
WHERE recipient_id = p_recipient_id;
|
||||
|
||||
@@ -440,7 +444,6 @@ CREATE TABLE smp_server.msg_queues (
|
||||
rcv_service_id bytea,
|
||||
ntf_service_id bytea,
|
||||
msg_can_write boolean DEFAULT true NOT NULL,
|
||||
msg_queue_expire boolean DEFAULT false NOT NULL,
|
||||
msg_queue_size bigint DEFAULT 0 NOT NULL
|
||||
)
|
||||
WITH (fillfactor='80', autovacuum_vacuum_scale_factor='0.02', autovacuum_analyze_scale_factor='0.01', autovacuum_vacuum_cost_limit='1000');
|
||||
@@ -485,6 +488,10 @@ ALTER TABLE ONLY smp_server.services
|
||||
|
||||
|
||||
|
||||
CREATE INDEX idx_messages_expire ON smp_server.messages USING btree (msg_ts, recipient_id) WHERE (NOT msg_quota);
|
||||
|
||||
|
||||
|
||||
CREATE INDEX idx_messages_recipient_id_message_id ON smp_server.messages USING btree (recipient_id, message_id);
|
||||
|
||||
|
||||
@@ -493,14 +500,6 @@ CREATE INDEX idx_messages_recipient_id_msg_quota ON smp_server.messages USING bt
|
||||
|
||||
|
||||
|
||||
CREATE INDEX idx_messages_recipient_id_msg_ts ON smp_server.messages USING btree (recipient_id, msg_ts);
|
||||
|
||||
|
||||
|
||||
CREATE INDEX idx_msg_queues_expire ON smp_server.msg_queues USING btree (recipient_id) WHERE ((deleted_at IS NULL) AND msg_queue_expire);
|
||||
|
||||
|
||||
|
||||
CREATE UNIQUE INDEX idx_msg_queues_link_id ON smp_server.msg_queues USING btree (link_id);
|
||||
|
||||
|
||||
|
||||
@@ -54,6 +54,7 @@ module Simplex.Messaging.Transport
|
||||
namesSMPVersion,
|
||||
serverInfoSMPVersion,
|
||||
nameAvailSMPVersion,
|
||||
fwdNoncesSMPVersion,
|
||||
simplexMQVersion,
|
||||
smpBlockSize,
|
||||
TransportConfig (..),
|
||||
@@ -177,6 +178,7 @@ smpBlockSize = 16384
|
||||
-- 20 - public namespaces resolver, RSLV command (6/20/2026)
|
||||
-- 21 - server public information in handshake (7/5/2026)
|
||||
-- 22 - RNAME answers name availability as well as the record (7/25/2026)
|
||||
-- 23 - version in forwarded command nonce, random nonce in forwarded responses (10/2/2026)
|
||||
|
||||
data SMPVersion
|
||||
|
||||
@@ -218,6 +220,9 @@ serverInfoSMPVersion = VersionSMP 21
|
||||
nameAvailSMPVersion :: VersionSMP
|
||||
nameAvailSMPVersion = VersionSMP 22
|
||||
|
||||
fwdNoncesSMPVersion :: VersionSMP
|
||||
fwdNoncesSMPVersion = VersionSMP 23
|
||||
|
||||
minClientSMPRelayVersion :: VersionSMP
|
||||
minClientSMPRelayVersion = VersionSMP 14
|
||||
|
||||
@@ -225,20 +230,20 @@ minServerSMPRelayVersion :: VersionSMP
|
||||
minServerSMPRelayVersion = VersionSMP 14
|
||||
|
||||
currentClientSMPRelayVersion :: VersionSMP
|
||||
currentClientSMPRelayVersion = VersionSMP 22
|
||||
currentClientSMPRelayVersion = VersionSMP 23
|
||||
|
||||
currentServerSMPRelayVersion :: VersionSMP
|
||||
currentServerSMPRelayVersion = VersionSMP 22
|
||||
currentServerSMPRelayVersion = VersionSMP 23
|
||||
|
||||
-- Max SMP protocol version to be used in e2e encrypted connection between
|
||||
-- client and server, as defined by SMP proxy. Normally set below the current
|
||||
-- version to prevent client version fingerprinting by the destination relays
|
||||
-- when clients upgrade at different times. Pinned to the current version (22)
|
||||
-- for this release because a proxied RSLV only carries availability from
|
||||
-- nameAvailSMPVersion (22), so the one-version anti-fingerprinting buffer does
|
||||
-- not apply yet; it reappears once the current version advances past 22.
|
||||
-- when clients upgrade at different times. Pinned to the current version (23)
|
||||
-- for this release because forwarded commands use the nonces from
|
||||
-- fwdNoncesSMPVersion (23), so the one-version anti-fingerprinting buffer does
|
||||
-- not apply yet; it reappears once the current version advances past 23.
|
||||
proxiedSMPRelayVersion :: VersionSMP
|
||||
proxiedSMPRelayVersion = VersionSMP 22
|
||||
proxiedSMPRelayVersion = VersionSMP 23
|
||||
|
||||
-- minimal supported protocol version is 14
|
||||
supportedClientSMPRelayVRange :: VersionRangeSMP
|
||||
|
||||
@@ -231,7 +231,11 @@ pattern Rcvd' :: AgentMsgId -> AgentMsgId -> AEvent 'AEConn
|
||||
pattern Rcvd' aMsgId rcvdMsgId <- RCVD MsgMeta {integrity = MsgOk, recipient = (aMsgId, _)} [MsgReceipt {agentMsgId = rcvdMsgId, msgRcptStatus = MROk}]
|
||||
|
||||
smpCfgVPrev :: ProtocolClientConfig SMPVersion
|
||||
smpCfgVPrev = (smpCfg agentCfg) {serverVRange = prevRange $ serverVRange $ smpCfg agentCfg}
|
||||
smpCfgVPrev =
|
||||
(smpCfg agentCfg)
|
||||
{ serverVRange = prevRange $ serverVRange $ smpCfg agentCfg,
|
||||
proxiedRelayVRange = prevRange $ proxiedRelayVRange $ smpCfg agentCfg
|
||||
}
|
||||
|
||||
-- ntfCfgVPrev :: ProtocolClientConfig NTFVersion
|
||||
-- ntfCfgVPrev = (ntfCfg agentCfg) {clientALPN = Nothing, serverVRange = V.mkVersionRange (VersionNTF 1) (VersionNTF 1)}
|
||||
|
||||
@@ -79,6 +79,7 @@ msgStoreTests = do
|
||||
someMsgStoreTests
|
||||
it "should correctly update message counts and canWrite flag" testUpdateMessageCounts
|
||||
it "tryDelPeekMsg (ACK not from NSE) should reset message counts when queue is empty" testResetMessageCounts
|
||||
it "should expire messages across commit batches" testExpireMessagesInBatches
|
||||
#endif
|
||||
describe "Journal message store: queue state backup expiration" $ do
|
||||
it "should remove old queue state backups" testRemoveQueueStateBackups
|
||||
@@ -327,35 +328,34 @@ testUpdateMessageCounts ms = do
|
||||
q <- ExceptT $ addQueue ms rId qr
|
||||
let write s = writeMsg ms q True =<< mkMessage s
|
||||
hasSize = checkQueueSize ms
|
||||
q `hasSize` (0, True, False)
|
||||
q `hasSize` (0, True)
|
||||
Just (Message {msgId = mId1}, True) <- write "message 1"
|
||||
q `hasSize` (1, True, True)
|
||||
q `hasSize` (1, True)
|
||||
Just (Message {msgId = mId2}, False) <- write "message 2"
|
||||
q `hasSize` (2, True, True)
|
||||
q `hasSize` (2, True)
|
||||
Just (Message {msgId = mId3}, False) <- write "message 3"
|
||||
q `hasSize` (3, True, True)
|
||||
q `hasSize` (3, True)
|
||||
Nothing <- write "message 4"
|
||||
q `hasSize` (4, False, True)
|
||||
q `hasSize` (4, False)
|
||||
Msg "message 1" <- tryPeekMsg ms q
|
||||
q `hasSize` (4, False, True)
|
||||
q `hasSize` (4, False)
|
||||
Msg "message 1" <- tryDelMsg ms q mId1
|
||||
q `hasSize` (3, False, True)
|
||||
q `hasSize` (3, False)
|
||||
Msg "message 2" <- tryPeekMsg ms q
|
||||
(Msg "message 2", Msg "message 3") <- tryDelPeekMsg ms q mId2
|
||||
q `hasSize` (2, False, True)
|
||||
q `hasSize` (2, False)
|
||||
(Msg "message 3", Just MessageQuota {msgId = mId4}) <- tryDelPeekMsg ms q mId3
|
||||
q `hasSize` (1, False, True)
|
||||
q `hasSize` (1, False)
|
||||
(Just MessageQuota {}, Nothing) <- tryDelPeekMsg ms q mId4
|
||||
q `hasSize` (0, True, False)
|
||||
q `hasSize` (0, True)
|
||||
|
||||
checkQueueSize :: PostgresMsgStore -> PostgresQueue -> (Int64, Bool, Bool) -> ExceptT ErrorType IO ()
|
||||
checkQueueSize ms q (size, canWrt, expire) = liftIO $ do
|
||||
[(size', canWrt', expire')] <-
|
||||
checkQueueSize :: PostgresMsgStore -> PostgresQueue -> (Int64, Bool) -> ExceptT ErrorType IO ()
|
||||
checkQueueSize ms q (size, canWrt) = liftIO $ do
|
||||
[(size', canWrt')] <-
|
||||
withTransaction (dbStore $ queueStore ms) $ \db ->
|
||||
DB.query db "SELECT msg_queue_size, msg_can_write, msg_queue_expire FROM msg_queues WHERE recipient_id = ?" (Only (recipientId q))
|
||||
DB.query db "SELECT msg_queue_size, msg_can_write FROM msg_queues WHERE recipient_id = ?" (Only (recipientId q))
|
||||
size' `shouldBe` size
|
||||
canWrt' `shouldBe` canWrt
|
||||
expire' `shouldBe` expire
|
||||
|
||||
testResetMessageCounts :: PostgresMsgStore -> IO ()
|
||||
testResetMessageCounts ms = do
|
||||
@@ -369,25 +369,63 @@ testResetMessageCounts ms = do
|
||||
Just (Message {msgId = mId2}, False) <- write "message 2"
|
||||
Just (Message {msgId = mId3}, False) <- write "message 3"
|
||||
Nothing <- write "message 4"
|
||||
q `hasSize` (4, False, True)
|
||||
q `hasSize` (4, False)
|
||||
liftIO $ setIncorrectSize q (10, True)
|
||||
Nothing <- write "message 5"
|
||||
q `hasSize` (11, False, True)
|
||||
q `hasSize` (11, False)
|
||||
(Msg "message 1", Msg "message 2") <- tryDelPeekMsg ms q mId1
|
||||
q `hasSize` (10, False, True)
|
||||
q `hasSize` (10, False)
|
||||
(Msg "message 2", Msg "message 3") <- tryDelPeekMsg ms q mId2
|
||||
q `hasSize` (9, False, True)
|
||||
q `hasSize` (9, False)
|
||||
(Msg "message 3", Just MessageQuota {msgId = mId4}) <- tryDelPeekMsg ms q mId3
|
||||
q `hasSize` (8, False, True)
|
||||
q `hasSize` (8, False)
|
||||
(Just MessageQuota {}, Just MessageQuota {msgId = mId5}) <- tryDelPeekMsg ms q mId4
|
||||
q `hasSize` (7, False, True)
|
||||
q `hasSize` (7, False)
|
||||
(Just MessageQuota {}, Nothing) <- tryDelPeekMsg ms q mId5
|
||||
q `hasSize` (0, True, False) -- reset
|
||||
q `hasSize` (0, True) -- reset
|
||||
where
|
||||
setIncorrectSize :: PostgresQueue -> (Int64, Bool) -> IO ()
|
||||
setIncorrectSize q (size, canWrt) =
|
||||
void $ withTransaction (dbStore $ queueStore ms) $ \db ->
|
||||
DB.execute db "UPDATE msg_queues SET msg_queue_size = ?, msg_can_write = ? WHERE recipient_id = ?" (size, canWrt, recipientId q)
|
||||
|
||||
testExpireMessagesInBatches :: PostgresMsgStore -> IO ()
|
||||
testExpireMessagesInBatches ms = do
|
||||
g <- C.newRandom
|
||||
emptiedQs <- replicateM emptiedCount $ newQueue g
|
||||
partialQs <- replicateM partialCount $ newQueue g
|
||||
overQuotaQs <- replicateM overQuotaCount $ newQueue g
|
||||
quotaMsgs <- runRight $ do
|
||||
forM_ (emptiedQs <> partialQs) $ \q -> void $ write q "old 1"
|
||||
forM_ emptiedQs $ \q -> void $ write q "old 2"
|
||||
mapM fillPastQuota overQuotaQs
|
||||
-- msg_ts has second granularity, so the recent messages need a new second to be kept
|
||||
threadDelay 1100000
|
||||
boundary <- systemSeconds <$> getSystemTime
|
||||
runRight_ $ forM_ partialQs $ \q -> void $ write q "recent"
|
||||
|
||||
MessageStats {expiredMsgsCount, storedMsgsCount, storedQueues} <- expireOldMessages False ms boundary 0
|
||||
expiredMsgsCount `shouldBe` (emptiedCount * 2 + partialCount + sum quotaMsgs)
|
||||
storedMsgsCount `shouldBe` (partialCount + overQuotaCount) -- recent messages and quota markers
|
||||
storedQueues `shouldBe` (emptiedCount + partialCount + overQuotaCount)
|
||||
runRight_ $ do
|
||||
forM_ emptiedQs $ \q -> checkQueueSize ms q (0, True)
|
||||
forM_ partialQs $ \q -> checkQueueSize ms q (1, True)
|
||||
-- the quota marker is never expired, and the queue stays blocked until it is acked
|
||||
forM_ overQuotaQs $ \q -> checkQueueSize ms q (1, False)
|
||||
where
|
||||
-- expire_old_messages pages through expired messages and commits per page, so these
|
||||
-- counts put a page boundary inside each group of queues.
|
||||
emptiedCount = 120 :: Int
|
||||
partialCount = 40 :: Int
|
||||
overQuotaCount = 10 :: Int
|
||||
newQueue g = do
|
||||
(rId, qr) <- testNewQueueRec g QMMessaging
|
||||
runRight $ ExceptT $ addQueue ms rId qr
|
||||
write q s = writeMsg ms q True =<< mkMessage s
|
||||
fillPastQuota q = go 0
|
||||
where
|
||||
go n = write q "fill" >>= maybe (pure n) (const $ go (n + 1))
|
||||
#endif
|
||||
|
||||
testQueueState :: JournalMsgStore s -> IO ()
|
||||
|
||||
@@ -22,7 +22,7 @@ import qualified Data.X509.Validation as XV
|
||||
import SMPClient
|
||||
import Simplex.FileTransfer.Protocol (FileInfo (..))
|
||||
import Simplex.FileTransfer.Server.Store (FileRec (..), FileRecipient (..), FileStoreClass (..), RoundedFileTime, STMFileStore (..))
|
||||
import Simplex.FileTransfer.Server.StoreLog (FileStoreLogRecord (..), readWriteFileStore)
|
||||
import Simplex.FileTransfer.Server.StoreLog (FileStoreLogRecord (..), readWriteFileStore, serverFileName)
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
import Simplex.Messaging.Encoding.String
|
||||
import Simplex.Messaging.Protocol
|
||||
@@ -37,6 +37,7 @@ import Simplex.Messaging.Server.StoreLog
|
||||
import Simplex.Messaging.SystemTime
|
||||
import Simplex.Messaging.Transport (SMPServiceRole (..))
|
||||
import Simplex.Messaging.Transport.Credentials (genCredentials)
|
||||
import System.FilePath ((</>))
|
||||
import Test.Hspec hiding (fit, it)
|
||||
import Util
|
||||
|
||||
@@ -269,6 +270,18 @@ fileStoreLogTests = do
|
||||
saved = [AddFile sId file createdAt Nothing EntityActive, BlockFile sId blockedWithNotice],
|
||||
compacted = [AddFile sId file createdAt Nothing (EntityBlocked blockedWithNotice)],
|
||||
state = M.fromList [(sId, (file, createdAt, Nothing, EntityBlocked blockedWithNotice))]
|
||||
},
|
||||
SLTC
|
||||
{ name = "put file with wrong file name",
|
||||
saved = [AddFile sId file createdAt (Just expiresAt) EntityActive, PutFile sId "/etc/passwd"],
|
||||
compacted = [AddFile sId file createdAt (Just expiresAt) EntityActive],
|
||||
state = M.fromList [(sId, (file, createdAt, Just expiresAt, EntityActive))]
|
||||
},
|
||||
SLTC
|
||||
{ name = "put file in another directory",
|
||||
saved = [AddFile sId file createdAt (Just expiresAt) EntityActive, PutFile sId ("/srv/xftp" </> serverFileName sId)],
|
||||
compacted = [AddFile sId file createdAt (Just expiresAt) EntityActive, PutFile sId ("/srv/xftp" </> serverFileName sId)],
|
||||
state = M.fromList [(sId, (file, createdAt, Just expiresAt, EntityActive))]
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import Data.Word (Word32)
|
||||
import Simplex.FileTransfer.Protocol (FileInfo (..), SFileParty (..))
|
||||
import Simplex.FileTransfer.Server.Store
|
||||
import Simplex.FileTransfer.Server.Store.Postgres (PostgresFileStore, importFileStore, exportFileStore)
|
||||
import Simplex.FileTransfer.Server.StoreLog (closeStoreLog, readWriteFileStore, writeFileStore)
|
||||
import Simplex.FileTransfer.Server.StoreLog (closeStoreLog, readWriteFileStore, serverFileName, writeFileStore)
|
||||
import Simplex.FileTransfer.Transport (XFTPErrorType (..))
|
||||
import qualified Simplex.Messaging.Crypto as C
|
||||
import Simplex.Messaging.Protocol (BlockingInfo (..), BlockingReason (..), EntityId (..))
|
||||
@@ -17,6 +17,7 @@ import Simplex.Messaging.Server.QueueStore (ServerEntityStatus (..))
|
||||
import Simplex.Messaging.Server.StoreLog (openWriteStoreLog)
|
||||
import Simplex.Messaging.SystemTime (RoundedSystemTime (..))
|
||||
import System.Directory (doesFileExist, removeFile)
|
||||
import System.FilePath ((</>))
|
||||
import Test.Hspec hiding (fit, it)
|
||||
import UnliftIO.STM
|
||||
import Util
|
||||
@@ -265,7 +266,7 @@ testMigrationRoundTrip = do
|
||||
sId2 = EntityId "migration_file_2"
|
||||
rId1 = EntityId "migration_rcp_1_"
|
||||
addFile stmStore sId1 fileInfo1 testCreatedAt (Just testExpiresAt) EntityActive `shouldReturn` Right ()
|
||||
void $ setFilePath stmStore sId1 "/tmp/file1"
|
||||
void $ setFilePath stmStore sId1 ("/tmp" </> serverFileName sId1)
|
||||
addRecipient stmStore sId1 (FileRecipient rId1 rcpKey1) `shouldReturn` Right ()
|
||||
let testBlockInfo = BlockingInfo {reason = BRSpam, notice = Nothing}
|
||||
addFile stmStore sId2 fileInfo2 testCreatedAt Nothing (EntityBlocked testBlockInfo) `shouldReturn` Right ()
|
||||
@@ -289,7 +290,7 @@ testMigrationRoundTrip = do
|
||||
case result1 of
|
||||
Right (FileRec {fileInfo = fi, filePath, expiresAt, fileStatus}, _) -> do
|
||||
size fi `shouldBe` 128000
|
||||
readTVarIO filePath `shouldReturn` Just "/tmp/file1"
|
||||
readTVarIO filePath `shouldReturn` Just ("/tmp" </> serverFileName sId1)
|
||||
expiresAt `shouldBe` Just testExpiresAt
|
||||
readTVarIO fileStatus `shouldReturn` EntityActive
|
||||
Left e -> expectationFailure $ "getFile sId1 failed: " <> show e
|
||||
|
||||
@@ -349,6 +349,16 @@ proxyCfgShortTimeout =
|
||||
nt = NetworkTimeout {backgroundTimeout = 4_000000, interactiveTimeout = 4_000000}
|
||||
in cfg' {smpAgentCfg = aCfg {smpCfg = cCfg {networkConfig = (networkConfig cCfg) {tcpConnectTimeout = nt}}}}
|
||||
|
||||
proxyCfgVPrev :: AStoreType -> AServerConfig
|
||||
proxyCfgVPrev msType =
|
||||
updateCfg (proxyCfgMS msType) $ \cfg' ->
|
||||
let aCfg = smpAgentCfg cfg'
|
||||
cCfg = smpCfg aCfg
|
||||
in cfg'
|
||||
{ smpServerVRange = prevRange $ smpServerVRange cfg',
|
||||
smpAgentCfg = aCfg {smpCfg = cCfg {serverVRange = prevRange $ serverVRange cCfg}}
|
||||
}
|
||||
|
||||
withSmpServerStoreMsgLogOn :: HasCallStack => (ASrvTransport, AStoreType) -> ServiceName -> (HasCallStack => ThreadId -> IO a) -> IO a
|
||||
withSmpServerStoreMsgLogOn (t, msType) =
|
||||
withSmpServerConfigOn t $ updateCfg (cfgMS msType) $ \cfg' -> cfg' {storeNtfsFile = Just testStoreNtfsFile, serverStatsBackupFile = Just testServerStatsBackupFile}
|
||||
|
||||
+30
-2
@@ -73,6 +73,8 @@ smpProxyTests = do
|
||||
xit "no SMP service at host/port" todo
|
||||
xit "bad SMP fingerprint" todo
|
||||
xit "batching proxy requests" todo
|
||||
it "relay rejects forwarded command with changed version" $ \_ ->
|
||||
testChangedFwdVersion
|
||||
describe "deliver message via SMP proxy" $ do
|
||||
let srv1 = SMPServer testHost testPort testKeyHash
|
||||
srv2 = SMPServer testHost2 testPort2 testKeyHash
|
||||
@@ -95,6 +97,11 @@ smpProxyTests = do
|
||||
deliverMessageViaProxy proxyServ relayServ C.SEd25519 msg1 msg2
|
||||
it "max message size, X25519 keys" . twoServersFirstProxy $
|
||||
deliverMessageViaProxy proxyServ relayServ C.SX25519 msg1 msg2
|
||||
describe "version compatibility" $ do
|
||||
let deliver clientVR = deliverMessagesViaProxyVR clientVR srv1 srv2 C.SEd448 ["hello 1"] ["hello 2"]
|
||||
it "prev client" . twoServersFirstProxy $ deliver (prevRange supportedClientSMPRelayVRange)
|
||||
it "prev proxy" . twoServersPrevProxy $ deliver supportedClientSMPRelayVRange
|
||||
it "prev relay" . twoServersPrevRelay $ deliver supportedClientSMPRelayVRange
|
||||
describe "stress test 1k" $ do
|
||||
let deliver n = deliverMessagesViaProxy srv1 srv2 C.SEd448 [] (map bshow [1 :: Int .. n])
|
||||
it "1x1000" . twoServersFirstProxy $ deliver 1000
|
||||
@@ -155,6 +162,9 @@ smpProxyTests = do
|
||||
twoServersFirstProxy test msType = twoServers_ (proxyCfgMS msType) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128, maxJournalMsgCount = 256}) test msType
|
||||
twoServersMoreConc test msType = twoServers_ (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {serverClientConcurrency = 128}) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128, maxJournalMsgCount = 256}) test msType
|
||||
twoServersNoConc test msType = twoServers_ (updateCfg (proxyCfgMS msType) $ \cfg_ -> cfg_ {serverClientConcurrency = 1}) (updateCfg (cfgMS msType) $ \cfg_ -> cfg_ {msgQueueQuota = 128, maxJournalMsgCount = 256}) test msType
|
||||
twoServersPrevProxy test msType = twoServers_ (proxyCfgVPrev msType) (cfgMS msType) test msType
|
||||
twoServersPrevRelay test msType = twoServers_ (proxyCfgMS msType) (prevServerVRange $ cfgMS msType) test msType
|
||||
prevServerVRange cfg' = updateCfg cfg' $ \cfg_ -> cfg_ {smpServerVRange = prevRange $ smpServerVRange cfg_}
|
||||
twoServers_ :: AServerConfig -> AServerConfig -> IO () -> AStoreType -> IO ()
|
||||
twoServers_ cfg1 cfg2 runTest (ASType qsType _) =
|
||||
withSmpServerConfigOn (transport @TLS) cfg1 testPort $ \_ ->
|
||||
@@ -167,11 +177,14 @@ deliverMessageViaProxy :: (C.AlgorithmI a, C.AuthAlgorithm a) => SMPServer -> SM
|
||||
deliverMessageViaProxy proxyServ relayServ alg msg msg' = deliverMessagesViaProxy proxyServ relayServ alg [msg] [msg']
|
||||
|
||||
deliverMessagesViaProxy :: (C.AlgorithmI a, C.AuthAlgorithm a) => SMPServer -> SMPServer -> C.SAlgorithm a -> [ByteString] -> [ByteString] -> IO ()
|
||||
deliverMessagesViaProxy proxyServ relayServ alg unsecuredMsgs securedMsgs = do
|
||||
deliverMessagesViaProxy = deliverMessagesViaProxyVR $ mkVersionRange minServerSMPRelayVersion currentClientSMPRelayVersion
|
||||
|
||||
deliverMessagesViaProxyVR :: (C.AlgorithmI a, C.AuthAlgorithm a) => VersionRangeSMP -> SMPServer -> SMPServer -> C.SAlgorithm a -> [ByteString] -> [ByteString] -> IO ()
|
||||
deliverMessagesViaProxyVR clientVR proxyServ relayServ alg unsecuredMsgs securedMsgs = do
|
||||
g <- C.newRandom
|
||||
-- set up proxy
|
||||
ts <- getCurrentTime
|
||||
pc' <- getProtocolClient g NRMInteractive (1, proxyServ, Nothing) defaultSMPClientConfig {serverVRange = mkVersionRange minServerSMPRelayVersion currentClientSMPRelayVersion} [] Nothing ts (\_ -> pure ())
|
||||
pc' <- getProtocolClient g NRMInteractive (1, proxyServ, Nothing) defaultSMPClientConfig {serverVRange = clientVR, proxiedRelayVRange = clientVR} [] Nothing ts (\_ -> pure ())
|
||||
pc <- either (fail . show) pure pc'
|
||||
THAuthClient {} <- maybe (fail "getProtocolClient returned no thAuth") pure $ thAuth $ thParams pc
|
||||
-- set up relay
|
||||
@@ -448,6 +461,21 @@ requestRelaySession =
|
||||
testSMPClient_ "localhost" testPort supportedServerSMPRelayVRange Nothing $ \(th :: THandleSMP TLS 'TClient) ->
|
||||
(\(_, _, reply) -> reply) <$> sendRecv th (Nothing, "1", NoEntity, SMP.PRXY testSMPServer2 Nothing)
|
||||
|
||||
testChangedFwdVersion :: IO ()
|
||||
testChangedFwdVersion =
|
||||
withSmpServerConfigOn (transport @TLS) cfg testPort $ \_ -> do
|
||||
g <- C.newRandom
|
||||
ts <- getCurrentTime
|
||||
rc <- either (fail . show) pure =<< getProtocolClient g NRMInteractive (1, testSMPServer, Nothing) defaultSMPClientConfig [] Nothing ts (\_ -> pure ())
|
||||
THAuthClient {peerServerPubKey} <- maybe (fail "getProtocolClient returned no thAuth") pure $ thAuth $ thParams rc
|
||||
(cmdPubKey, cmdPrivKey) <- atomically $ C.generateKeyPair g
|
||||
nonce@(C.CbNonce corrId) <- atomically $ C.randomCbNonce g
|
||||
let v = currentClientSMPRelayVersion
|
||||
et <- either (fail . show) (pure . SMP.EncTransmission) $ C.cbEncrypt (C.dh' peerServerPubKey cmdPrivKey) (SMP.encTransmissionNonce v nonce) "" SMP.paddedProxiedTLength
|
||||
let forward fwdVersion = forwardSMPTransmission rc (SMP.CorrId corrId) fwdVersion cmdPubKey et
|
||||
_ <- runExceptT' $ forward v
|
||||
runExceptT (forward $ prevVersion v) `shouldReturn` Left (PCEProtocolError SMP.CRYPTO)
|
||||
|
||||
-- Shared "phase 2" of the reconnection tests: start a healthy relay, confirm it is reachable
|
||||
-- directly (PING, not via the proxy) so a proxy failure can only mean the proxy didn't reconnect,
|
||||
-- let any stored connection error expire, then require the proxy to establish the session (PKEY).
|
||||
|
||||
+2
-1
@@ -109,7 +109,8 @@ main = do
|
||||
describe "SMP server schema dump" $
|
||||
postgresSchemaDumpTest
|
||||
serverMigrations
|
||||
[ "20250320_short_links" -- snd_secure moves to the bottom on down migration
|
||||
[ "20250320_short_links", -- snd_secure moves to the bottom on down migration
|
||||
"20260918_expire_messages" -- msg_queue_expire moves to the bottom on down migration
|
||||
] -- skipComparisonForDownMigrations
|
||||
testStoreDBOpts
|
||||
"src/Simplex/Messaging/Server/QueueStore/Postgres/server_schema.sql"
|
||||
|
||||
Reference in New Issue
Block a user