Compare commits

...

27 Commits

Author SHA1 Message Date
Jade Ellis
b5927380f9 ci: Use dataaxiom/ghcr-cleanup-action 2025-11-08 23:53:00 +00:00
Jade Ellis
f7867cf6ca ci: Clean up old images 2025-11-08 23:29:25 +00:00
Jade Ellis
2ca6887a5d chore(ci): Fix merge error 2025-11-08 23:08:10 +00:00
Jade Ellis
368685f8cd ci: Re-run mirror script when files change 2025-11-08 23:00:37 +00:00
Jade Ellis
ad2d192b94 ci: Use PATs for github registry
https://stackoverflow.com/questions/76821352/how-can-you-authenticate-to-the-github-container-registry-using-a-github-app

thx github
2025-11-08 23:00:31 +00:00
Jade Ellis
3214e94cdb ci: Mirror to ghcr 2025-11-08 22:59:27 +00:00
timedout
37c537379d chore(ci): Add git.nexy7574.co.uk image mirror (#1149)
secrets were added to the org

Reviewed-on: https://forgejo.ellis.link/continuwuation/continuwuity/pulls/1149
Co-authored-by: timedout <git@nexy7574.co.uk>
Co-committed-by: timedout <git@nexy7574.co.uk>
2025-11-08 22:56:16 +00:00
Jade Ellis
3c01c5f085 chore: Don't try to update patched deps automatically 2025-11-08 21:17:04 +00:00
Renovate Bot
4c552bb8ca chore(deps): update pre-commit hook pre-commit/pre-commit-hooks to v6 2025-11-08 20:56:00 +00:00
Jade Ellis
ce73d29855 chore: Fix typos 2025-11-08 20:54:49 +00:00
Renovate Bot
d6e314744b chore(deps): update pre-commit hook crate-ci/typos to v1.39.0 2025-11-08 14:34:13 +00:00
Jade
ec603188de ci: Enable pre-commit in renovate 2025-11-08 14:31:35 +00:00
timedout
fbf48addc7 fix(user_can): Fix room creators being unable to redact events in v12 rooms 2025-10-27 14:34:50 +00:00
nexy7574
cbf726580f fix: Kicks in !v12 are impossible 2025-10-27 14:34:50 +00:00
nexy7574
28f258fc8c fix: Incorrect interpretation of 5.5.4 2025-10-27 14:34:50 +00:00
nexy7574
8b3acfd770 fix: Inverted creatorship check 2025-10-27 14:34:50 +00:00
nexy7574
a581e8de01 fix: Don't check restricted join rules for invite joins 2025-10-27 14:34:50 +00:00
nexy7574
7c74db5e74 fix: Weird re-application of partially resolved state 2025-10-27 14:34:50 +00:00
nexy7574
b17b4235f3 fix: Unbans and kicks incorrectly checked creatorship in !v12 2025-10-27 14:34:50 +00:00
aviac
ec3564e8aa chore: use upstream rust-jemalloc-sys-unprefixed after flake.lock update 2025-10-27 12:55:21 +00:00
aviac
9a887ac04b chore: fix CI to make all checks green
- define a nix default package
- try to fix CI
- fix/improve (?) CI even more (??)
2025-10-27 12:55:21 +00:00
aviac
fed808a3c6 feat: add taplo.toml to check now that we have it 2025-10-27 12:55:21 +00:00
aviac
37983b33a2 feat: add treefmt 2025-10-27 12:55:21 +00:00
aviac
1b2224fac6 feat: add hydra jobs to build all packages 2025-10-27 12:55:21 +00:00
aviac
c1c165ab48 fix: apply rocksdb changes in checks and shll 2025-10-27 12:55:20 +00:00
aviac
68bea1816f feat(nix): flake-parts, first draft 2025-10-27 12:55:20 +00:00
Odd E. Ebbesen
cb7875e479 fix(#1134): Update docs and implementation of admin media delete-past-remote-media (#1136)
Reviewed-on: https://forgejo.ellis.link/continuwuation/continuwuity/pulls/1136
Co-authored-by: Odd E. Ebbesen <git@oddware.net>
Co-committed-by: Odd E. Ebbesen <git@oddware.net>
2025-10-27 12:31:25 +00:00
39 changed files with 1208 additions and 1128 deletions

View File

@@ -40,6 +40,12 @@ creds:
- registry: registry.gitlab.com
user: "{{env \"GITLAB_USERNAME\"}}"
pass: "{{env \"GITLAB_TOKEN\"}}"
- registry: git.nexy7574.co.uk
user: "{{env \"N7574_GIT_USERNAME\"}}"
pass: "{{env \"N7574_GIT_TOKEN\"}}"
- registry: ghcr.io
user: "{{env \"GH_PACKAGES_USER\"}}"
pass: "{{env \"GH_PACKAGES_TOKEN\"}}"
# Global defaults
defaults:
@@ -53,3 +59,11 @@ sync:
target: registry.gitlab.com/continuwuity/continuwuity
type: repository
<<: *tags-main
- source: *source
target: git.nexy7574.co.uk/mirrored/continuwuity
type: repository
<<: *tags-releases
- source: *source
target: ghcr.io/continuwuity/continuwuity
type: repository
<<: *tags-main

View File

@@ -0,0 +1,110 @@
name: Cleanup Registry Images
on:
schedule:
# Run daily at 01:30 UTC
- cron: '30 1 * * *'
workflow_dispatch:
inputs:
dry_run:
description: 'Dry run (check only, no actual deletion)'
required: false
default: false
type: boolean
pull_request:
types: [closed]
delete:
# Triggered when branches are deleted
concurrency:
group: "cleanup-registry"
cancel-in-progress: false
env:
BUILTIN_REGISTRY: forgejo.ellis.link
IMAGE_PATH: forgejo.ellis.link/continuwuation/continuwuity
jobs:
cleanup-pr-images:
name: Cleanup PR Images
runs-on: ubuntu-latest
if: github.event_name == 'pull_request' && github.event.action == 'closed'
permissions:
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v5
with:
persist-credentials: false
- name: Delete PR image
uses: https://github.com/dataaxiom/ghcr-cleanup-action@v1
with:
token: ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }}
owner: continuwuation
repository: continuwuity
package: continuwuity
registry-url: https://${{ env.BUILTIN_REGISTRY }}
delete-tags: pr-${{ github.event.pull_request.number }}
dry-run: false
cleanup-old-commits:
name: Cleanup Old Commit Images
runs-on: ubuntu-latest
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
permissions:
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v5
with:
persist-credentials: false
- name: Set dry-run mode
id: params
run: |
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
echo "dry_run=${{ inputs.dry_run }}" >> $GITHUB_OUTPUT
else
# Scheduled runs perform actual cleanup
echo "dry_run=false" >> $GITHUB_OUTPUT
fi
- name: Cleanup old SHA commit images
uses: https://github.com/dataaxiom/ghcr-cleanup-action@v1
with:
token: ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }}
owner: continuwuation
repository: continuwuity
package: continuwuity
registry-url: https://${{ env.BUILTIN_REGISTRY }}
delete-tags: sha-*
exclude-tags: latest,main,v*.*.*,*.*.*,*-maxperf
older-than: 30 days
dry-run: ${{ steps.params.outputs.dry_run }}
delete-ghost-images: true
delete-partial-images: true
delete-orphaned-images: true
cleanup-branch-images:
name: Cleanup Deleted Branch Images
runs-on: ubuntu-latest
if: github.event_name == 'delete' && github.event.ref_type == 'branch'
permissions:
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v5
with:
persist-credentials: false
- name: Delete branch image
uses: https://github.com/dataaxiom/ghcr-cleanup-action@v1
with:
token: ${{ secrets.BUILTIN_REGISTRY_PASSWORD || secrets.GITHUB_TOKEN }}
owner: continuwuation
repository: continuwuity
package: continuwuity
registry-url: https://${{ env.BUILTIN_REGISTRY }}
delete-tags: branch-${{ github.event.ref }}
dry-run: false

View File

@@ -11,7 +11,13 @@ on:
required: false
default: false
type: boolean
push:
branches:
- main
paths:
# Re-run when config changes
- '.forgejo/regsync/regsync.yml'
- '.forgejo/workflows/mirror-images.yml'
concurrency:
group: "mirror-images"
cancel-in-progress: true
@@ -24,12 +30,25 @@ jobs:
BUILTIN_REGISTRY_PASSWORD: ${{ secrets.BUILTIN_REGISTRY_PASSWORD }}
GITLAB_USERNAME: ${{ vars.GITLAB_USERNAME }}
GITLAB_TOKEN: ${{ secrets.GITLAB_TOKEN }}
N7574_GIT_USERNAME: ${{ vars.N7574_GIT_USERNAME }}
N7574_GIT_TOKEN: ${{ secrets.N7574_GIT_TOKEN }}
GH_PACKAGES_USER: ${{ vars.GH_PACKAGES_USER }}
GH_PACKAGES_TOKEN: ${{ secrets.GH_PACKAGES_TOKEN }}
steps:
- name: Checkout repository
uses: actions/checkout@v5
with:
persist-credentials: false
# - uses: https://github.com/actions/create-github-app-token@v2
# id: app-token
# with:
# app-id: ${{ vars.GH_APP_ID }}
# private-key: ${{ secrets.GH_APP_PRIVATE_KEY }}
# github-api-url: https://api.github.com
# owner: continuwuity
# repositories: continuwuity
- name: Install regctl
uses: https://forgejo.ellis.link/continuwuation/regclient-actions/regctl-installer@main
with:

View File

@@ -7,6 +7,7 @@ on:
- "Cargo.lock"
- "Cargo.toml"
- "rust-toolchain.toml"
- "nix/**/*"
- ".forgejo/workflows/update-flake-hashes.yml"
jobs:
@@ -40,13 +41,22 @@ jobs:
echo "Base: $base"
echo "HEAD: $(git rev-parse HEAD)"
git diff --name-only $base HEAD > changed_files.txt
echo "files=$(cat changed_files.txt)" >> $FORGEJO_OUTPUT
echo "detected changes in $(cat changed_files.txt)"
# Join files with commas
files=$(paste -sd, changed_files.txt)
echo "files=$files" >> $FORGEJO_OUTPUT
- name: Debug output
run: |
echo "State of output"
echo "Changed files: ${{ steps.changes.outputs.files }}"
- name: Get new toolchain hash
if: contains(steps.changes.outputs.files, 'Cargo.toml') || contains(steps.changes.outputs.files, 'Cargo.lock') || contains(steps.changes.outputs.files, 'rust-toolchain.toml')
run: |
# Set the current sha256 to an empty hash to make `nix build` calculate a new one
awk '/fromToolchainFile *\{/{found=1; print; next} found && /sha256 =/{sub(/sha256 = .*/, "sha256 = pkgsHost.lib.fakeSha256;"); found=0} 1' flake.nix > temp.nix && mv temp.nix flake.nix
awk '/fromToolchainFile *\{/{found=1; print; next} found && /sha256 =/{sub(/sha256 = .*/, "sha256 = lib.fakeSha256;"); found=0} 1' nix/packages/rust.nix > temp.nix
mv temp.nix nix/packages/rust.nix
# Build continuwuity and filter for the new hash
# We do `|| true` because we want this to fail without stopping the workflow
@@ -54,19 +64,21 @@ jobs:
# Place the new hash in place of the empty hash
new_hash=$(cat new_toolchain_hash.txt)
sed -i "s|pkgsHost.lib.fakeSha256|\"$new_hash\"|" flake.nix
sed -i "s|lib.fakeSha256|\"$new_hash\"|" nix/packages/rust.nix
echo "New hash:"
awk -F'"' '/fromToolchainFile/{found=1; next} found && /sha256 =/{print $2; found=0}' flake.nix
awk -F'"' '/fromToolchainFile/{found=1; next} found && /sha256 =/{print $2; found=0}' nix/packages/rust.nix
echo "Expected new hash:"
cat new_toolchain_hash.txt
rm new_toolchain_hash.txt
- name: Get new rocksdb hash
if: contains(steps.changes.outputs.files, '.nix') || contains(steps.changes.outputs.files, 'flake.lock')
run: |
# Set the current sha256 to an empty hash to make `nix build` calculate a new one
awk '/repo = "rocksdb";/{found=1; print; next} found && /sha256 =/{sub(/sha256 = .*/, "sha256 = pkgsHost.lib.fakeSha256;"); found=0} 1' flake.nix > temp.nix && mv temp.nix flake.nix
awk '/repo = "rocksdb";/{found=1; print; next} found && /sha256 =/{sub(/sha256 = .*/, "sha256 = lib.fakeSha256;"); found=0} 1' nix/packages/rocksdb/package.nix > temp.nix
mv temp.nix nix/packages/rocksdb/package.nix
# Build continuwuity and filter for the new hash
# We do `|| true` because we want this to fail without stopping the workflow
@@ -74,17 +86,17 @@ jobs:
# Place the new hash in place of the empty hash
new_hash=$(cat new_rocksdb_hash.txt)
sed -i "s|pkgsHost.lib.fakeSha256|\"$new_hash\"|" flake.nix
sed -i "s|lib.fakeSha256|\"$new_hash\"|" nix/packages/rocksdb/package.nix
echo "New hash:"
awk -F'"' '/repo = "rocksdb";/{found=1; next} found && /sha256 =/{print $2; found=0}' flake.nix
awk -F'"' '/repo = "rocksdb";/{found=1; next} found && /sha256 =/{print $2; found=0}' nix/packages/rocksdb/package.nix
echo "Expected new hash:"
cat new_rocksdb_hash.txt
rm new_rocksdb_hash.txt
- name: Show diff
run: git diff flake.nix
run: git diff flake.nix nix
- name: Push changes
run: |

View File

@@ -7,7 +7,7 @@ default_stages:
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v5.0.0
rev: v6.0.0
hooks:
- id: fix-byte-order-marker
- id: check-case-conflict
@@ -23,7 +23,7 @@ repos:
- id: check-added-large-files
- repo: https://github.com/crate-ci/typos
rev: v1.26.0
rev: v1.39.0
hooks:
- id: typos
- id: typos

View File

@@ -1078,7 +1078,10 @@ ###### **Subcommands:**
* `delete` — - Deletes a single media file from our database and on the filesystem via a single MXC URL or event ID (not redacted)
* `delete-list` — - Deletes a codeblock list of MXC URLs from our database and on the filesystem. This will always ignore errors
* `delete-past-remote-media` - Deletes all remote (and optionally local) media created before or after [duration] time using filesystem metadata first created at date, or fallback to last modified date. This will always ignore errors by default
* `delete-past-remote-media` — Deletes all remote (and optionally local) media created before/after
[duration] ago, using filesystem metadata first created at date, or
fallback to last modified date. This will always ignore errors by
default.
* `delete-all-from-user` — - Deletes all the local media from a local user on our server. This will always ignore errors by default
* `delete-all-from-server` — - Deletes all remote media from the specified remote server. This will always ignore errors by default
* `get-file-info`
@@ -1110,13 +1113,25 @@ ## `admin media delete-list`
## `admin media delete-past-remote-media`
- Deletes all remote (and optionally local) media created before or after [duration] time using filesystem metadata first created at date, or fallback to last modified date. This will always ignore errors by default
Deletes all remote (and optionally local) media created before/after
[duration] ago, using filesystem metadata first created at date, or
fallback to last modified date. This will always ignore errors by
default.
* Examples:
* Delete all remote media older than a year:
`!admin media delete-past-remote-media -b 1y`
* Delete all remote and local media from 3 days ago, up until now:
`!admin media delete-past-remote-media -a 3d --yes-i-want-to-delete-local-media`
**Usage:** `admin media delete-past-remote-media [OPTIONS] <DURATION>`
###### **Arguments:**
* `<DURATION>` — - The relative time (e.g. 30s, 5m, 7d) within which to search
* `<DURATION>` — - The relative time (e.g. 30s, 5m, 7d) from now within which to search
###### **Options:**

View File

@@ -241,7 +241,7 @@ ## Documentation
### Code Comments
- Reference related documentation or parts of the specification
- When a task has multiple ways of being acheved, explain your reasoning for your decision
- When a task has multiple ways of being achieved, explain your reasoning for your decision
- Update comments when code changes
```rs

451
flake.lock generated
View File

@@ -1,94 +1,28 @@
{
"nodes": {
"attic": {
"inputs": {
"crane": "crane",
"flake-compat": "flake-compat",
"flake-parts": "flake-parts",
"nix-github-actions": "nix-github-actions",
"nixpkgs": "nixpkgs",
"nixpkgs-stable": "nixpkgs-stable"
},
"advisory-db": {
"flake": false,
"locked": {
"lastModified": 1758711588,
"narHash": "sha256-0nZlCCDC5PfndsQJXXtcyrtrfW49I3KadGMDlutzaGU=",
"owner": "zhaofengli",
"repo": "attic",
"rev": "12cbeca141f46e1ade76728bce8adc447f2166c6",
"lastModified": 1761112158,
"narHash": "sha256-RIXu/7eyKpQHjsPuAUODO81I4ni8f+WYSb7K4mTG6+0=",
"owner": "rustsec",
"repo": "advisory-db",
"rev": "58f3aaec0e1776f4a900737be8cd7cb00972210d",
"type": "github"
},
"original": {
"owner": "zhaofengli",
"ref": "main",
"repo": "attic",
"type": "github"
}
},
"cachix": {
"inputs": {
"devenv": "devenv",
"flake-compat": "flake-compat_2",
"git-hooks": "git-hooks",
"nixpkgs": "nixpkgs_2"
},
"locked": {
"lastModified": 1756385612,
"narHash": "sha256-+NU5MMhuPHHRyvZZWNFG7zt+leRSPsJu1MwhOUzkPUk=",
"owner": "cachix",
"repo": "cachix",
"rev": "dc24688cd67518c3711d511fa369c0f5a131063a",
"type": "github"
},
"original": {
"owner": "cachix",
"ref": "master",
"repo": "cachix",
"type": "github"
}
},
"cachix_2": {
"inputs": {
"devenv": [
"cachix",
"devenv"
],
"flake-compat": [
"cachix",
"devenv"
],
"git-hooks": [
"cachix",
"devenv",
"git-hooks"
],
"nixpkgs": [
"cachix",
"devenv",
"nixpkgs"
]
},
"locked": {
"lastModified": 1748883665,
"narHash": "sha256-R0W7uAg+BLoHjMRMQ8+oiSbTq8nkGz5RDpQ+ZfxxP3A=",
"owner": "cachix",
"repo": "cachix",
"rev": "f707778d902af4d62d8dd92c269f8e70de09acbe",
"type": "github"
},
"original": {
"owner": "cachix",
"ref": "latest",
"repo": "cachix",
"owner": "rustsec",
"repo": "advisory-db",
"type": "github"
}
},
"crane": {
"locked": {
"lastModified": 1751562746,
"narHash": "sha256-smpugNIkmDeicNz301Ll1bD7nFOty97T79m4GUMUczA=",
"lastModified": 1760924934,
"narHash": "sha256-tuuqY5aU7cUkR71sO2TraVKK2boYrdW3gCSXUkF4i44=",
"owner": "ipetkov",
"repo": "crane",
"rev": "aed2020fd3dc26e1e857d4107a5a67a33ab6c1fd",
"rev": "c6b4d5308293d0d04fcfeee92705017537cad02f",
"type": "github"
},
"original": {
@@ -97,53 +31,6 @@
"type": "github"
}
},
"crane_2": {
"locked": {
"lastModified": 1759893430,
"narHash": "sha256-yAy4otLYm9iZ+NtQwTMEbqHwswSFUbhn7x826RR6djw=",
"owner": "ipetkov",
"repo": "crane",
"rev": "1979a2524cb8c801520bd94c38bb3d5692419d93",
"type": "github"
},
"original": {
"owner": "ipetkov",
"ref": "master",
"repo": "crane",
"type": "github"
}
},
"devenv": {
"inputs": {
"cachix": "cachix_2",
"flake-compat": [
"cachix",
"flake-compat"
],
"git-hooks": [
"cachix",
"git-hooks"
],
"nix": "nix",
"nixpkgs": [
"cachix",
"nixpkgs"
]
},
"locked": {
"lastModified": 1754404745,
"narHash": "sha256-BdbW/iTImczgcuATgQIa9sPGuYIBxVq2xqcvICsa2AQ=",
"owner": "cachix",
"repo": "devenv",
"rev": "6563b21105168f90394dfaf58284b078af2d7275",
"type": "github"
},
"original": {
"owner": "cachix",
"repo": "devenv",
"type": "github"
}
},
"fenix": {
"inputs": {
"nixpkgs": [
@@ -152,53 +39,20 @@
"rust-analyzer-src": "rust-analyzer-src"
},
"locked": {
"lastModified": 1760510549,
"narHash": "sha256-NP+kmLMm7zSyv4Fufv+eSJXyqjLMUhUfPT6lXRlg/bU=",
"lastModified": 1761115517,
"narHash": "sha256-Fev/ag/c3Fp3JBwHfup3lpA5FlNXfkoshnQ7dssBgJ0=",
"owner": "nix-community",
"repo": "fenix",
"rev": "ef7178cf086f267113b5c48fdeb6e510729c8214",
"rev": "320433651636186ea32b387cff05d6bbfa30cea7",
"type": "github"
},
"original": {
"owner": "nix-community",
"ref": "main",
"repo": "fenix",
"type": "github"
}
},
"flake-compat": {
"flake": false,
"locked": {
"lastModified": 1747046372,
"narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-compat_2": {
"flake": false,
"locked": {
"lastModified": 1747046372,
"narHash": "sha256-CIVLLkVgvHYbgI2UpXvIIBJ12HWgX+fjA8Xf8PUmqCY=",
"owner": "edolstra",
"repo": "flake-compat",
"rev": "9100a0f413b0c601e0533d1d94ffd501ce2e7885",
"type": "github"
},
"original": {
"owner": "edolstra",
"repo": "flake-compat",
"type": "github"
}
},
"flake-compat_3": {
"flake": false,
"locked": {
"lastModified": 1747046372,
@@ -217,17 +71,14 @@
},
"flake-parts": {
"inputs": {
"nixpkgs-lib": [
"attic",
"nixpkgs"
]
"nixpkgs-lib": "nixpkgs-lib"
},
"locked": {
"lastModified": 1751413152,
"narHash": "sha256-Tyw1RjYEsp5scoigs1384gIg6e0GoBVjms4aXFfRssQ=",
"lastModified": 1760948891,
"narHash": "sha256-TmWcdiUUaWk8J4lpjzu4gCGxWY6/Ok7mOK4fIFfBuU4=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "77826244401ea9de6e3bac47c2db46005e1f30b5",
"rev": "864599284fc7c0ba6357ed89ed5e2cd5040f0c04",
"type": "github"
},
"original": {
@@ -236,214 +87,13 @@
"type": "github"
}
},
"flake-parts_2": {
"inputs": {
"nixpkgs-lib": [
"cachix",
"devenv",
"nix",
"nixpkgs"
]
},
"locked": {
"lastModified": 1733312601,
"narHash": "sha256-4pDvzqnegAfRkPwO3wmwBhVi/Sye1mzps0zHWYnP88c=",
"owner": "hercules-ci",
"repo": "flake-parts",
"rev": "205b12d8b7cd4802fbcb8e8ef6a0f1408781a4f9",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "flake-parts",
"type": "github"
}
},
"flake-utils": {
"inputs": {
"systems": "systems"
},
"locked": {
"lastModified": 1731533236,
"narHash": "sha256-l0KFg5HjrsfsO/JpG+r7fRrqm12kzFHyUHqHCVpMMbI=",
"owner": "numtide",
"repo": "flake-utils",
"rev": "11707dc2f618dd54ca8739b309ec4fc024de578b",
"type": "github"
},
"original": {
"owner": "numtide",
"ref": "main",
"repo": "flake-utils",
"type": "github"
}
},
"git-hooks": {
"inputs": {
"flake-compat": [
"cachix",
"flake-compat"
],
"gitignore": "gitignore",
"nixpkgs": [
"cachix",
"nixpkgs"
]
},
"locked": {
"lastModified": 1750779888,
"narHash": "sha256-wibppH3g/E2lxU43ZQHC5yA/7kIKLGxVEnsnVK1BtRg=",
"owner": "cachix",
"repo": "git-hooks.nix",
"rev": "16ec914f6fb6f599ce988427d9d94efddf25fe6d",
"type": "github"
},
"original": {
"owner": "cachix",
"repo": "git-hooks.nix",
"type": "github"
}
},
"gitignore": {
"inputs": {
"nixpkgs": [
"cachix",
"git-hooks",
"nixpkgs"
]
},
"locked": {
"lastModified": 1709087332,
"narHash": "sha256-HG2cCnktfHsKV0s4XW83gU3F57gaTljL9KNSuG6bnQs=",
"owner": "hercules-ci",
"repo": "gitignore.nix",
"rev": "637db329424fd7e46cf4185293b9cc8c88c95394",
"type": "github"
},
"original": {
"owner": "hercules-ci",
"repo": "gitignore.nix",
"type": "github"
}
},
"nix": {
"inputs": {
"flake-compat": [
"cachix",
"devenv",
"flake-compat"
],
"flake-parts": "flake-parts_2",
"git-hooks-nix": [
"cachix",
"devenv",
"git-hooks"
],
"nixpkgs": [
"cachix",
"devenv",
"nixpkgs"
],
"nixpkgs-23-11": [
"cachix",
"devenv"
],
"nixpkgs-regression": [
"cachix",
"devenv"
]
},
"locked": {
"lastModified": 1752773918,
"narHash": "sha256-dOi/M6yNeuJlj88exI+7k154z+hAhFcuB8tZktiW7rg=",
"owner": "cachix",
"repo": "nix",
"rev": "031c3cf42d2e9391eee373507d8c12e0f9606779",
"type": "github"
},
"original": {
"owner": "cachix",
"ref": "devenv-2.30",
"repo": "nix",
"type": "github"
}
},
"nix-filter": {
"locked": {
"lastModified": 1757882181,
"narHash": "sha256-+cCxYIh2UNalTz364p+QYmWHs0P+6wDhiWR4jDIKQIU=",
"owner": "numtide",
"repo": "nix-filter",
"rev": "59c44d1909c72441144b93cf0f054be7fe764de5",
"type": "github"
},
"original": {
"owner": "numtide",
"ref": "main",
"repo": "nix-filter",
"type": "github"
}
},
"nix-github-actions": {
"inputs": {
"nixpkgs": [
"attic",
"nixpkgs"
]
},
"locked": {
"lastModified": 1737420293,
"narHash": "sha256-F1G5ifvqTpJq7fdkT34e/Jy9VCyzd5XfJ9TO8fHhJWE=",
"owner": "nix-community",
"repo": "nix-github-actions",
"rev": "f4158fa080ef4503c8f4c820967d946c2af31ec9",
"type": "github"
},
"original": {
"owner": "nix-community",
"repo": "nix-github-actions",
"type": "github"
}
},
"nixpkgs": {
"locked": {
"lastModified": 1751949589,
"narHash": "sha256-mgFxAPLWw0Kq+C8P3dRrZrOYEQXOtKuYVlo9xvPntt8=",
"lastModified": 1760878510,
"narHash": "sha256-K5Osef2qexezUfs0alLvZ7nQFTGS9DL2oTVsIXsqLgs=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "9b008d60392981ad674e04016d25619281550a9d",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs-stable": {
"locked": {
"lastModified": 1751741127,
"narHash": "sha256-t75Shs76NgxjZSgvvZZ9qOmz5zuBE8buUaYD28BMTxg=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "29e290002bfff26af1db6f64d070698019460302",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixos-25.05",
"repo": "nixpkgs",
"type": "github"
}
},
"nixpkgs_2": {
"locked": {
"lastModified": 1754214453,
"narHash": "sha256-Q/I2xJn/j1wpkGhWkQnm20nShYnG7TI99foDBpXm1SY=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "5b09dc45f24cf32316283e62aec81ffee3c3e376",
"rev": "5e2a59a5b1a82f89f2c7e598302a9cacebb72a67",
"type": "github"
},
"original": {
@@ -453,42 +103,40 @@
"type": "github"
}
},
"nixpkgs_3": {
"nixpkgs-lib": {
"locked": {
"lastModified": 1760504863,
"narHash": "sha256-h13YFQMi91nXkkRoJMIfezorz5SbD6849jw5L0fjK4I=",
"owner": "NixOS",
"repo": "nixpkgs",
"rev": "82c2e0d6dde50b17ae366d2aa36f224dc19af469",
"lastModified": 1754788789,
"narHash": "sha256-x2rJ+Ovzq0sCMpgfgGaaqgBSwY+LST+WbZ6TytnT9Rk=",
"owner": "nix-community",
"repo": "nixpkgs.lib",
"rev": "a73b9c743612e4244d865a2fdee11865283c04e6",
"type": "github"
},
"original": {
"owner": "NixOS",
"ref": "nixpkgs-unstable",
"repo": "nixpkgs",
"owner": "nix-community",
"repo": "nixpkgs.lib",
"type": "github"
}
},
"root": {
"inputs": {
"attic": "attic",
"cachix": "cachix",
"crane": "crane_2",
"advisory-db": "advisory-db",
"crane": "crane",
"fenix": "fenix",
"flake-compat": "flake-compat_3",
"flake-utils": "flake-utils",
"nix-filter": "nix-filter",
"nixpkgs": "nixpkgs_3"
"flake-compat": "flake-compat",
"flake-parts": "flake-parts",
"nixpkgs": "nixpkgs",
"treefmt-nix": "treefmt-nix"
}
},
"rust-analyzer-src": {
"flake": false,
"locked": {
"lastModified": 1760457219,
"narHash": "sha256-WJOUGx42hrhmvvYcGkwea+BcJuQJLcns849OnewQqX4=",
"lastModified": 1761077270,
"narHash": "sha256-O1uTuvI/rUlubJ8AXKyzh1WSWV3qCZX0huTFUvWLN4E=",
"owner": "rust-lang",
"repo": "rust-analyzer",
"rev": "8747cf81540bd1bbbab9ee2702f12c33aa887b46",
"rev": "39990a923c8bca38f5bd29dc4c96e20ee7808d5d",
"type": "github"
},
"original": {
@@ -498,18 +146,23 @@
"type": "github"
}
},
"systems": {
"treefmt-nix": {
"inputs": {
"nixpkgs": [
"nixpkgs"
]
},
"locked": {
"lastModified": 1681028828,
"narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=",
"owner": "nix-systems",
"repo": "default",
"rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e",
"lastModified": 1760945191,
"narHash": "sha256-ZRVs8UqikBa4Ki3X4KCnMBtBW0ux1DaT35tgsnB1jM4=",
"owner": "numtide",
"repo": "treefmt-nix",
"rev": "f56b1934f5f8fcab8deb5d38d42fd692632b47c2",
"type": "github"
},
"original": {
"owner": "nix-systems",
"repo": "default",
"owner": "numtide",
"repo": "treefmt-nix",
"type": "github"
}
}

371
flake.nix
View File

@@ -1,351 +1,46 @@
{
description = "A nix flake for the continuwuity project";
inputs = {
attic.url = "github:zhaofengli/attic?ref=main";
cachix.url = "github:cachix/cachix?ref=master";
crane = {
url = "github:ipetkov/crane?ref=master";
};
# basics
flake-parts.url = "github:hercules-ci/flake-parts";
nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable";
# for rust via nix
crane.url = "github:ipetkov/crane";
fenix = {
url = "github:nix-community/fenix?ref=main";
url = "github:nix-community/fenix";
inputs.nixpkgs.follows = "nixpkgs";
};
# for vuln checks
advisory-db = {
url = "github:rustsec/advisory-db";
flake = false;
};
treefmt-nix = {
url = "github:numtide/treefmt-nix";
inputs.nixpkgs.follows = "nixpkgs";
};
# for default.nix
flake-compat = {
url = "github:edolstra/flake-compat?ref=master";
flake = false;
};
flake-utils.url = "github:numtide/flake-utils?ref=main";
nix-filter.url = "github:numtide/nix-filter?ref=main";
nixpkgs.url = "github:NixOS/nixpkgs?ref=nixpkgs-unstable";
};
outputs =
inputs:
inputs.flake-utils.lib.eachDefaultSystem (
system:
let
pkgsHost = import inputs.nixpkgs {
inherit system;
};
fnx = inputs.fenix.packages.${system};
# The Rust toolchain to use
toolchain = fnx.combine [
(fnx.fromToolchainFile {
file = ./rust-toolchain.toml;
# See also `rust-toolchain.toml`
sha256 = "sha256-+9FmLhAOezBZCOziO0Qct1NOrfpjNsXxc/8I0c7BdKE=";
})
fnx.complete.rustfmt
];
mkScope =
pkgs:
pkgs.lib.makeScope pkgs.newScope (self: {
inherit pkgs inputs;
craneLib = (inputs.crane.mkLib pkgs).overrideToolchain (_: toolchain);
main = self.callPackage ./pkg/nix/pkgs/main { };
liburing = pkgs.liburing.overrideAttrs {
# Tests weren't building
outputs = [
"out"
"dev"
"man"
];
buildFlags = [ "library" ];
};
rocksdb =
(pkgs.rocksdb_9_10.override {
# Override the liburing input for the build with our own so
# we have it built with the library flag
inherit (self) liburing;
}).overrideAttrs
(old: {
src = pkgsHost.fetchFromGitea {
domain = "forgejo.ellis.link";
owner = "continuwuation";
repo = "rocksdb";
rev = "10.5.fb";
sha256 = "sha256-X4ApGLkHF9ceBtBg77dimEpu720I79ffLoyPa8JMHaU=";
};
version = "v10.5.fb";
cmakeFlags =
pkgs.lib.subtractLists [
# No real reason to have snappy or zlib, no one uses this
"-DWITH_SNAPPY=1"
"-DZLIB=1"
"-DWITH_ZLIB=1"
# We don't need to use ldb or sst_dump (core_tools)
"-DWITH_CORE_TOOLS=1"
# We don't need to build rocksdb tests
"-DWITH_TESTS=1"
# We use rust-rocksdb via C interface and don't need C++ RTTI
"-DUSE_RTTI=1"
# This doesn't exist in RocksDB, and USE_SSE is deprecated for
# PORTABLE=$(march)
"-DFORCE_SSE42=1"
# PORTABLE will get set in main/default.nix
"-DPORTABLE=1"
] old.cmakeFlags
++ [
# No real reason to have snappy, no one uses this
"-DWITH_SNAPPY=0"
"-DZLIB=0"
"-DWITH_ZLIB=0"
# We don't need to use ldb or sst_dump (core_tools)
"-DWITH_CORE_TOOLS=0"
# We don't need trace tools
"-DWITH_TRACE_TOOLS=0"
# We don't need to build rocksdb tests
"-DWITH_TESTS=0"
# We use rust-rocksdb via C interface and don't need C++ RTTI
"-DUSE_RTTI=0"
];
# outputs has "tools" which we don't need or use
outputs = [ "out" ];
# preInstall hooks has stuff for messing with ldb/sst_dump which we don't need or use
preInstall = "";
# We have this already at https://forgejo.ellis.link/continuwuation/rocksdb/commit/a935c0273e1ba44eacf88ce3685a9b9831486155
# Unsetting this so we don't have to revert it and make this nix exclusive
patches = [ ];
postPatch = ''
# Fix gcc-13 build failures due to missing <cstdint> and
# <system_error> includes, fixed upstream since 8.x
sed -e '1i #include <cstdint>' -i db/compaction/compaction_iteration_stats.h
sed -e '1i #include <cstdint>' -i table/block_based/data_block_hash_index.h
sed -e '1i #include <cstdint>' -i util/string_util.h
sed -e '1i #include <cstdint>' -i include/rocksdb/utilities/checkpoint.h
'';
});
});
scopeHost = mkScope pkgsHost;
mkCrossScope =
crossSystem:
let
pkgsCrossStatic =
(import inputs.nixpkgs {
inherit system;
crossSystem = {
config = crossSystem;
};
}).pkgsStatic;
in
mkScope pkgsCrossStatic;
in
{
packages =
{
default = scopeHost.main.override {
disable_features = [
# Don't include experimental features
"experimental"
# jemalloc profiling/stats features are expensive and shouldn't
# be expected on non-debug builds.
"jemalloc_prof"
"jemalloc_stats"
# This is non-functional on nix for some reason
"hardened_malloc"
# conduwuit_mods is a development-only hot reload feature
"conduwuit_mods"
];
};
default-debug = scopeHost.main.override {
profile = "dev";
# Debug build users expect full logs
disable_release_max_log_level = true;
disable_features = [
# Don't include experimental features
"experimental"
# This is non-functional on nix for some reason
"hardened_malloc"
# conduwuit_mods is a development-only hot reload feature
"conduwuit_mods"
];
};
# Just a test profile used for things like CI and complement
default-test = scopeHost.main.override {
profile = "test";
disable_release_max_log_level = true;
disable_features = [
# Don't include experimental features
"experimental"
# this is non-functional on nix for some reason
"hardened_malloc"
# conduwuit_mods is a development-only hot reload feature
"conduwuit_mods"
];
};
all-features = scopeHost.main.override {
all_features = true;
disable_features = [
# Don't include experimental features
"experimental"
# jemalloc profiling/stats features are expensive and shouldn't
# be expected on non-debug builds.
"jemalloc_prof"
"jemalloc_stats"
# This is non-functional on nix for some reason
"hardened_malloc"
# conduwuit_mods is a development-only hot reload feature
"conduwuit_mods"
];
};
all-features-debug = scopeHost.main.override {
profile = "dev";
all_features = true;
# Debug build users expect full logs
disable_release_max_log_level = true;
disable_features = [
# Don't include experimental features
"experimental"
# This is non-functional on nix for some reason
"hardened_malloc"
# conduwuit_mods is a development-only hot reload feature
"conduwuit_mods"
];
};
hmalloc = scopeHost.main.override { features = [ "hardened_malloc" ]; };
}
// builtins.listToAttrs (
builtins.concatLists (
builtins.map
(
crossSystem:
let
binaryName = "static-${crossSystem}";
scopeCrossStatic = mkCrossScope crossSystem;
in
[
# An output for a statically-linked binary
{
name = binaryName;
value = scopeCrossStatic.main;
}
# An output for a statically-linked binary with x86_64 haswell
# target optimisations
{
name = "${binaryName}-x86_64-haswell-optimised";
value = scopeCrossStatic.main.override {
x86_64_haswell_target_optimised =
if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false;
};
}
# An output for a statically-linked unstripped debug ("dev") binary
{
name = "${binaryName}-debug";
value = scopeCrossStatic.main.override {
profile = "dev";
# debug build users expect full logs
disable_release_max_log_level = true;
};
}
# An output for a statically-linked unstripped debug binary with the
# "test" profile (for CI usage only)
{
name = "${binaryName}-test";
value = scopeCrossStatic.main.override {
profile = "test";
disable_release_max_log_level = true;
disable_features = [
# dont include experimental features
"experimental"
# this is non-functional on nix for some reason
"hardened_malloc"
# conduwuit_mods is a development-only hot reload feature
"conduwuit_mods"
];
};
}
# An output for a statically-linked binary with `--all-features`
{
name = "${binaryName}-all-features";
value = scopeCrossStatic.main.override {
all_features = true;
disable_features = [
# dont include experimental features
"experimental"
# jemalloc profiling/stats features are expensive and shouldn't
# be expected on non-debug builds.
"jemalloc_prof"
"jemalloc_stats"
# this is non-functional on nix for some reason
"hardened_malloc"
# conduwuit_mods is a development-only hot reload feature
"conduwuit_mods"
];
};
}
# An output for a statically-linked binary with `--all-features` and with x86_64 haswell
# target optimisations
{
name = "${binaryName}-all-features-x86_64-haswell-optimised";
value = scopeCrossStatic.main.override {
all_features = true;
disable_features = [
# dont include experimental features
"experimental"
# jemalloc profiling/stats features are expensive and shouldn't
# be expected on non-debug builds.
"jemalloc_prof"
"jemalloc_stats"
# this is non-functional on nix for some reason
"hardened_malloc"
# conduwuit_mods is a development-only hot reload feature
"conduwuit_mods"
];
x86_64_haswell_target_optimised =
if (crossSystem == "x86_64-linux-gnu" || crossSystem == "x86_64-linux-musl") then true else false;
};
}
# An output for a statically-linked unstripped debug ("dev") binary with `--all-features`
{
name = "${binaryName}-all-features-debug";
value = scopeCrossStatic.main.override {
profile = "dev";
all_features = true;
# debug build users expect full logs
disable_release_max_log_level = true;
disable_features = [
# dont include experimental features
"experimental"
# this is non-functional on nix for some reason
"hardened_malloc"
# conduwuit_mods is a development-only hot reload feature
"conduwuit_mods"
];
};
}
# An output for a statically-linked binary with hardened_malloc
{
name = "${binaryName}-hmalloc";
value = scopeCrossStatic.main.override {
features = [ "hardened_malloc" ];
};
}
]
)
[
#"x86_64-apple-darwin"
#"aarch64-apple-darwin"
"x86_64-linux-gnu"
"x86_64-linux-musl"
"aarch64-linux-musl"
]
)
);
}
);
inputs@{ flake-parts, ... }:
flake-parts.lib.mkFlake { inherit inputs; } {
imports = [ ./nix ];
systems = [
# good support
"x86_64-linux"
# support untested but theoretically there
"aarch64-linux"
];
};
}

108
nix/checks/default.nix Normal file
View File

@@ -0,0 +1,108 @@
{ inputs, ... }:
{
perSystem =
{
self',
lib,
pkgs,
...
}:
let
uwulib = inputs.self.uwulib.init pkgs;
rocksdbAllFeatures = self'.packages.rocksdb.override {
enableJemalloc = true;
enableLiburing = true;
};
commonAttrs = (uwulib.build.commonAttrs { }) // {
buildInputs = [
pkgs.liburing
pkgs.rust-jemalloc-sys-unprefixed
rocksdbAllFeatures
];
nativeBuildInputs = [
pkgs.pkg-config
# bindgen needs the build platform's libclang. Apparently due to "splicing
# weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't quite do the
# right thing here.
pkgs.rustPlatform.bindgenHook
];
env = {
LIBCLANG_PATH = lib.makeLibraryPath [ pkgs.llvmPackages.libclang.lib ];
LD_LIBRARY_PATH = lib.makeLibraryPath [
pkgs.liburing
pkgs.rust-jemalloc-sys-unprefixed
rocksdbAllFeatures
];
}
// uwulib.environment.buildPackageEnv
// {
ROCKSDB_INCLUDE_DIR = "${rocksdbAllFeatures}/include";
ROCKSDB_LIB_DIR = "${rocksdbAllFeatures}/lib";
};
};
cargoArtifacts = self'.packages.continuwuity-all-features-deps;
in
{
# taken from
#
# https://crane.dev/examples/quick-start.html
checks = {
continuwuity-all-features-build = self'.packages.continuwuity-all-features-bin;
continuwuity-all-features-clippy = uwulib.build.craneLibForChecks.cargoClippy (
commonAttrs
// {
inherit cargoArtifacts;
cargoClippyExtraArgs = "-- --deny warnings";
}
);
continuwuity-all-features-docs = uwulib.build.craneLibForChecks.cargoDoc (
commonAttrs
// {
inherit cargoArtifacts;
# This can be commented out or tweaked as necessary, e.g. set to
# `--deny rustdoc::broken-intra-doc-links` to only enforce that lint
env.RUSTDOCFLAGS = "--deny warnings";
}
);
# Check formatting
continuwuity-all-features-fmt = uwulib.build.craneLibForChecks.cargoFmt {
src = uwulib.build.src;
};
continuwuity-all-features-toml-fmt = uwulib.build.craneLibForChecks.taploFmt {
src = pkgs.lib.sources.sourceFilesBySuffices uwulib.build.src [ ".toml" ];
# taplo arguments can be further customized below as needed
taploExtraArgs = "--config ${inputs.self}/taplo.toml";
};
# Audit dependencies
continuwuity-all-features-audit = uwulib.build.craneLibForChecks.cargoAudit {
inherit (inputs) advisory-db;
src = uwulib.build.src;
};
# Audit licenses
continuwuity-all-features-deny = uwulib.build.craneLibForChecks.cargoDeny {
src = uwulib.build.src;
};
# Run tests with cargo-nextest
# Consider setting `doCheck = false` on `continuwuity-all-features` if you do not want
# the tests to run twice
continuwuity-all-features-nextest = uwulib.build.craneLibForChecks.cargoNextest (
commonAttrs
// {
inherit cargoArtifacts;
partitions = 1;
partitionType = "count";
cargoNextestPartitionsExtraArgs = "--no-tests=pass";
}
);
};
};
}

11
nix/default.nix Normal file
View File

@@ -0,0 +1,11 @@
{
imports = [
./checks
./packages
./shells
./tests
./hydra.nix
./fmt.nix
];
}

37
nix/fmt.nix Normal file
View File

@@ -0,0 +1,37 @@
{ inputs, ... }:
{
# load the flake module from upstream
imports = [ inputs.treefmt-nix.flakeModule ];
perSystem =
{ self', lib, ... }:
{
treefmt = {
# repo root as project root
projectRoot = inputs.self;
# the formatters
programs = {
nixfmt.enable = true;
typos = {
enable = true;
configFile = "${inputs.self}/.typos.toml";
};
taplo = {
enable = true;
settings = lib.importTOML "${inputs.self}/taplo.toml";
};
};
settings.formatter.rustfmt = {
command = "${lib.getExe' self'.packages.dev-toolchain "rustfmt"}";
includes = [ "**/*.rs" ];
options = [
"--unstable-features"
"--edition=2024"
"--config-path=${inputs.self}/rustfmt.toml"
];
};
};
};
}

9
nix/hydra.nix Normal file
View File

@@ -0,0 +1,9 @@
{ inputs, ... }:
let
lib = inputs.nixpkgs.lib;
in
{
flake.hydraJobs.packages = builtins.mapAttrs (
_name: lib.hydraJob
) inputs.self.packages.x86_64-linux;
}

View File

@@ -0,0 +1,60 @@
{ inputs, ... }:
{
perSystem =
{
self',
lib,
pkgs,
...
}:
let
uwulib = inputs.self.uwulib.init pkgs;
in
{
packages =
lib.pipe
[
# this is the default variant
{
variantName = "default";
commonAttrsArgs.profile = "release";
rocksdb = self'.packages.rocksdb;
features = { };
}
# this is the variant with all features enabled (liburing + jemalloc)
{
variantName = "all-features";
commonAttrsArgs.profile = "release";
rocksdb = self'.packages.rocksdb.override {
enableJemalloc = true;
enableLiburing = true;
};
features = {
enabledFeatures = "all";
disabledFeatures = uwulib.features.defaultDisabledFeatures ++ [ "bindgen-static" ];
};
}
]
[
(builtins.map (cfg: rec {
deps = {
name = "continuwuity-${cfg.variantName}-deps";
value = uwulib.build.buildDeps {
features = uwulib.features.calcFeatures cfg.features;
inherit (cfg) commonAttrsArgs rocksdb;
};
};
bin = {
name = "continuwuity-${cfg.variantName}-bin";
value = uwulib.build.buildPackage {
deps = self'.packages.${deps.name};
features = uwulib.features.calcFeatures cfg.features;
inherit (cfg) commonAttrsArgs rocksdb;
};
};
}))
(builtins.concatMap builtins.attrValues)
builtins.listToAttrs
];
};
}

14
nix/packages/default.nix Normal file
View File

@@ -0,0 +1,14 @@
{
imports = [
./continuwuity
./rocksdb
./rust.nix
./uwulib
];
perSystem =
{ self', ... }:
{
packages.default = self'.packages.continuwuity-default-bin;
};
}

View File

@@ -0,0 +1,12 @@
{
perSystem =
{
pkgs,
...
}:
{
packages = {
rocksdb = pkgs.callPackage ./package.nix { };
};
};
}

View File

@@ -0,0 +1,88 @@
{
lib,
stdenv,
rocksdb,
liburing,
rust-jemalloc-sys-unprefixed,
enableJemalloc ? false,
enableLiburing ? false,
fetchFromGitea,
...
}:
let
notDarwin = !stdenv.hostPlatform.isDarwin;
in
(rocksdb.override {
# Override the liburing input for the build with our own so
# we have it built with the library flag
inherit liburing;
jemalloc = rust-jemalloc-sys-unprefixed;
# rocksdb fails to build with prefixed jemalloc, which is required on
# darwin due to [1]. In this case, fall back to building rocksdb with
# libc malloc. This should not cause conflicts, because all of the
# jemalloc symbols are prefixed.
#
# [1]: https://github.com/tikv/jemallocator/blob/ab0676d77e81268cd09b059260c75b38dbef2d51/jemalloc-sys/src/env.rs#L17
enableJemalloc = enableJemalloc && notDarwin;
# for some reason enableLiburing in nixpkgs rocksdb is default true
# which breaks Darwin entirely
enableLiburing = enableLiburing && notDarwin;
}).overrideAttrs
(old: {
src = fetchFromGitea {
domain = "forgejo.ellis.link";
owner = "continuwuation";
repo = "rocksdb";
rev = "10.5.fb";
sha256 = "sha256-X4ApGLkHF9ceBtBg77dimEpu720I79ffLoyPa8JMHaU=";
};
version = "10.5.fb";
cmakeFlags =
lib.subtractLists (builtins.map (flag: lib.cmakeBool flag true) [
# No real reason to have snappy or zlib, no one uses this
"WITH_SNAPPY"
"ZLIB"
"WITH_ZLIB"
# We don't need to use ldb or sst_dump (core_tools)
"WITH_CORE_TOOLS"
# We don't need to build rocksdb tests
"WITH_TESTS"
# We use rust-rocksdb via C interface and don't need C++ RTTI
"USE_RTTI"
# This doesn't exist in RocksDB, and USE_SSE is deprecated for
# PORTABLE=$(march)
"FORCE_SSE42"
]) old.cmakeFlags
++ (builtins.map (flag: lib.cmakeBool flag false) [
# No real reason to have snappy, no one uses this
"WITH_SNAPPY"
"ZLIB"
"WITH_ZLIB"
# We don't need to use ldb or sst_dump (core_tools)
"WITH_CORE_TOOLS"
# We don't need trace tools
"WITH_TRACE_TOOLS"
# We don't need to build rocksdb tests
"WITH_TESTS"
# We use rust-rocksdb via C interface and don't need C++ RTTI
"USE_RTTI"
]);
enableLiburing = enableLiburing && notDarwin;
# outputs has "tools" which we don't need or use
outputs = [ "out" ];
# preInstall hooks has stuff for messing with ldb/sst_dump which we don't need or use
preInstall = "";
# We have this already at https://forgejo.ellis.link/continuwuation/rocksdb/commit/a935c0273e1ba44eacf88ce3685a9b9831486155
# Unsetting `patches` so we don't have to revert it and make this nix exclusive
patches = [ ];
})

32
nix/packages/rust.nix Normal file
View File

@@ -0,0 +1,32 @@
{ inputs, ... }:
{
perSystem =
{
system,
lib,
...
}:
{
packages =
let
fnx = inputs.fenix.packages.${system};
stable = fnx.fromToolchainFile {
file = inputs.self + "/rust-toolchain.toml";
# See also `rust-toolchain.toml`
sha256 = "sha256-+9FmLhAOezBZCOziO0Qct1NOrfpjNsXxc/8I0c7BdKE=";
};
in
{
# used for building nix stuff (doesn't include rustfmt overhead)
build-toolchain = stable;
# used for dev shells
dev-toolchain = fnx.combine [
stable
# use the nightly rustfmt because we use nightly features
fnx.complete.rustfmt
];
};
};
}

View File

@@ -0,0 +1,108 @@
args@{ pkgs, inputs, ... }:
let
inherit (pkgs) lib;
uwuenv = import ./environment.nix args;
selfpkgs = inputs.self.packages.${pkgs.stdenv.system};
in
rec {
# basic, very minimal instance of the crane library with a minimal rust toolchain
craneLib = (inputs.crane.mkLib pkgs).overrideToolchain (_: selfpkgs.build-toolchain);
# the checks require more rust toolchain components, hence we have this separate instance of the crane library
craneLibForChecks = (inputs.crane.mkLib pkgs).overrideToolchain (_: selfpkgs.dev-toolchain);
# meta information (name, version, etc) of the rust crate based on the Cargo.toml
crateInfo = craneLib.crateNameFromCargoToml { cargoToml = "${inputs.self}/Cargo.toml"; };
src =
let
# see https://crane.dev/API.html#cranelibfiltercargosources
#
# we need to keep the `web` directory which would be filtered out by the regular source filtering function
#
# https://crane.dev/API.html#cranelibcleancargosource
isWebTemplate = path: _type: builtins.match ".*src/web.*" path != null;
isRust = craneLib.filterCargoSources;
isNix = path: _type: builtins.match ".+/nix.*" path != null;
webOrRustNotNix = p: t: !(isNix p t) && (isWebTemplate p t || isRust p t);
in
lib.cleanSourceWith {
src = inputs.self;
filter = webOrRustNotNix;
name = "source";
};
# common attrs that are shared between building continuwuity's deps and the package itself
commonAttrs =
{
profile ? "dev",
...
}:
{
inherit (crateInfo)
pname
version
;
inherit src;
# this prevents unnecessary rebuilds
strictDeps = true;
dontStrip = profile == "dev" || profile == "test";
dontPatchELF = profile == "dev" || profile == "test";
doCheck = true;
nativeBuildInputs = [
# bindgen needs the build platform's libclang. Apparently due to "splicing
# weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't quite do the
# right thing here.
pkgs.rustPlatform.bindgenHook
];
};
makeRocksDBEnv =
{ rocksdb }:
{
ROCKSDB_INCLUDE_DIR = "${rocksdb}/include";
ROCKSDB_LIB_DIR = "${rocksdb}/lib";
};
# function that builds the continuwuity dependencies derivation
buildDeps =
{
rocksdb,
features,
commonAttrsArgs,
}:
craneLib.buildDepsOnly (
(commonAttrs commonAttrsArgs)
// {
env = uwuenv.buildDepsOnlyEnv // (makeRocksDBEnv { inherit rocksdb; });
inherit (features) cargoExtraArgs;
}
);
# function that builds the continuwuity package
buildPackage =
{
deps,
rocksdb,
features,
commonAttrsArgs,
}:
let
rocksdbEnv = makeRocksDBEnv { inherit rocksdb; };
in
craneLib.buildPackage (
(commonAttrs commonAttrsArgs)
// {
cargoArtifacts = deps;
doCheck = true;
env = uwuenv.buildPackageEnv // rocksdbEnv;
passthru.env = uwuenv.buildPackageEnv // rocksdbEnv;
meta.mainProgram = crateInfo.pname;
inherit (features) cargoExtraArgs;
}
);
}

View File

@@ -0,0 +1,10 @@
{ inputs, ... }:
{
flake.uwulib = {
init = pkgs: {
features = import ./features.nix { inherit pkgs inputs; };
environment = import ./environment.nix { inherit pkgs inputs; };
build = import ./build.nix { inherit pkgs inputs; };
};
};
}

View File

@@ -0,0 +1,18 @@
args@{ pkgs, inputs, ... }:
let
uwubuild = import ./build.nix args;
in
rec {
buildDepsOnlyEnv = {
# https://crane.dev/faq/rebuilds-bindgen.html
NIX_OUTPATH_USED_AS_RANDOM_SEED = "aaaaaaaaaa";
CARGO_PROFILE = "release";
}
// uwubuild.craneLib.mkCrossToolchainEnv (p: pkgs.clangStdenv);
buildPackageEnv = {
GIT_COMMIT_HASH = inputs.self.rev or inputs.self.dirtyRev or "";
GIT_COMMIT_HASH_SHORT = inputs.self.shortRev or inputs.self.dirtyShortRev or "";
}
// buildDepsOnlyEnv;
}

View File

@@ -0,0 +1,77 @@
{ pkgs, inputs, ... }:
let
inherit (pkgs) lib;
in
rec {
defaultDisabledFeatures = [
# dont include experimental features
"experimental"
# jemalloc profiling/stats features are expensive and shouldn't
# be expected on non-debug builds.
"jemalloc_prof"
"jemalloc_stats"
# this is non-functional on nix for some reason
"hardened_malloc"
# conduwuit_mods is a development-only hot reload feature
"conduwuit_mods"
# we don't want to enable this feature set by default but be more specific about it
"full"
];
# We perform default-feature unification in nix, because some of the dependencies
# on the nix side depend on feature values.
calcFeatures =
{
tomlPath ? "${inputs.self}/src/main",
# either a list of feature names or a string "all" which enables all non-default features
enabledFeatures ? [ ],
disabledFeatures ? defaultDisabledFeatures,
default_features ? true,
disable_release_max_log_level ? false,
}:
let
# simple helper to get the contents of a Cargo.toml file in a nix format
getToml = path: lib.importTOML "${path}/Cargo.toml";
# get all the features except for the default features
allFeatures = lib.pipe tomlPath [
getToml
(manifest: manifest.features)
lib.attrNames
(lib.remove "default")
];
# get just the default enabled features
allDefaultFeatures = lib.pipe tomlPath [
getToml
(manifest: manifest.features.default)
];
# depending on the value of enabledFeatures choose just a set or all non-default features
#
# - [ list of features ] -> choose exactly the features listed
# - "all" -> choose all non-default features
additionalFeatures = if enabledFeatures == "all" then allFeatures else enabledFeatures;
# unification with default features (if enabled)
features = lib.unique (additionalFeatures ++ lib.optionals default_features allDefaultFeatures);
# prepare the features that are subtracted from the set
disabledFeatures' =
disabledFeatures ++ lib.optionals disable_release_max_log_level [ "release_max_log_level" ];
# construct the final feature set
finalFeatures = lib.subtractLists disabledFeatures' features;
in
{
# final feature set, useful for querying it
features = finalFeatures;
# crane flag with the relevant features
cargoExtraArgs = builtins.concatStringsSep " " [
"--no-default-features"
"--locked"
(lib.optionalString (finalFeatures != [ ]) "--features")
(builtins.concatStringsSep "," finalFeatures)
];
};
}

29
nix/shells/default.nix Normal file
View File

@@ -0,0 +1,29 @@
{ inputs, ... }:
{
perSystem =
{
self',
lib,
pkgs,
...
}:
let
uwulib = inputs.self.uwulib.init pkgs;
rocksdbAllFeatures = self'.packages.rocksdb.override {
enableJemalloc = true;
enableLiburing = true;
};
in
{
# basic nix shell containing all things necessary to build continuwuity in all flavors manually (on x86_64-linux)
devShells.default = uwulib.build.craneLib.devShell {
packages = [
pkgs.pkg-config
pkgs.liburing
pkgs.rust-jemalloc-sys-unprefixed
rocksdbAllFeatures
];
env.LIBCLANG_PATH = lib.makeLibraryPath [ pkgs.llvmPackages.libclang.lib ];
};
};
}

124
nix/tests/default.nix Normal file
View File

@@ -0,0 +1,124 @@
{
perSystem =
{
self',
lib,
pkgs,
...
}:
{
# run some nixos tests as checks
checks = lib.pipe self'.packages [
# we take all packages (names)
builtins.attrNames
# we filter out all packages that end with `-bin` (which we are interested in for testing)
(builtins.filter (lib.hasSuffix "-bin"))
# for each of these binaries we built the basic nixos test
#
# this test was initially yoinked from
#
# https://github.com/NixOS/nixpkgs/blob/960ce26339661b1b69c6f12b9063ca51b688615f/nixos/tests/matrix/continuwuity.nix
(builtins.map (name: {
name = "test-${name}";
value = pkgs.testers.runNixOSTest {
inherit name;
nodes = {
continuwuity = {
services.matrix-continuwuity = {
enable = true;
package = self'.packages.${name};
settings.global = {
server_name = name;
address = [ "0.0.0.0" ];
allow_registration = true;
yes_i_am_very_very_sure_i_want_an_open_registration_server_prone_to_abuse = true;
};
extraEnvironment.RUST_BACKTRACE = "yes";
};
networking.firewall.allowedTCPPorts = [ 6167 ];
};
client =
{ pkgs, ... }:
{
environment.systemPackages = [
(pkgs.writers.writePython3Bin "do_test" { libraries = [ pkgs.python3Packages.matrix-nio ]; } ''
import asyncio
import nio
async def main() -> None:
# Connect to continuwuity
client = nio.AsyncClient("http://continuwuity:6167", "alice")
# Register as user alice
response = await client.register("alice", "my-secret-password")
# Log in as user alice
response = await client.login("my-secret-password")
# Create a new room
response = await client.room_create(federate=False)
print("Matrix room create response:", response)
assert isinstance(response, nio.RoomCreateResponse)
room_id = response.room_id
# Join the room
response = await client.join(room_id)
print("Matrix join response:", response)
assert isinstance(response, nio.JoinResponse)
# Send a message to the room
response = await client.room_send(
room_id=room_id,
message_type="m.room.message",
content={
"msgtype": "m.text",
"body": "Hello continuwuity!"
}
)
print("Matrix room send response:", response)
assert isinstance(response, nio.RoomSendResponse)
# Sync responses
response = await client.sync(timeout=30000)
print("Matrix sync response:", response)
assert isinstance(response, nio.SyncResponse)
# Check the message was received by continuwuity
last_message = response.rooms.join[room_id].timeline.events[-1].body
assert last_message == "Hello continuwuity!"
# Leave the room
response = await client.room_leave(room_id)
print("Matrix room leave response:", response)
assert isinstance(response, nio.RoomLeaveResponse)
# Close the client
await client.close()
if __name__ == "__main__":
asyncio.run(main())
'')
];
};
};
testScript = ''
start_all()
with subtest("start continuwuity"):
continuwuity.wait_for_unit("continuwuity.service")
continuwuity.wait_for_open_port(6167)
with subtest("ensure messages can be exchanged"):
client.succeed("do_test >&2")
'';
};
}))
builtins.listToAttrs
];
};
}

View File

@@ -1,83 +0,0 @@
{ lib
, pkgsBuildHost
, rust
, stdenv
}:
lib.optionalAttrs stdenv.hostPlatform.isStatic
{
ROCKSDB_STATIC = "";
}
//
{
CARGO_BUILD_RUSTFLAGS =
lib.concatStringsSep
" "
(lib.optionals
stdenv.hostPlatform.isStatic
[ "-C" "relocation-model=static" ]
++ lib.optionals
(stdenv.buildPlatform.config != stdenv.hostPlatform.config)
[
"-l"
"c"
"-l"
"stdc++"
"-L"
"${stdenv.cc.cc.lib}/${stdenv.hostPlatform.config}/lib"
]
);
}
# What follows is stolen from [here][0]. Its purpose is to properly
# configure compilers and linkers for various stages of the build, and
# even covers the case of build scripts that need native code compiled and
# run on the build platform (I think).
#
# [0]: https://github.com/NixOS/nixpkgs/blob/nixpkgs-unstable/pkgs/build-support/rust/lib/default.nix#L48-L68
//
(
let
inherit (rust.lib) envVars;
in
lib.optionalAttrs
(stdenv.targetPlatform.rust.rustcTarget
!= stdenv.hostPlatform.rust.rustcTarget)
(
let
inherit (stdenv.targetPlatform.rust) cargoEnvVarTarget;
in
{
"CC_${cargoEnvVarTarget}" = envVars.ccForTarget;
"CXX_${cargoEnvVarTarget}" = envVars.cxxForTarget;
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForTarget;
}
)
//
(
let
inherit (stdenv.hostPlatform.rust) cargoEnvVarTarget rustcTarget;
in
{
"CC_${cargoEnvVarTarget}" = envVars.ccForHost;
"CXX_${cargoEnvVarTarget}" = envVars.cxxForHost;
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForHost;
CARGO_BUILD_TARGET = rustcTarget;
}
)
//
(
let
inherit (stdenv.buildPlatform.rust) cargoEnvVarTarget;
in
{
"CC_${cargoEnvVarTarget}" = envVars.ccForBuild;
"CXX_${cargoEnvVarTarget}" = envVars.cxxForBuild;
"CARGO_TARGET_${cargoEnvVarTarget}_LINKER" = envVars.ccForBuild;
HOST_CC = "${pkgsBuildHost.stdenv.cc}/bin/cc";
HOST_CXX = "${pkgsBuildHost.stdenv.cc}/bin/c++";
}
)
)

View File

@@ -1,224 +0,0 @@
# Dependencies (keep sorted)
{ craneLib
, inputs
, jq
, lib
, libiconv
, liburing
, pkgsBuildHost
, rocksdb
, removeReferencesTo
, rust
, rust-jemalloc-sys
, stdenv
# Options (keep sorted)
, all_features ? false
, default_features ? true
# default list of disabled features
, disable_features ? [
# dont include experimental features
"experimental"
# jemalloc profiling/stats features are expensive and shouldn't
# be expected on non-debug builds.
"jemalloc_prof"
"jemalloc_stats"
# this is non-functional on nix for some reason
"hardened_malloc"
# conduwuit_mods is a development-only hot reload feature
"conduwuit_mods"
]
, disable_release_max_log_level ? false
, features ? [ ]
, profile ? "release"
# rocksdb compiled with -march=haswell and target-cpu=haswell rustflag
# haswell is pretty much any x86 cpu made in the last 12 years, and
# supports modern CPU extensions that rocksdb can make use of.
# disable if trying to make a portable x86_64 build for very old hardware
, x86_64_haswell_target_optimised ? false
}:
let
# We perform default-feature unification in nix, because some of the dependencies
# on the nix side depend on feature values.
crateFeatures = path:
let manifest = lib.importTOML "${path}/Cargo.toml"; in
lib.remove "default" (lib.attrNames manifest.features);
crateDefaultFeatures = path:
(lib.importTOML "${path}/Cargo.toml").features.default;
allDefaultFeatures = crateDefaultFeatures "${inputs.self}/src/main";
allFeatures = crateFeatures "${inputs.self}/src/main";
features' = lib.unique
(features ++
lib.optionals default_features allDefaultFeatures ++
lib.optionals all_features allFeatures);
disable_features' = disable_features ++ lib.optionals disable_release_max_log_level [ "release_max_log_level" ];
features'' = lib.subtractLists disable_features' features';
featureEnabled = feature: builtins.elem feature features'';
enableLiburing = featureEnabled "io_uring" && !stdenv.hostPlatform.isDarwin;
# This derivation will set the JEMALLOC_OVERRIDE variable, causing the
# tikv-jemalloc-sys crate to use the nixpkgs jemalloc instead of building it's
# own. In order for this to work, we need to set flags on the build that match
# whatever flags tikv-jemalloc-sys was going to use. These are dependent on
# which features we enable in tikv-jemalloc-sys.
rust-jemalloc-sys' = (rust-jemalloc-sys.override {
# tikv-jemalloc-sys/unprefixed_malloc_on_supported_platforms feature
unprefixed = true;
}).overrideAttrs (old: {
configureFlags = old.configureFlags ++
# we dont need docs
[ "--disable-doc" ] ++
# we dont need cxx/C++ integration
[ "--disable-cxx" ] ++
# tikv-jemalloc-sys/profiling feature
lib.optional (featureEnabled "jemalloc_prof") "--enable-prof" ++
# tikv-jemalloc-sys/stats feature
(if (featureEnabled "jemalloc_stats") then [ "--enable-stats" ] else [ "--disable-stats" ]);
});
buildDepsOnlyEnv =
let
rocksdb' = (rocksdb.override {
jemalloc = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys';
# rocksdb fails to build with prefixed jemalloc, which is required on
# darwin due to [1]. In this case, fall back to building rocksdb with
# libc malloc. This should not cause conflicts, because all of the
# jemalloc symbols are prefixed.
#
# [1]: https://github.com/tikv/jemallocator/blob/ab0676d77e81268cd09b059260c75b38dbef2d51/jemalloc-sys/src/env.rs#L17
enableJemalloc = featureEnabled "jemalloc" && !stdenv.hostPlatform.isDarwin;
# for some reason enableLiburing in nixpkgs rocksdb is default true
# which breaks Darwin entirely
inherit enableLiburing;
}).overrideAttrs (old: {
inherit enableLiburing;
cmakeFlags = (if x86_64_haswell_target_optimised then
(lib.subtractLists [
# dont make a portable build if x86_64_haswell_target_optimised is enabled
"-DPORTABLE=1"
]
old.cmakeFlags
++ [ "-DPORTABLE=haswell" ]) else [ "-DPORTABLE=1" ]
)
++ old.cmakeFlags;
# outputs has "tools" which we dont need or use
outputs = [ "out" ];
# preInstall hooks has stuff for messing with ldb/sst_dump which we dont need or use
preInstall = "";
});
in
{
# https://crane.dev/faq/rebuilds-bindgen.html
NIX_OUTPATH_USED_AS_RANDOM_SEED = "aaaaaaaaaa";
CARGO_PROFILE = profile;
ROCKSDB_INCLUDE_DIR = "${rocksdb'}/include";
ROCKSDB_LIB_DIR = "${rocksdb'}/lib";
}
//
(import ./cross-compilation-env.nix {
# Keep sorted
inherit
lib
pkgsBuildHost
rust
stdenv;
});
buildPackageEnv = {
GIT_COMMIT_HASH = inputs.self.rev or inputs.self.dirtyRev or "";
GIT_COMMIT_HASH_SHORT = inputs.self.shortRev or inputs.self.dirtyShortRev or "";
} // buildDepsOnlyEnv // {
# Only needed in static stdenv because these are transitive dependencies of rocksdb
CARGO_BUILD_RUSTFLAGS = buildDepsOnlyEnv.CARGO_BUILD_RUSTFLAGS
+ lib.optionalString (enableLiburing && stdenv.hostPlatform.isStatic)
" -L${lib.getLib liburing}/lib -luring"
+ lib.optionalString x86_64_haswell_target_optimised
" -Ctarget-cpu=haswell";
};
commonAttrs = {
inherit
(craneLib.crateNameFromCargoToml {
cargoToml = "${inputs.self}/Cargo.toml";
})
pname
version;
src = let filter = inputs.nix-filter.lib; in filter {
root = inputs.self;
# Keep sorted
include = [
".cargo"
"Cargo.lock"
"Cargo.toml"
"src"
"xtask"
];
};
doCheck = true;
cargoExtraArgs = "--no-default-features --locked "
+ lib.optionalString
(features'' != [ ])
"--features " + (builtins.concatStringsSep "," features'');
dontStrip = profile == "dev" || profile == "test";
dontPatchELF = profile == "dev" || profile == "test";
buildInputs = lib.optional (featureEnabled "jemalloc") rust-jemalloc-sys'
# needed to build Rust applications on macOS
++ lib.optionals stdenv.hostPlatform.isDarwin [
# https://github.com/NixOS/nixpkgs/issues/206242
# ld: library not found for -liconv
libiconv
# https://stackoverflow.com/questions/69869574/properly-adding-darwin-apple-sdk-to-a-nix-shell
# https://discourse.nixos.org/t/compile-a-rust-binary-on-macos-dbcrossbar/8612
pkgsBuildHost.darwin.apple_sdk.frameworks.Security
];
nativeBuildInputs = [
# bindgen needs the build platform's libclang. Apparently due to "splicing
# weirdness", pkgs.rustPlatform.bindgenHook on its own doesn't quite do the
# right thing here.
pkgsBuildHost.rustPlatform.bindgenHook
# We don't actually depend on `jq`, but crane's `buildPackage` does, but
# its `buildDepsOnly` doesn't. This causes those two derivations to have
# differing values for `NIX_CFLAGS_COMPILE`, which contributes to spurious
# rebuilds of bindgen and its depedents.
jq
];
};
in
craneLib.buildPackage (commonAttrs // {
cargoArtifacts = craneLib.buildDepsOnly (commonAttrs // {
env = buildDepsOnlyEnv;
});
doCheck = true;
cargoExtraArgs = "--no-default-features --locked "
+ lib.optionalString
(features'' != [ ])
"--features " + (builtins.concatStringsSep "," features'');
env = buildPackageEnv;
passthru = {
env = buildPackageEnv;
};
meta.mainProgram = commonAttrs.pname;
})

View File

@@ -10,6 +10,9 @@
"nix": {
"enabled": true
},
"pre-commit": {
"enabled": true
},
"labels": ["Dependencies", "Dependencies/Renovate"],
"ignoreDeps": [
"tikv-jemallocator",
@@ -18,7 +21,16 @@
"opentelemetry",
"opentelemetry_sdk",
"opentelemetry-jaeger",
"tracing-opentelemetry"
"tracing-opentelemetry",
"tracing-subscriber",
"tracing",
"tracing-core",
"tracing-log",
"rustyline-async",
"event-listener",
"async-channel",
"core_affinity",
"hyper-util"
],
"github-actions": {
"enabled": true,

View File

@@ -2,7 +2,8 @@
use conduwuit::{
Err, Result, debug, debug_info, debug_warn, error, info, trace,
utils::time::parse_timepoint_ago, warn,
utils::time::{TimeDirection, parse_timepoint_ago},
warn,
};
use conduwuit_service::media::Dim;
use ruma::{Mxc, OwnedEventId, OwnedMxcUri, OwnedServerName};
@@ -235,14 +236,19 @@ pub(super) async fn delete_past_remote_media(
}
assert!(!(before && after), "--before and --after should not be specified together");
let duration = parse_timepoint_ago(&duration)?;
let direction = if after {
TimeDirection::After
} else {
TimeDirection::Before
};
let time_boundary = parse_timepoint_ago(&duration)?;
let deleted_count = self
.services
.media
.delete_all_remote_media_at_after_time(
duration,
before,
after,
.delete_all_media_within_timeframe(
time_boundary,
direction,
yes_i_want_to_delete_local_media,
)
.await?;

View File

@@ -27,12 +27,24 @@ pub enum MediaCommand {
/// filesystem. This will always ignore errors.
DeleteList,
/// - Deletes all remote (and optionally local) media created before or
/// after [duration] time using filesystem metadata first created at date,
/// or fallback to last modified date. This will always ignore errors by
/// default.
/// Deletes all remote (and optionally local) media created before/after
/// [duration] ago, using filesystem metadata first created at date, or
/// fallback to last modified date. This will always ignore errors by
/// default.
///
/// * Examples:
/// * Delete all remote media older than a year:
///
/// `!admin media delete-past-remote-media -b 1y`
///
/// * Delete all remote and local media from 3 days ago, up until now:
///
/// `!admin media delete-past-remote-media -a 3d
/// --yes-i-want-to-delete-local-media`
#[command(verbatim_doc_comment)]
DeletePastRemoteMedia {
/// - The relative time (e.g. 30s, 5m, 7d) within which to search
/// - The relative time (e.g. 30s, 5m, 7d) from now within which to
/// search
duration: String,
/// - Only delete media created before [duration] ago

View File

@@ -44,7 +44,7 @@ pub(crate) async fn get_hierarchy_route(
.as_ref()
.and_then(|s| PaginationToken::from_str(s).ok());
// Should prevent unexpeded behaviour in (bad) clients
// Should prevent unexpected behaviour in (bad) clients
if let Some(ref token) = key {
if token.suggested_only != body.suggested_only || token.max_depth != max_depth {
return Err!(Request(InvalidParam(

View File

@@ -1,6 +1,6 @@
use axum::extract::State;
use conduwuit::{
Err, Error, Result, debug_info, matrix::pdu::PduBuilder, utils::IterStream, warn,
Err, Error, Result, debug_info, info, matrix::pdu::PduBuilder, utils::IterStream, warn,
};
use conduwuit_service::Services;
use futures::StreamExt;
@@ -22,6 +22,7 @@
/// # `GET /_matrix/federation/v1/make_join/{roomId}/{userId}`
///
/// Creates a join template.
#[tracing::instrument(skip_all, fields(room_id = %body.room_id, user_id = %body.user_id, origin = %body.origin()))]
pub(crate) async fn create_join_event_template_route(
State(services): State<crate::State>,
body: Ruma<prepare_join_event::v1::Request>,
@@ -72,11 +73,16 @@ pub(crate) async fn create_join_event_template_route(
}
let state_lock = services.rooms.state.mutex.lock(&body.room_id).await;
let is_invited = services
.rooms
.state_cache
.is_invited(&body.user_id, &body.room_id)
.await;
let join_authorized_via_users_server: Option<OwnedUserId> = {
use RoomVersionId::*;
if matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6 | V7) {
// room version does not support restricted join rules
if matches!(room_version_id, V1 | V2 | V3 | V4 | V5 | V6 | V7) || is_invited {
// room version does not support restricted join rules, or the user is currently
// already invited
None
} else if user_can_perform_restricted_join(
&services,
@@ -103,6 +109,10 @@ pub(crate) async fn create_join_event_template_route(
.await
.map(ToOwned::to_owned)
else {
info!(
"No local user is able to authorize the join of {} into {}",
&body.user_id, &body.room_id
);
return Err!(Request(UnableToGrantJoin(
"No user on this server is able to assist in joining."
)));
@@ -167,6 +177,7 @@ pub(crate) async fn user_can_perform_restricted_join(
)
.await
else {
// No join rules means there's nothing to authorise (defaults to invite)
return Ok(false);
};

View File

@@ -615,15 +615,21 @@ pub async fn auth_check<E, F, Fut>(
Ok(true)
}
fn is_creator<EV>(v: &RoomVersion, c: &BTreeSet<OwnedUserId>, ce: &EV, user_id: &UserId) -> bool
fn is_creator<EV>(
v: &RoomVersion,
c: &BTreeSet<OwnedUserId>,
ce: &EV,
user_id: &UserId,
have_pls: bool,
) -> bool
where
EV: Event + Send + Sync,
{
if v.explicitly_privilege_room_creators {
c.contains(user_id)
} else if v.use_room_create_sender {
} else if v.use_room_create_sender && !have_pls {
ce.sender() == user_id
} else {
} else if !have_pls {
#[allow(deprecated)]
let creator = from_json_str::<RoomCreateEventContent>(ce.content().get())
.unwrap()
@@ -632,6 +638,8 @@ fn is_creator<EV>(v: &RoomVersion, c: &BTreeSet<OwnedUserId>, ce: &EV, user_id:
.unwrap();
creator == user_id
} else {
false
}
}
@@ -724,10 +732,11 @@ struct GetThirdPartyInvite {
}
trace!(?creators, "creators for room");
let mut join_rules = JoinRule::Invite;
if let Some(jr) = &join_rules_event {
join_rules = from_json_str::<RoomJoinRulesEventContent>(jr.content().get())?.join_rule;
}
let join_rules = if let Some(jr) = &join_rules_event {
from_json_str::<RoomJoinRulesEventContent>(jr.content().get())?.join_rule
} else {
JoinRule::Invite
};
let power_levels_event_id = power_levels_event.as_ref().map(Event::event_id);
let sender_membership_event_id = sender_membership_event.as_ref().map(Event::event_id);
@@ -753,8 +762,13 @@ struct GetThirdPartyInvite {
(int!(0), int!(0))
};
let user_joined = user_for_join_auth_membership == &MembershipState::Join;
let okay_power = is_creator(room_version, &creators, create_room, user_for_join_auth)
|| auth_user_pl >= invite_level;
let okay_power = is_creator(
room_version,
&creators,
create_room,
user_for_join_auth,
power_levels_event.as_ref().is_some(),
) || auth_user_pl >= invite_level;
trace!(
auth_user_pl=?auth_user_pl,
invite_level=?invite_level,
@@ -769,8 +783,20 @@ struct GetThirdPartyInvite {
trace!("No auth user given for join auth");
false
};
let sender_creator = is_creator(room_version, &creators, create_room, sender);
let target_creator = is_creator(room_version, &creators, create_room, target_user);
let sender_creator = is_creator(
room_version,
&creators,
create_room,
sender,
power_levels_event.as_ref().is_some(),
);
let target_creator = is_creator(
room_version,
&creators,
create_room,
target_user,
power_levels_event.as_ref().is_some(),
);
Ok(match target_membership {
| MembershipState::Join => {
@@ -985,7 +1011,7 @@ struct GetThirdPartyInvite {
},
| MembershipState::Leave => {
let can_unban = if target_user_current_membership == MembershipState::Ban {
sender_creator || sender_power.filter(|&p| p < &power_levels.ban).is_some()
sender_creator || sender_power.filter(|&p| p >= &power_levels.ban).is_some()
} else {
true
};
@@ -993,7 +1019,24 @@ struct GetThirdPartyInvite {
target_user_current_membership,
MembershipState::Ban | MembershipState::Leave
) {
sender_creator || sender_power.filter(|&p| p < &power_levels.kick).is_some()
if sender_creator {
// sender is a creator
true
} else if sender_power.filter(|&p| p >= &power_levels.kick).is_none() {
// sender lacks kick power level
false
} else if let Some(sp) = sender_power {
if let Some(tp) = target_power {
// sender must have more power than target
sp > tp
} else {
// target has default power level
true
}
} else {
// sender has default power level
false
}
} else {
true
};
@@ -1023,7 +1066,7 @@ struct GetThirdPartyInvite {
"sender cannot kick another user as they are not joined to the room",
);
false
} else if !can_unban {
} else if !(can_unban && can_kick) {
// If the target is banned, only a room creator or someone with ban power
// level can unban them
warn!(

View File

@@ -217,14 +217,8 @@ pub async fn resolve<'a, Pdu, Sets, SetIter, Hasher, Fetch, FetchFut, Exists, Ex
)
.await?;
// Add unconflicted state to the resolved state
// We priorities the unconflicting state
// Ensure unconflicting state is in the final state
resolved_state.extend(clean);
if stateres_version == StateResolutionVersion::V2_1 {
resolved_state.extend(resolved_control);
// TODO(hydra): this feels disgusting and wrong but it allows
// the state to resolve properly?
}
debug!("state resolution finished");
trace!( map = ?resolved_state, "final resolved state" );

View File

@@ -276,3 +276,22 @@ async fn set_intersection_sorted_stream2() {
.await;
assert!(r.eq(&["ccc", "ggg", "iii"]));
}
#[test]
fn is_within_bounds() {
use std::time::{Duration, SystemTime};
use utils::time::{TimeDirection, is_within_bounds};
let now = SystemTime::now();
let yesterday = now - Duration::from_secs(86400);
assert!(is_within_bounds(yesterday, now, TimeDirection::Before));
assert!(!is_within_bounds(yesterday, now, TimeDirection::After));
let tomorrow = now + Duration::from_secs(86400);
assert!(is_within_bounds(tomorrow, now, TimeDirection::After));
assert!(!is_within_bounds(tomorrow, now, TimeDirection::Before));
assert!(is_within_bounds(now, now, TimeDirection::Before));
assert!(is_within_bounds(now, now, TimeDirection::After));
}

View File

@@ -126,3 +126,24 @@ pub enum Unit {
Micros(u128),
Nanos(u128),
}
#[derive(Eq, PartialEq, Clone, Copy, Debug)]
pub enum TimeDirection {
Before,
After,
}
/// Checks if `item_time` is before or after `time_boundary`.
/// If both times are the same, it will return true for both directions, as the
/// matching is inclusive.
#[must_use]
pub fn is_within_bounds(
item_time: SystemTime,
time_boundary: SystemTime,
direction: TimeDirection,
) -> bool {
match direction {
| TimeDirection::Before => item_time <= time_boundary,
| TimeDirection::After => item_time >= time_boundary,
}
}

View File

@@ -11,7 +11,10 @@
use base64::{Engine as _, engine::general_purpose};
use conduwuit::{
Err, Result, Server, debug, debug_error, debug_info, debug_warn, err, error, trace,
utils::{self, MutexMap},
utils::{
self, MutexMap,
time::{self, TimeDirection},
},
warn,
};
use ruma::{Mxc, OwnedMxcUri, UserId, http_headers::ContentDisposition};
@@ -226,13 +229,12 @@ pub async fn get_all_mxcs(&self) -> Result<Vec<OwnedMxcUri>> {
Ok(mxcs)
}
/// Deletes all remote only media files in the given at or after
/// time/duration. Returns a usize with the amount of media files deleted.
pub async fn delete_all_remote_media_at_after_time(
/// Deletes all media files in the given time frame.
/// Returns a usize with the amount of media files deleted.
pub async fn delete_all_media_within_timeframe(
&self,
time: SystemTime,
before: bool,
after: bool,
time_boundary: SystemTime,
direction: TimeDirection,
yes_i_want_to_delete_local_media: bool,
) -> Result<usize> {
let all_keys = self.db.get_all_media_keys().await;
@@ -299,18 +301,14 @@ pub async fn delete_all_remote_media_at_after_time(
debug!("File created at: {file_created_at:?}");
if file_created_at >= time && before {
if time::is_within_bounds(file_created_at, time_boundary, direction) {
debug!(
"File is within (before) user duration, pushing to list of file paths and \
keys to delete."
);
remote_mxcs.push(mxc.to_string());
} else if file_created_at <= time && after {
debug!(
"File is not within (after) user duration, pushing to list of file paths \
and keys to delete."
"File is within bounds ({direction:?} {time_boundary:?}), pushing to list \
of file paths and keys to delete.",
);
remote_mxcs.push(mxc.to_string());
} else {
debug!("File is outside bounds ({direction:?} {time_boundary:?}), ignoring.");
}
}
@@ -318,7 +316,7 @@ pub async fn delete_all_remote_media_at_after_time(
return Err!(Database("Did not found any eligible MXCs to delete."));
}
debug_info!("Deleting media now in the past {time:?}");
debug_info!("Deleting media now {direction:?} {time_boundary:?}");
let mut deletion_count: usize = 0;

View File

@@ -1,9 +1,10 @@
use conduwuit::{Err, Result, implement, matrix::Event, pdu::PduBuilder};
use conduwuit::{Err, Result, RoomVersion, implement, matrix::Event, pdu::PduBuilder};
use ruma::{
EventId, RoomId, UserId,
events::{
StateEventType, TimelineEventType,
room::{
create::RoomCreateEventContent,
history_visibility::{HistoryVisibility, RoomHistoryVisibilityEventContent},
member::{MembershipState, RoomMemberEventContent},
power_levels::{RoomPowerLevels, RoomPowerLevelsEventContent},
@@ -44,6 +45,23 @@ pub async fn user_can_redact(
)));
}
let room_create = self
.room_state_get(room_id, &StateEventType::RoomCreate, "")
.await?;
let create_content: RoomCreateEventContent =
serde_json::from_str(room_create.content().get())?;
let room_features = RoomVersion::new(&create_content.room_version)?;
if room_features.explicitly_privilege_room_creators {
let sender_owned = sender.to_owned();
if sender == room_create.sender()
|| create_content
.additional_creators
.is_some_and(|cs| cs.contains(&sender_owned))
{
return Ok(true);
}
}
match self
.room_state_get_content::<RoomPowerLevelsEventContent>(
room_id,
@@ -68,18 +86,10 @@ pub async fn user_can_redact(
},
| _ => {
// Falling back on m.room.create to judge power level
match self
.room_state_get(room_id, &StateEventType::RoomCreate, "")
.await
{
| Ok(room_create) => Ok(room_create.sender() == sender
|| redacting_event
.as_ref()
.is_ok_and(|redacting_event| redacting_event.sender() == sender)),
| _ => Err!(Database(
"No m.room.power_levels or m.room.create events in database for room"
)),
}
Ok(room_create.sender() == sender
|| redacting_event
.as_ref()
.is_ok_and(|redacting_event| redacting_event.sender() == sender))
},
}
}

View File

@@ -345,7 +345,7 @@ pub async fn displayname(&self, user_id: &UserId) -> Result<String> {
}
/// Sets a new displayname or removes it if displayname is None. You still
/// need to nofify all rooms of this change.
/// need to notify all rooms of this change.
pub fn set_displayname(&self, user_id: &UserId, displayname: Option<String>) {
if let Some(displayname) = displayname {
self.db.userid_displayname.insert(user_id, displayname);

6
taplo.toml Normal file
View File

@@ -0,0 +1,6 @@
[formatting]
align_entries = true # Align entries vertically. Entries that have table headers, comments, or blank lines between them are not aligned.
reorder_arrays = true # Alphabetically reorder array values that are not separated by blank lines.
reorder_inline_tables = true # Alphabetically reorder inline tables.
reorder_keys = true # Alphabetically reorder keys that are not separated by blank lines.