From dff8bf273cdab6c2680e4df76c84057e39f27bc8 Mon Sep 17 00:00:00 2001 From: torlando-tech Date: Mon, 4 May 2026 18:08:59 -0400 Subject: [PATCH] Track A.2: vendor libbz2 + 37 fork-only files into pyxis MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Configuration moves to make pyxis buildable against vanilla attermann/microReticulum @ 0.3.0 instead of the fork: - lib/libbz2/ — pulled from torlando-tech/microReticulum @ ca355e5 (the previous submodule pin). Used by LXMF compression. Will eventually move into the standalone microLXMF library (Track B). - src-shim/ — the 37 fork-only files NOT already vendored into pyxis/lib/ as part of the original split. Includes: - LXMF/* (11 files) — to extract as a standalone library (Track B) - Cryptography/{BZ2,Ratchet}.{cpp,h} — used by LXMF + RNS 1.x compat - Instrumentation/{BootProfiler,MemoryMonitor}.{cpp,h} — pyxis-specific - Utilities/{Print,Stream}.{cpp,h} — pyxis logging supplements - root: Buffer, BytesPool, ChannelData, Display+Graphics, FileStream, FileSystem, MessageBase, ObjectPool, PSRAMAllocator, SegmentAccumulator - platformio.ini lib_deps: drop the symlink:// to deps/microReticulum/lib/libbz2 (gone in upstream), add microStore as a github URL dep (vanilla upstream needs it for Bytes.h's microStore/Codec.h include). - platformio.ini build_flags: add -Isrc-shim plus its subdirectories so the vendored ../X.h-style includes (rewritten to ) resolve through pyxis's own paths first. src-shim/* was bulk-rewritten so `#include "../X.h"` (relative to the fork's src/) becomes `#include ` resolved via -I deps/microReticulum/src. Pyxis still doesn't build at this point — the next layer of breakage is API-level (not file-level): pyxis main.cpp calls fork-only Identity/LXMRouter/Transport methods that vanilla upstream doesn't have. See the followup commit's notes / branch description for the full list. --- lib/libbz2/LICENSE | 42 + lib/libbz2/blocksort.c | 1094 +++++++++++++ lib/libbz2/bz_error.c | 10 + lib/libbz2/bzlib.c | 1572 ++++++++++++++++++ lib/libbz2/bzlib.h | 282 ++++ lib/libbz2/bzlib_private.h | 509 ++++++ lib/libbz2/compress.c | 672 ++++++++ lib/libbz2/crctable.c | 104 ++ lib/libbz2/decompress.c | 652 ++++++++ lib/libbz2/huffman.c | 205 +++ lib/libbz2/library.json | 22 + lib/libbz2/randtable.c | 84 + platformio.ini | 18 +- src-shim/Buffer.cpp | 404 +++++ src-shim/Buffer.h | 163 ++ src-shim/BytesPool.h | 356 +++++ src-shim/ChannelData.h | 369 +++++ src-shim/Cryptography/BZ2.cpp | 179 +++ src-shim/Cryptography/BZ2.h | 10 + src-shim/Cryptography/Ratchet.cpp | 193 +++ src-shim/Cryptography/Ratchet.h | 181 +++ src-shim/Display.cpp | 411 +++++ src-shim/Display.h | 139 ++ src-shim/DisplayGraphics.h | 226 +++ src-shim/FileStream.h | 144 ++ src-shim/FileSystem.h | 129 ++ src-shim/Instrumentation/BootProfiler.cpp | 237 +++ src-shim/Instrumentation/BootProfiler.h | 171 ++ src-shim/Instrumentation/MemoryMonitor.cpp | 263 +++ src-shim/Instrumentation/MemoryMonitor.h | 135 ++ src-shim/LXMF/LXMRouter.cpp | 1688 ++++++++++++++++++++ src-shim/LXMF/LXMRouter.h | 679 ++++++++ src-shim/LXMF/LXMessage.cpp | 746 +++++++++ src-shim/LXMF/LXMessage.h | 418 +++++ src-shim/LXMF/LXStamper.cpp | 241 +++ src-shim/LXMF/LXStamper.h | 121 ++ src-shim/LXMF/MessageStore.cpp | 735 +++++++++ src-shim/LXMF/MessageStore.h | 370 +++++ src-shim/LXMF/PropagationNodeManager.cpp | 326 ++++ src-shim/LXMF/PropagationNodeManager.h | 233 +++ src-shim/LXMF/Type.h | 161 ++ src-shim/MessageBase.h | 26 + src-shim/ObjectPool.h | 160 ++ src-shim/PSRAMAllocator.h | 87 + src-shim/SegmentAccumulator.cpp | 231 +++ src-shim/SegmentAccumulator.h | 162 ++ src-shim/Utilities/Print.cpp | 354 ++++ src-shim/Utilities/Print.h | 121 ++ src-shim/Utilities/Stream.cpp | 339 ++++ src-shim/Utilities/Stream.h | 143 ++ 50 files changed, 16313 insertions(+), 4 deletions(-) create mode 100644 lib/libbz2/LICENSE create mode 100644 lib/libbz2/blocksort.c create mode 100644 lib/libbz2/bz_error.c create mode 100644 lib/libbz2/bzlib.c create mode 100644 lib/libbz2/bzlib.h create mode 100644 lib/libbz2/bzlib_private.h create mode 100644 lib/libbz2/compress.c create mode 100644 lib/libbz2/crctable.c create mode 100644 lib/libbz2/decompress.c create mode 100644 lib/libbz2/huffman.c create mode 100644 lib/libbz2/library.json create mode 100644 lib/libbz2/randtable.c create mode 100644 src-shim/Buffer.cpp create mode 100644 src-shim/Buffer.h create mode 100644 src-shim/BytesPool.h create mode 100644 src-shim/ChannelData.h create mode 100644 src-shim/Cryptography/BZ2.cpp create mode 100644 src-shim/Cryptography/BZ2.h create mode 100644 src-shim/Cryptography/Ratchet.cpp create mode 100644 src-shim/Cryptography/Ratchet.h create mode 100644 src-shim/Display.cpp create mode 100644 src-shim/Display.h create mode 100644 src-shim/DisplayGraphics.h create mode 100644 src-shim/FileStream.h create mode 100644 src-shim/FileSystem.h create mode 100644 src-shim/Instrumentation/BootProfiler.cpp create mode 100644 src-shim/Instrumentation/BootProfiler.h create mode 100644 src-shim/Instrumentation/MemoryMonitor.cpp create mode 100644 src-shim/Instrumentation/MemoryMonitor.h create mode 100644 src-shim/LXMF/LXMRouter.cpp create mode 100644 src-shim/LXMF/LXMRouter.h create mode 100644 src-shim/LXMF/LXMessage.cpp create mode 100644 src-shim/LXMF/LXMessage.h create mode 100644 src-shim/LXMF/LXStamper.cpp create mode 100644 src-shim/LXMF/LXStamper.h create mode 100644 src-shim/LXMF/MessageStore.cpp create mode 100644 src-shim/LXMF/MessageStore.h create mode 100644 src-shim/LXMF/PropagationNodeManager.cpp create mode 100644 src-shim/LXMF/PropagationNodeManager.h create mode 100644 src-shim/LXMF/Type.h create mode 100644 src-shim/MessageBase.h create mode 100644 src-shim/ObjectPool.h create mode 100644 src-shim/PSRAMAllocator.h create mode 100644 src-shim/SegmentAccumulator.cpp create mode 100644 src-shim/SegmentAccumulator.h create mode 100644 src-shim/Utilities/Print.cpp create mode 100644 src-shim/Utilities/Print.h create mode 100644 src-shim/Utilities/Stream.cpp create mode 100644 src-shim/Utilities/Stream.h diff --git a/lib/libbz2/LICENSE b/lib/libbz2/LICENSE new file mode 100644 index 00000000..81a37eab --- /dev/null +++ b/lib/libbz2/LICENSE @@ -0,0 +1,42 @@ + +-------------------------------------------------------------------------- + +This program, "bzip2", the associated library "libbzip2", and all +documentation, are copyright (C) 1996-2019 Julian R Seward. All +rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. The origin of this software must not be misrepresented; you must + not claim that you wrote the original software. If you use this + software in a product, an acknowledgment in the product + documentation would be appreciated but is not required. + +3. Altered source versions must be plainly marked as such, and must + not be misrepresented as being the original software. + +4. The name of the author may not be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY +DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE +GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Julian Seward, jseward@acm.org +bzip2/libbzip2 version 1.0.8 of 13 July 2019 + +-------------------------------------------------------------------------- diff --git a/lib/libbz2/blocksort.c b/lib/libbz2/blocksort.c new file mode 100644 index 00000000..92d81fe2 --- /dev/null +++ b/lib/libbz2/blocksort.c @@ -0,0 +1,1094 @@ + +/*-------------------------------------------------------------*/ +/*--- Block sorting machinery ---*/ +/*--- blocksort.c ---*/ +/*-------------------------------------------------------------*/ + +/* ------------------------------------------------------------------ + This file is part of bzip2/libbzip2, a program and library for + lossless, block-sorting data compression. + + bzip2/libbzip2 version 1.0.8 of 13 July 2019 + Copyright (C) 1996-2019 Julian Seward + + Please read the WARNING, DISCLAIMER and PATENTS sections in the + README file. + + This program is released under the terms of the license contained + in the file LICENSE. + ------------------------------------------------------------------ */ + + +#include "bzlib_private.h" + +/*---------------------------------------------*/ +/*--- Fallback O(N log(N)^2) sorting ---*/ +/*--- algorithm, for repetitive blocks ---*/ +/*---------------------------------------------*/ + +/*---------------------------------------------*/ +static +__inline__ +void fallbackSimpleSort ( UInt32* fmap, + UInt32* eclass, + Int32 lo, + Int32 hi ) +{ + Int32 i, j, tmp; + UInt32 ec_tmp; + + if (lo == hi) return; + + if (hi - lo > 3) { + for ( i = hi-4; i >= lo; i-- ) { + tmp = fmap[i]; + ec_tmp = eclass[tmp]; + for ( j = i+4; j <= hi && ec_tmp > eclass[fmap[j]]; j += 4 ) + fmap[j-4] = fmap[j]; + fmap[j-4] = tmp; + } + } + + for ( i = hi-1; i >= lo; i-- ) { + tmp = fmap[i]; + ec_tmp = eclass[tmp]; + for ( j = i+1; j <= hi && ec_tmp > eclass[fmap[j]]; j++ ) + fmap[j-1] = fmap[j]; + fmap[j-1] = tmp; + } +} + + +/*---------------------------------------------*/ +#define fswap(zz1, zz2) \ + { Int32 zztmp = zz1; zz1 = zz2; zz2 = zztmp; } + +#define fvswap(zzp1, zzp2, zzn) \ +{ \ + Int32 yyp1 = (zzp1); \ + Int32 yyp2 = (zzp2); \ + Int32 yyn = (zzn); \ + while (yyn > 0) { \ + fswap(fmap[yyp1], fmap[yyp2]); \ + yyp1++; yyp2++; yyn--; \ + } \ +} + + +#define fmin(a,b) ((a) < (b)) ? (a) : (b) + +#define fpush(lz,hz) { stackLo[sp] = lz; \ + stackHi[sp] = hz; \ + sp++; } + +#define fpop(lz,hz) { sp--; \ + lz = stackLo[sp]; \ + hz = stackHi[sp]; } + +#define FALLBACK_QSORT_SMALL_THRESH 10 +#define FALLBACK_QSORT_STACK_SIZE 100 + + +static +void fallbackQSort3 ( UInt32* fmap, + UInt32* eclass, + Int32 loSt, + Int32 hiSt ) +{ + Int32 unLo, unHi, ltLo, gtHi, n, m; + Int32 sp, lo, hi; + UInt32 med, r, r3; + Int32 stackLo[FALLBACK_QSORT_STACK_SIZE]; + Int32 stackHi[FALLBACK_QSORT_STACK_SIZE]; + + r = 0; + + sp = 0; + fpush ( loSt, hiSt ); + + while (sp > 0) { + + AssertH ( sp < FALLBACK_QSORT_STACK_SIZE - 1, 1004 ); + + fpop ( lo, hi ); + if (hi - lo < FALLBACK_QSORT_SMALL_THRESH) { + fallbackSimpleSort ( fmap, eclass, lo, hi ); + continue; + } + + /* Random partitioning. Median of 3 sometimes fails to + avoid bad cases. Median of 9 seems to help but + looks rather expensive. This too seems to work but + is cheaper. Guidance for the magic constants + 7621 and 32768 is taken from Sedgewick's algorithms + book, chapter 35. + */ + r = ((r * 7621) + 1) % 32768; + r3 = r % 3; + if (r3 == 0) med = eclass[fmap[lo]]; else + if (r3 == 1) med = eclass[fmap[(lo+hi)>>1]]; else + med = eclass[fmap[hi]]; + + unLo = ltLo = lo; + unHi = gtHi = hi; + + while (1) { + while (1) { + if (unLo > unHi) break; + n = (Int32)eclass[fmap[unLo]] - (Int32)med; + if (n == 0) { + fswap(fmap[unLo], fmap[ltLo]); + ltLo++; unLo++; + continue; + }; + if (n > 0) break; + unLo++; + } + while (1) { + if (unLo > unHi) break; + n = (Int32)eclass[fmap[unHi]] - (Int32)med; + if (n == 0) { + fswap(fmap[unHi], fmap[gtHi]); + gtHi--; unHi--; + continue; + }; + if (n < 0) break; + unHi--; + } + if (unLo > unHi) break; + fswap(fmap[unLo], fmap[unHi]); unLo++; unHi--; + } + + AssertD ( unHi == unLo-1, "fallbackQSort3(2)" ); + + if (gtHi < ltLo) continue; + + n = fmin(ltLo-lo, unLo-ltLo); fvswap(lo, unLo-n, n); + m = fmin(hi-gtHi, gtHi-unHi); fvswap(unLo, hi-m+1, m); + + n = lo + unLo - ltLo - 1; + m = hi - (gtHi - unHi) + 1; + + if (n - lo > hi - m) { + fpush ( lo, n ); + fpush ( m, hi ); + } else { + fpush ( m, hi ); + fpush ( lo, n ); + } + } +} + +#undef fmin +#undef fpush +#undef fpop +#undef fswap +#undef fvswap +#undef FALLBACK_QSORT_SMALL_THRESH +#undef FALLBACK_QSORT_STACK_SIZE + + +/*---------------------------------------------*/ +/* Pre: + nblock > 0 + eclass exists for [0 .. nblock-1] + ((UChar*)eclass) [0 .. nblock-1] holds block + ptr exists for [0 .. nblock-1] + + Post: + ((UChar*)eclass) [0 .. nblock-1] holds block + All other areas of eclass destroyed + fmap [0 .. nblock-1] holds sorted order + bhtab [ 0 .. 2+(nblock/32) ] destroyed +*/ + +#define SET_BH(zz) bhtab[(zz) >> 5] |= ((UInt32)1 << ((zz) & 31)) +#define CLEAR_BH(zz) bhtab[(zz) >> 5] &= ~((UInt32)1 << ((zz) & 31)) +#define ISSET_BH(zz) (bhtab[(zz) >> 5] & ((UInt32)1 << ((zz) & 31))) +#define WORD_BH(zz) bhtab[(zz) >> 5] +#define UNALIGNED_BH(zz) ((zz) & 0x01f) + +static +void fallbackSort ( UInt32* fmap, + UInt32* eclass, + UInt32* bhtab, + Int32 nblock, + Int32 verb ) +{ + Int32 ftab[257]; + Int32 ftabCopy[256]; + Int32 H, i, j, k, l, r, cc, cc1; + Int32 nNotDone; + Int32 nBhtab; + UChar* eclass8 = (UChar*)eclass; + + /*-- + Initial 1-char radix sort to generate + initial fmap and initial BH bits. + --*/ + if (verb >= 4) + VPrintf0 ( " bucket sorting ...\n" ); + for (i = 0; i < 257; i++) ftab[i] = 0; + for (i = 0; i < nblock; i++) ftab[eclass8[i]]++; + for (i = 0; i < 256; i++) ftabCopy[i] = ftab[i]; + for (i = 1; i < 257; i++) ftab[i] += ftab[i-1]; + + for (i = 0; i < nblock; i++) { + j = eclass8[i]; + k = ftab[j] - 1; + ftab[j] = k; + fmap[k] = i; + } + + nBhtab = 2 + (nblock / 32); + for (i = 0; i < nBhtab; i++) bhtab[i] = 0; + for (i = 0; i < 256; i++) SET_BH(ftab[i]); + + /*-- + Inductively refine the buckets. Kind-of an + "exponential radix sort" (!), inspired by the + Manber-Myers suffix array construction algorithm. + --*/ + + /*-- set sentinel bits for block-end detection --*/ + for (i = 0; i < 32; i++) { + SET_BH(nblock + 2*i); + CLEAR_BH(nblock + 2*i + 1); + } + + /*-- the log(N) loop --*/ + H = 1; + while (1) { + + if (verb >= 4) + VPrintf1 ( " depth %6d has ", H ); + + j = 0; + for (i = 0; i < nblock; i++) { + if (ISSET_BH(i)) j = i; + k = fmap[i] - H; if (k < 0) k += nblock; + eclass[k] = j; + } + + nNotDone = 0; + r = -1; + while (1) { + + /*-- find the next non-singleton bucket --*/ + k = r + 1; + while (ISSET_BH(k) && UNALIGNED_BH(k)) k++; + if (ISSET_BH(k)) { + while (WORD_BH(k) == 0xffffffff) k += 32; + while (ISSET_BH(k)) k++; + } + l = k - 1; + if (l >= nblock) break; + while (!ISSET_BH(k) && UNALIGNED_BH(k)) k++; + if (!ISSET_BH(k)) { + while (WORD_BH(k) == 0x00000000) k += 32; + while (!ISSET_BH(k)) k++; + } + r = k - 1; + if (r >= nblock) break; + + /*-- now [l, r] bracket current bucket --*/ + if (r > l) { + nNotDone += (r - l + 1); + fallbackQSort3 ( fmap, eclass, l, r ); + + /*-- scan bucket and generate header bits-- */ + cc = -1; + for (i = l; i <= r; i++) { + cc1 = eclass[fmap[i]]; + if (cc != cc1) { SET_BH(i); cc = cc1; }; + } + } + } + + if (verb >= 4) + VPrintf1 ( "%6d unresolved strings\n", nNotDone ); + + H *= 2; + if (H > nblock || nNotDone == 0) break; + } + + /*-- + Reconstruct the original block in + eclass8 [0 .. nblock-1], since the + previous phase destroyed it. + --*/ + if (verb >= 4) + VPrintf0 ( " reconstructing block ...\n" ); + j = 0; + for (i = 0; i < nblock; i++) { + while (ftabCopy[j] == 0) j++; + ftabCopy[j]--; + eclass8[fmap[i]] = (UChar)j; + } + AssertH ( j < 256, 1005 ); +} + +#undef SET_BH +#undef CLEAR_BH +#undef ISSET_BH +#undef WORD_BH +#undef UNALIGNED_BH + + +/*---------------------------------------------*/ +/*--- The main, O(N^2 log(N)) sorting ---*/ +/*--- algorithm. Faster for "normal" ---*/ +/*--- non-repetitive blocks. ---*/ +/*---------------------------------------------*/ + +/*---------------------------------------------*/ +static +__inline__ +Bool mainGtU ( UInt32 i1, + UInt32 i2, + UChar* block, + UInt16* quadrant, + UInt32 nblock, + Int32* budget ) +{ + Int32 k; + UChar c1, c2; + UInt16 s1, s2; + + AssertD ( i1 != i2, "mainGtU" ); + /* 1 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + i1++; i2++; + /* 2 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + i1++; i2++; + /* 3 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + i1++; i2++; + /* 4 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + i1++; i2++; + /* 5 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + i1++; i2++; + /* 6 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + i1++; i2++; + /* 7 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + i1++; i2++; + /* 8 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + i1++; i2++; + /* 9 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + i1++; i2++; + /* 10 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + i1++; i2++; + /* 11 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + i1++; i2++; + /* 12 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + i1++; i2++; + + k = nblock + 8; + + do { + /* 1 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + s1 = quadrant[i1]; s2 = quadrant[i2]; + if (s1 != s2) return (s1 > s2); + i1++; i2++; + /* 2 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + s1 = quadrant[i1]; s2 = quadrant[i2]; + if (s1 != s2) return (s1 > s2); + i1++; i2++; + /* 3 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + s1 = quadrant[i1]; s2 = quadrant[i2]; + if (s1 != s2) return (s1 > s2); + i1++; i2++; + /* 4 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + s1 = quadrant[i1]; s2 = quadrant[i2]; + if (s1 != s2) return (s1 > s2); + i1++; i2++; + /* 5 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + s1 = quadrant[i1]; s2 = quadrant[i2]; + if (s1 != s2) return (s1 > s2); + i1++; i2++; + /* 6 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + s1 = quadrant[i1]; s2 = quadrant[i2]; + if (s1 != s2) return (s1 > s2); + i1++; i2++; + /* 7 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + s1 = quadrant[i1]; s2 = quadrant[i2]; + if (s1 != s2) return (s1 > s2); + i1++; i2++; + /* 8 */ + c1 = block[i1]; c2 = block[i2]; + if (c1 != c2) return (c1 > c2); + s1 = quadrant[i1]; s2 = quadrant[i2]; + if (s1 != s2) return (s1 > s2); + i1++; i2++; + + if (i1 >= nblock) i1 -= nblock; + if (i2 >= nblock) i2 -= nblock; + + k -= 8; + (*budget)--; + } + while (k >= 0); + + return False; +} + + +/*---------------------------------------------*/ +/*-- + Knuth's increments seem to work better + than Incerpi-Sedgewick here. Possibly + because the number of elems to sort is + usually small, typically <= 20. +--*/ +static +Int32 incs[14] = { 1, 4, 13, 40, 121, 364, 1093, 3280, + 9841, 29524, 88573, 265720, + 797161, 2391484 }; + +static +void mainSimpleSort ( UInt32* ptr, + UChar* block, + UInt16* quadrant, + Int32 nblock, + Int32 lo, + Int32 hi, + Int32 d, + Int32* budget ) +{ + Int32 i, j, h, bigN, hp; + UInt32 v; + + bigN = hi - lo + 1; + if (bigN < 2) return; + + hp = 0; + while (incs[hp] < bigN) hp++; + hp--; + + for (; hp >= 0; hp--) { + h = incs[hp]; + + i = lo + h; + while (True) { + + /*-- copy 1 --*/ + if (i > hi) break; + v = ptr[i]; + j = i; + while ( mainGtU ( + ptr[j-h]+d, v+d, block, quadrant, nblock, budget + ) ) { + ptr[j] = ptr[j-h]; + j = j - h; + if (j <= (lo + h - 1)) break; + } + ptr[j] = v; + i++; + + /*-- copy 2 --*/ + if (i > hi) break; + v = ptr[i]; + j = i; + while ( mainGtU ( + ptr[j-h]+d, v+d, block, quadrant, nblock, budget + ) ) { + ptr[j] = ptr[j-h]; + j = j - h; + if (j <= (lo + h - 1)) break; + } + ptr[j] = v; + i++; + + /*-- copy 3 --*/ + if (i > hi) break; + v = ptr[i]; + j = i; + while ( mainGtU ( + ptr[j-h]+d, v+d, block, quadrant, nblock, budget + ) ) { + ptr[j] = ptr[j-h]; + j = j - h; + if (j <= (lo + h - 1)) break; + } + ptr[j] = v; + i++; + + if (*budget < 0) return; + } + } +} + + +/*---------------------------------------------*/ +/*-- + The following is an implementation of + an elegant 3-way quicksort for strings, + described in a paper "Fast Algorithms for + Sorting and Searching Strings", by Robert + Sedgewick and Jon L. Bentley. +--*/ + +#define mswap(zz1, zz2) \ + { Int32 zztmp = zz1; zz1 = zz2; zz2 = zztmp; } + +#define mvswap(zzp1, zzp2, zzn) \ +{ \ + Int32 yyp1 = (zzp1); \ + Int32 yyp2 = (zzp2); \ + Int32 yyn = (zzn); \ + while (yyn > 0) { \ + mswap(ptr[yyp1], ptr[yyp2]); \ + yyp1++; yyp2++; yyn--; \ + } \ +} + +static +__inline__ +UChar mmed3 ( UChar a, UChar b, UChar c ) +{ + UChar t; + if (a > b) { t = a; a = b; b = t; }; + if (b > c) { + b = c; + if (a > b) b = a; + } + return b; +} + +#define mmin(a,b) ((a) < (b)) ? (a) : (b) + +#define mpush(lz,hz,dz) { stackLo[sp] = lz; \ + stackHi[sp] = hz; \ + stackD [sp] = dz; \ + sp++; } + +#define mpop(lz,hz,dz) { sp--; \ + lz = stackLo[sp]; \ + hz = stackHi[sp]; \ + dz = stackD [sp]; } + + +#define mnextsize(az) (nextHi[az]-nextLo[az]) + +#define mnextswap(az,bz) \ + { Int32 tz; \ + tz = nextLo[az]; nextLo[az] = nextLo[bz]; nextLo[bz] = tz; \ + tz = nextHi[az]; nextHi[az] = nextHi[bz]; nextHi[bz] = tz; \ + tz = nextD [az]; nextD [az] = nextD [bz]; nextD [bz] = tz; } + + +#define MAIN_QSORT_SMALL_THRESH 20 +#define MAIN_QSORT_DEPTH_THRESH (BZ_N_RADIX + BZ_N_QSORT) +#define MAIN_QSORT_STACK_SIZE 100 + +static +void mainQSort3 ( UInt32* ptr, + UChar* block, + UInt16* quadrant, + Int32 nblock, + Int32 loSt, + Int32 hiSt, + Int32 dSt, + Int32* budget ) +{ + Int32 unLo, unHi, ltLo, gtHi, n, m, med; + Int32 sp, lo, hi, d; + + Int32 stackLo[MAIN_QSORT_STACK_SIZE]; + Int32 stackHi[MAIN_QSORT_STACK_SIZE]; + Int32 stackD [MAIN_QSORT_STACK_SIZE]; + + Int32 nextLo[3]; + Int32 nextHi[3]; + Int32 nextD [3]; + + sp = 0; + mpush ( loSt, hiSt, dSt ); + + while (sp > 0) { + + AssertH ( sp < MAIN_QSORT_STACK_SIZE - 2, 1001 ); + + mpop ( lo, hi, d ); + if (hi - lo < MAIN_QSORT_SMALL_THRESH || + d > MAIN_QSORT_DEPTH_THRESH) { + mainSimpleSort ( ptr, block, quadrant, nblock, lo, hi, d, budget ); + if (*budget < 0) return; + continue; + } + + med = (Int32) + mmed3 ( block[ptr[ lo ]+d], + block[ptr[ hi ]+d], + block[ptr[ (lo+hi)>>1 ]+d] ); + + unLo = ltLo = lo; + unHi = gtHi = hi; + + while (True) { + while (True) { + if (unLo > unHi) break; + n = ((Int32)block[ptr[unLo]+d]) - med; + if (n == 0) { + mswap(ptr[unLo], ptr[ltLo]); + ltLo++; unLo++; continue; + }; + if (n > 0) break; + unLo++; + } + while (True) { + if (unLo > unHi) break; + n = ((Int32)block[ptr[unHi]+d]) - med; + if (n == 0) { + mswap(ptr[unHi], ptr[gtHi]); + gtHi--; unHi--; continue; + }; + if (n < 0) break; + unHi--; + } + if (unLo > unHi) break; + mswap(ptr[unLo], ptr[unHi]); unLo++; unHi--; + } + + AssertD ( unHi == unLo-1, "mainQSort3(2)" ); + + if (gtHi < ltLo) { + mpush(lo, hi, d+1 ); + continue; + } + + n = mmin(ltLo-lo, unLo-ltLo); mvswap(lo, unLo-n, n); + m = mmin(hi-gtHi, gtHi-unHi); mvswap(unLo, hi-m+1, m); + + n = lo + unLo - ltLo - 1; + m = hi - (gtHi - unHi) + 1; + + nextLo[0] = lo; nextHi[0] = n; nextD[0] = d; + nextLo[1] = m; nextHi[1] = hi; nextD[1] = d; + nextLo[2] = n+1; nextHi[2] = m-1; nextD[2] = d+1; + + if (mnextsize(0) < mnextsize(1)) mnextswap(0,1); + if (mnextsize(1) < mnextsize(2)) mnextswap(1,2); + if (mnextsize(0) < mnextsize(1)) mnextswap(0,1); + + AssertD (mnextsize(0) >= mnextsize(1), "mainQSort3(8)" ); + AssertD (mnextsize(1) >= mnextsize(2), "mainQSort3(9)" ); + + mpush (nextLo[0], nextHi[0], nextD[0]); + mpush (nextLo[1], nextHi[1], nextD[1]); + mpush (nextLo[2], nextHi[2], nextD[2]); + } +} + +#undef mswap +#undef mvswap +#undef mpush +#undef mpop +#undef mmin +#undef mnextsize +#undef mnextswap +#undef MAIN_QSORT_SMALL_THRESH +#undef MAIN_QSORT_DEPTH_THRESH +#undef MAIN_QSORT_STACK_SIZE + + +/*---------------------------------------------*/ +/* Pre: + nblock > N_OVERSHOOT + block32 exists for [0 .. nblock-1 +N_OVERSHOOT] + ((UChar*)block32) [0 .. nblock-1] holds block + ptr exists for [0 .. nblock-1] + + Post: + ((UChar*)block32) [0 .. nblock-1] holds block + All other areas of block32 destroyed + ftab [0 .. 65536 ] destroyed + ptr [0 .. nblock-1] holds sorted order + if (*budget < 0), sorting was abandoned +*/ + +#define BIGFREQ(b) (ftab[((b)+1) << 8] - ftab[(b) << 8]) +#define SETMASK (1 << 21) +#define CLEARMASK (~(SETMASK)) + +static +void mainSort ( UInt32* ptr, + UChar* block, + UInt16* quadrant, + UInt32* ftab, + Int32 nblock, + Int32 verb, + Int32* budget ) +{ + Int32 i, j, k, ss, sb; + Int32 runningOrder[256]; + Bool bigDone[256]; + Int32 copyStart[256]; + Int32 copyEnd [256]; + UChar c1; + Int32 numQSorted; + UInt16 s; + if (verb >= 4) VPrintf0 ( " main sort initialise ...\n" ); + + /*-- set up the 2-byte frequency table --*/ + for (i = 65536; i >= 0; i--) ftab[i] = 0; + + j = block[0] << 8; + i = nblock-1; + for (; i >= 3; i -= 4) { + quadrant[i] = 0; + j = (j >> 8) | ( ((UInt16)block[i]) << 8); + ftab[j]++; + quadrant[i-1] = 0; + j = (j >> 8) | ( ((UInt16)block[i-1]) << 8); + ftab[j]++; + quadrant[i-2] = 0; + j = (j >> 8) | ( ((UInt16)block[i-2]) << 8); + ftab[j]++; + quadrant[i-3] = 0; + j = (j >> 8) | ( ((UInt16)block[i-3]) << 8); + ftab[j]++; + } + for (; i >= 0; i--) { + quadrant[i] = 0; + j = (j >> 8) | ( ((UInt16)block[i]) << 8); + ftab[j]++; + } + + /*-- (emphasises close relationship of block & quadrant) --*/ + for (i = 0; i < BZ_N_OVERSHOOT; i++) { + block [nblock+i] = block[i]; + quadrant[nblock+i] = 0; + } + + if (verb >= 4) VPrintf0 ( " bucket sorting ...\n" ); + + /*-- Complete the initial radix sort --*/ + for (i = 1; i <= 65536; i++) ftab[i] += ftab[i-1]; + + s = block[0] << 8; + i = nblock-1; + for (; i >= 3; i -= 4) { + s = (s >> 8) | (block[i] << 8); + j = ftab[s] -1; + ftab[s] = j; + ptr[j] = i; + s = (s >> 8) | (block[i-1] << 8); + j = ftab[s] -1; + ftab[s] = j; + ptr[j] = i-1; + s = (s >> 8) | (block[i-2] << 8); + j = ftab[s] -1; + ftab[s] = j; + ptr[j] = i-2; + s = (s >> 8) | (block[i-3] << 8); + j = ftab[s] -1; + ftab[s] = j; + ptr[j] = i-3; + } + for (; i >= 0; i--) { + s = (s >> 8) | (block[i] << 8); + j = ftab[s] -1; + ftab[s] = j; + ptr[j] = i; + } + + /*-- + Now ftab contains the first loc of every small bucket. + Calculate the running order, from smallest to largest + big bucket. + --*/ + for (i = 0; i <= 255; i++) { + bigDone [i] = False; + runningOrder[i] = i; + } + + { + Int32 vv; + Int32 h = 1; + do h = 3 * h + 1; while (h <= 256); + do { + h = h / 3; + for (i = h; i <= 255; i++) { + vv = runningOrder[i]; + j = i; + while ( BIGFREQ(runningOrder[j-h]) > BIGFREQ(vv) ) { + runningOrder[j] = runningOrder[j-h]; + j = j - h; + if (j <= (h - 1)) goto zero; + } + zero: + runningOrder[j] = vv; + } + } while (h != 1); + } + + /*-- + The main sorting loop. + --*/ + + numQSorted = 0; + + for (i = 0; i <= 255; i++) { + + /*-- + Process big buckets, starting with the least full. + Basically this is a 3-step process in which we call + mainQSort3 to sort the small buckets [ss, j], but + also make a big effort to avoid the calls if we can. + --*/ + ss = runningOrder[i]; + + /*-- + Step 1: + Complete the big bucket [ss] by quicksorting + any unsorted small buckets [ss, j], for j != ss. + Hopefully previous pointer-scanning phases have already + completed many of the small buckets [ss, j], so + we don't have to sort them at all. + --*/ + for (j = 0; j <= 255; j++) { + if (j != ss) { + sb = (ss << 8) + j; + if ( ! (ftab[sb] & SETMASK) ) { + Int32 lo = ftab[sb] & CLEARMASK; + Int32 hi = (ftab[sb+1] & CLEARMASK) - 1; + if (hi > lo) { + if (verb >= 4) + VPrintf4 ( " qsort [0x%x, 0x%x] " + "done %d this %d\n", + ss, j, numQSorted, hi - lo + 1 ); + mainQSort3 ( + ptr, block, quadrant, nblock, + lo, hi, BZ_N_RADIX, budget + ); + numQSorted += (hi - lo + 1); + if (*budget < 0) return; + } + } + ftab[sb] |= SETMASK; + } + } + + AssertH ( !bigDone[ss], 1006 ); + + /*-- + Step 2: + Now scan this big bucket [ss] so as to synthesise the + sorted order for small buckets [t, ss] for all t, + including, magically, the bucket [ss,ss] too. + This will avoid doing Real Work in subsequent Step 1's. + --*/ + { + for (j = 0; j <= 255; j++) { + copyStart[j] = ftab[(j << 8) + ss] & CLEARMASK; + copyEnd [j] = (ftab[(j << 8) + ss + 1] & CLEARMASK) - 1; + } + for (j = ftab[ss << 8] & CLEARMASK; j < copyStart[ss]; j++) { + k = ptr[j]-1; if (k < 0) k += nblock; + c1 = block[k]; + if (!bigDone[c1]) + ptr[ copyStart[c1]++ ] = k; + } + for (j = (ftab[(ss+1) << 8] & CLEARMASK) - 1; j > copyEnd[ss]; j--) { + k = ptr[j]-1; if (k < 0) k += nblock; + c1 = block[k]; + if (!bigDone[c1]) + ptr[ copyEnd[c1]-- ] = k; + } + } + + AssertH ( (copyStart[ss]-1 == copyEnd[ss]) + || + /* Extremely rare case missing in bzip2-1.0.0 and 1.0.1. + Necessity for this case is demonstrated by compressing + a sequence of approximately 48.5 million of character + 251; 1.0.0/1.0.1 will then die here. */ + (copyStart[ss] == 0 && copyEnd[ss] == nblock-1), + 1007 ) + + for (j = 0; j <= 255; j++) ftab[(j << 8) + ss] |= SETMASK; + + /*-- + Step 3: + The [ss] big bucket is now done. Record this fact, + and update the quadrant descriptors. Remember to + update quadrants in the overshoot area too, if + necessary. The "if (i < 255)" test merely skips + this updating for the last bucket processed, since + updating for the last bucket is pointless. + + The quadrant array provides a way to incrementally + cache sort orderings, as they appear, so as to + make subsequent comparisons in fullGtU() complete + faster. For repetitive blocks this makes a big + difference (but not big enough to be able to avoid + the fallback sorting mechanism, exponential radix sort). + + The precise meaning is: at all times: + + for 0 <= i < nblock and 0 <= j <= nblock + + if block[i] != block[j], + + then the relative values of quadrant[i] and + quadrant[j] are meaningless. + + else { + if quadrant[i] < quadrant[j] + then the string starting at i lexicographically + precedes the string starting at j + + else if quadrant[i] > quadrant[j] + then the string starting at j lexicographically + precedes the string starting at i + + else + the relative ordering of the strings starting + at i and j has not yet been determined. + } + --*/ + bigDone[ss] = True; + + if (i < 255) { + Int32 bbStart = ftab[ss << 8] & CLEARMASK; + Int32 bbSize = (ftab[(ss+1) << 8] & CLEARMASK) - bbStart; + Int32 shifts = 0; + + while ((bbSize >> shifts) > 65534) shifts++; + + for (j = bbSize-1; j >= 0; j--) { + Int32 a2update = ptr[bbStart + j]; + UInt16 qVal = (UInt16)(j >> shifts); + quadrant[a2update] = qVal; + if (a2update < BZ_N_OVERSHOOT) + quadrant[a2update + nblock] = qVal; + } + AssertH ( ((bbSize-1) >> shifts) <= 65535, 1002 ); + } + + } + + if (verb >= 4) + VPrintf3 ( " %d pointers, %d sorted, %d scanned\n", + nblock, numQSorted, nblock - numQSorted ); +} + +#undef BIGFREQ +#undef SETMASK +#undef CLEARMASK + + +/*---------------------------------------------*/ +/* Pre: + nblock > 0 + arr2 exists for [0 .. nblock-1 +N_OVERSHOOT] + ((UChar*)arr2) [0 .. nblock-1] holds block + arr1 exists for [0 .. nblock-1] + + Post: + ((UChar*)arr2) [0 .. nblock-1] holds block + All other areas of block destroyed + ftab [ 0 .. 65536 ] destroyed + arr1 [0 .. nblock-1] holds sorted order +*/ +void BZ2_blockSort ( EState* s ) +{ + UInt32* ptr = s->ptr; + UChar* block = s->block; + UInt32* ftab = s->ftab; + Int32 nblock = s->nblock; + Int32 verb = s->verbosity; + Int32 wfact = s->workFactor; + UInt16* quadrant; + Int32 budget; + Int32 budgetInit; + Int32 i; + + if (nblock < 10000) { + fallbackSort ( s->arr1, s->arr2, ftab, nblock, verb ); + } else { + /* Calculate the location for quadrant, remembering to get + the alignment right. Assumes that &(block[0]) is at least + 2-byte aligned -- this should be ok since block is really + the first section of arr2. + */ + i = nblock+BZ_N_OVERSHOOT; + if (i & 1) i++; + quadrant = (UInt16*)(&(block[i])); + + /* (wfact-1) / 3 puts the default-factor-30 + transition point at very roughly the same place as + with v0.1 and v0.9.0. + Not that it particularly matters any more, since the + resulting compressed stream is now the same regardless + of whether or not we use the main sort or fallback sort. + */ + if (wfact < 1 ) wfact = 1; + if (wfact > 100) wfact = 100; + budgetInit = nblock * ((wfact-1) / 3); + budget = budgetInit; + + mainSort ( ptr, block, quadrant, ftab, nblock, verb, &budget ); + if (verb >= 3) + VPrintf3 ( " %d work, %d block, ratio %5.2f\n", + budgetInit - budget, + nblock, + (float)(budgetInit - budget) / + (float)(nblock==0 ? 1 : nblock) ); + if (budget < 0) { + if (verb >= 2) + VPrintf0 ( " too repetitive; using fallback" + " sorting algorithm\n" ); + fallbackSort ( s->arr1, s->arr2, ftab, nblock, verb ); + } + } + + s->origPtr = -1; + for (i = 0; i < s->nblock; i++) + if (ptr[i] == 0) + { s->origPtr = i; break; }; + + AssertH( s->origPtr != -1, 1003 ); +} + + +/*-------------------------------------------------------------*/ +/*--- end blocksort.c ---*/ +/*-------------------------------------------------------------*/ diff --git a/lib/libbz2/bz_error.c b/lib/libbz2/bz_error.c new file mode 100644 index 00000000..b55b83d9 --- /dev/null +++ b/lib/libbz2/bz_error.c @@ -0,0 +1,10 @@ +// bz_internal_error implementation for BZ_NO_STDIO builds +// This function is called when bzip2 detects an internal error + +#include "bzlib.h" + +void bz_internal_error(int errcode) { + // In embedded systems, we let the error propagate through return codes + // The calling code checks return values and handles errors appropriately + (void)errcode; // Unused parameter +} diff --git a/lib/libbz2/bzlib.c b/lib/libbz2/bzlib.c new file mode 100644 index 00000000..21786551 --- /dev/null +++ b/lib/libbz2/bzlib.c @@ -0,0 +1,1572 @@ + +/*-------------------------------------------------------------*/ +/*--- Library top-level functions. ---*/ +/*--- bzlib.c ---*/ +/*-------------------------------------------------------------*/ + +/* ------------------------------------------------------------------ + This file is part of bzip2/libbzip2, a program and library for + lossless, block-sorting data compression. + + bzip2/libbzip2 version 1.0.8 of 13 July 2019 + Copyright (C) 1996-2019 Julian Seward + + Please read the WARNING, DISCLAIMER and PATENTS sections in the + README file. + + This program is released under the terms of the license contained + in the file LICENSE. + ------------------------------------------------------------------ */ + +/* CHANGES + 0.9.0 -- original version. + 0.9.0a/b -- no changes in this file. + 0.9.0c -- made zero-length BZ_FLUSH work correctly in bzCompress(). + fixed bzWrite/bzRead to ignore zero-length requests. + fixed bzread to correctly handle read requests after EOF. + wrong parameter order in call to bzDecompressInit in + bzBuffToBuffDecompress. Fixed. +*/ + +#include "bzlib_private.h" + + +/*---------------------------------------------------*/ +/*--- Compression stuff ---*/ +/*---------------------------------------------------*/ + + +/*---------------------------------------------------*/ +#ifndef BZ_NO_STDIO +void BZ2_bz__AssertH__fail ( int errcode ) +{ + fprintf(stderr, + "\n\nbzip2/libbzip2: internal error number %d.\n" + "This is a bug in bzip2/libbzip2, %s.\n" + "Please report it to: bzip2-devel@sourceware.org. If this happened\n" + "when you were using some program which uses libbzip2 as a\n" + "component, you should also report this bug to the author(s)\n" + "of that program. Please make an effort to report this bug;\n" + "timely and accurate bug reports eventually lead to higher\n" + "quality software. Thanks.\n\n", + errcode, + BZ2_bzlibVersion() + ); + + if (errcode == 1007) { + fprintf(stderr, + "\n*** A special note about internal error number 1007 ***\n" + "\n" + "Experience suggests that a common cause of i.e. 1007\n" + "is unreliable memory or other hardware. The 1007 assertion\n" + "just happens to cross-check the results of huge numbers of\n" + "memory reads/writes, and so acts (unintendedly) as a stress\n" + "test of your memory system.\n" + "\n" + "I suggest the following: try compressing the file again,\n" + "possibly monitoring progress in detail with the -vv flag.\n" + "\n" + "* If the error cannot be reproduced, and/or happens at different\n" + " points in compression, you may have a flaky memory system.\n" + " Try a memory-test program. I have used Memtest86\n" + " (www.memtest86.com). At the time of writing it is free (GPLd).\n" + " Memtest86 tests memory much more thorougly than your BIOSs\n" + " power-on test, and may find failures that the BIOS doesn't.\n" + "\n" + "* If the error can be repeatably reproduced, this is a bug in\n" + " bzip2, and I would very much like to hear about it. Please\n" + " let me know, and, ideally, save a copy of the file causing the\n" + " problem -- without which I will be unable to investigate it.\n" + "\n" + ); + } + + exit(3); +} +#endif + + +/*---------------------------------------------------*/ +static +int bz_config_ok ( void ) +{ + if (sizeof(int) != 4) return 0; + if (sizeof(short) != 2) return 0; + if (sizeof(char) != 1) return 0; + return 1; +} + + +/*---------------------------------------------------*/ +static +void* default_bzalloc ( void* opaque, Int32 items, Int32 size ) +{ + void* v = malloc ( items * size ); + return v; +} + +static +void default_bzfree ( void* opaque, void* addr ) +{ + if (addr != NULL) free ( addr ); +} + + +/*---------------------------------------------------*/ +static +void prepare_new_block ( EState* s ) +{ + Int32 i; + s->nblock = 0; + s->numZ = 0; + s->state_out_pos = 0; + BZ_INITIALISE_CRC ( s->blockCRC ); + for (i = 0; i < 256; i++) s->inUse[i] = False; + s->blockNo++; +} + + +/*---------------------------------------------------*/ +static +void init_RL ( EState* s ) +{ + s->state_in_ch = 256; + s->state_in_len = 0; +} + + +static +Bool isempty_RL ( EState* s ) +{ + if (s->state_in_ch < 256 && s->state_in_len > 0) + return False; else + return True; +} + + +/*---------------------------------------------------*/ +int BZ_API(BZ2_bzCompressInit) + ( bz_stream* strm, + int blockSize100k, + int verbosity, + int workFactor ) +{ + Int32 n; + EState* s; + + if (!bz_config_ok()) return BZ_CONFIG_ERROR; + + if (strm == NULL || + blockSize100k < 1 || blockSize100k > 9 || + workFactor < 0 || workFactor > 250) + return BZ_PARAM_ERROR; + + if (workFactor == 0) workFactor = 30; + if (strm->bzalloc == NULL) strm->bzalloc = default_bzalloc; + if (strm->bzfree == NULL) strm->bzfree = default_bzfree; + + s = BZALLOC( sizeof(EState) ); + if (s == NULL) return BZ_MEM_ERROR; + s->strm = strm; + + s->arr1 = NULL; + s->arr2 = NULL; + s->ftab = NULL; + + n = 100000 * blockSize100k; + s->arr1 = BZALLOC( n * sizeof(UInt32) ); + s->arr2 = BZALLOC( (n+BZ_N_OVERSHOOT) * sizeof(UInt32) ); + s->ftab = BZALLOC( 65537 * sizeof(UInt32) ); + + if (s->arr1 == NULL || s->arr2 == NULL || s->ftab == NULL) { + if (s->arr1 != NULL) BZFREE(s->arr1); + if (s->arr2 != NULL) BZFREE(s->arr2); + if (s->ftab != NULL) BZFREE(s->ftab); + if (s != NULL) BZFREE(s); + return BZ_MEM_ERROR; + } + + s->blockNo = 0; + s->state = BZ_S_INPUT; + s->mode = BZ_M_RUNNING; + s->combinedCRC = 0; + s->blockSize100k = blockSize100k; + s->nblockMAX = 100000 * blockSize100k - 19; + s->verbosity = verbosity; + s->workFactor = workFactor; + + s->block = (UChar*)s->arr2; + s->mtfv = (UInt16*)s->arr1; + s->zbits = NULL; + s->ptr = (UInt32*)s->arr1; + + strm->state = s; + strm->total_in_lo32 = 0; + strm->total_in_hi32 = 0; + strm->total_out_lo32 = 0; + strm->total_out_hi32 = 0; + init_RL ( s ); + prepare_new_block ( s ); + return BZ_OK; +} + + +/*---------------------------------------------------*/ +static +void add_pair_to_block ( EState* s ) +{ + Int32 i; + UChar ch = (UChar)(s->state_in_ch); + for (i = 0; i < s->state_in_len; i++) { + BZ_UPDATE_CRC( s->blockCRC, ch ); + } + s->inUse[s->state_in_ch] = True; + switch (s->state_in_len) { + case 1: + s->block[s->nblock] = (UChar)ch; s->nblock++; + break; + case 2: + s->block[s->nblock] = (UChar)ch; s->nblock++; + s->block[s->nblock] = (UChar)ch; s->nblock++; + break; + case 3: + s->block[s->nblock] = (UChar)ch; s->nblock++; + s->block[s->nblock] = (UChar)ch; s->nblock++; + s->block[s->nblock] = (UChar)ch; s->nblock++; + break; + default: + s->inUse[s->state_in_len-4] = True; + s->block[s->nblock] = (UChar)ch; s->nblock++; + s->block[s->nblock] = (UChar)ch; s->nblock++; + s->block[s->nblock] = (UChar)ch; s->nblock++; + s->block[s->nblock] = (UChar)ch; s->nblock++; + s->block[s->nblock] = ((UChar)(s->state_in_len-4)); + s->nblock++; + break; + } +} + + +/*---------------------------------------------------*/ +static +void flush_RL ( EState* s ) +{ + if (s->state_in_ch < 256) add_pair_to_block ( s ); + init_RL ( s ); +} + + +/*---------------------------------------------------*/ +#define ADD_CHAR_TO_BLOCK(zs,zchh0) \ +{ \ + UInt32 zchh = (UInt32)(zchh0); \ + /*-- fast track the common case --*/ \ + if (zchh != zs->state_in_ch && \ + zs->state_in_len == 1) { \ + UChar ch = (UChar)(zs->state_in_ch); \ + BZ_UPDATE_CRC( zs->blockCRC, ch ); \ + zs->inUse[zs->state_in_ch] = True; \ + zs->block[zs->nblock] = (UChar)ch; \ + zs->nblock++; \ + zs->state_in_ch = zchh; \ + } \ + else \ + /*-- general, uncommon cases --*/ \ + if (zchh != zs->state_in_ch || \ + zs->state_in_len == 255) { \ + if (zs->state_in_ch < 256) \ + add_pair_to_block ( zs ); \ + zs->state_in_ch = zchh; \ + zs->state_in_len = 1; \ + } else { \ + zs->state_in_len++; \ + } \ +} + + +/*---------------------------------------------------*/ +static +Bool copy_input_until_stop ( EState* s ) +{ + Bool progress_in = False; + + if (s->mode == BZ_M_RUNNING) { + + /*-- fast track the common case --*/ + while (True) { + /*-- block full? --*/ + if (s->nblock >= s->nblockMAX) break; + /*-- no input? --*/ + if (s->strm->avail_in == 0) break; + progress_in = True; + ADD_CHAR_TO_BLOCK ( s, (UInt32)(*((UChar*)(s->strm->next_in))) ); + s->strm->next_in++; + s->strm->avail_in--; + s->strm->total_in_lo32++; + if (s->strm->total_in_lo32 == 0) s->strm->total_in_hi32++; + } + + } else { + + /*-- general, uncommon case --*/ + while (True) { + /*-- block full? --*/ + if (s->nblock >= s->nblockMAX) break; + /*-- no input? --*/ + if (s->strm->avail_in == 0) break; + /*-- flush/finish end? --*/ + if (s->avail_in_expect == 0) break; + progress_in = True; + ADD_CHAR_TO_BLOCK ( s, (UInt32)(*((UChar*)(s->strm->next_in))) ); + s->strm->next_in++; + s->strm->avail_in--; + s->strm->total_in_lo32++; + if (s->strm->total_in_lo32 == 0) s->strm->total_in_hi32++; + s->avail_in_expect--; + } + } + return progress_in; +} + + +/*---------------------------------------------------*/ +static +Bool copy_output_until_stop ( EState* s ) +{ + Bool progress_out = False; + + while (True) { + + /*-- no output space? --*/ + if (s->strm->avail_out == 0) break; + + /*-- block done? --*/ + if (s->state_out_pos >= s->numZ) break; + + progress_out = True; + *(s->strm->next_out) = s->zbits[s->state_out_pos]; + s->state_out_pos++; + s->strm->avail_out--; + s->strm->next_out++; + s->strm->total_out_lo32++; + if (s->strm->total_out_lo32 == 0) s->strm->total_out_hi32++; + } + + return progress_out; +} + + +/*---------------------------------------------------*/ +static +Bool handle_compress ( bz_stream* strm ) +{ + Bool progress_in = False; + Bool progress_out = False; + EState* s = strm->state; + + while (True) { + + if (s->state == BZ_S_OUTPUT) { + progress_out |= copy_output_until_stop ( s ); + if (s->state_out_pos < s->numZ) break; + if (s->mode == BZ_M_FINISHING && + s->avail_in_expect == 0 && + isempty_RL(s)) break; + prepare_new_block ( s ); + s->state = BZ_S_INPUT; + if (s->mode == BZ_M_FLUSHING && + s->avail_in_expect == 0 && + isempty_RL(s)) break; + } + + if (s->state == BZ_S_INPUT) { + progress_in |= copy_input_until_stop ( s ); + if (s->mode != BZ_M_RUNNING && s->avail_in_expect == 0) { + flush_RL ( s ); + BZ2_compressBlock ( s, (Bool)(s->mode == BZ_M_FINISHING) ); + s->state = BZ_S_OUTPUT; + } + else + if (s->nblock >= s->nblockMAX) { + BZ2_compressBlock ( s, False ); + s->state = BZ_S_OUTPUT; + } + else + if (s->strm->avail_in == 0) { + break; + } + } + + } + + return progress_in || progress_out; +} + + +/*---------------------------------------------------*/ +int BZ_API(BZ2_bzCompress) ( bz_stream *strm, int action ) +{ + Bool progress; + EState* s; + if (strm == NULL) return BZ_PARAM_ERROR; + s = strm->state; + if (s == NULL) return BZ_PARAM_ERROR; + if (s->strm != strm) return BZ_PARAM_ERROR; + + preswitch: + switch (s->mode) { + + case BZ_M_IDLE: + return BZ_SEQUENCE_ERROR; + + case BZ_M_RUNNING: + if (action == BZ_RUN) { + progress = handle_compress ( strm ); + return progress ? BZ_RUN_OK : BZ_PARAM_ERROR; + } + else + if (action == BZ_FLUSH) { + s->avail_in_expect = strm->avail_in; + s->mode = BZ_M_FLUSHING; + goto preswitch; + } + else + if (action == BZ_FINISH) { + s->avail_in_expect = strm->avail_in; + s->mode = BZ_M_FINISHING; + goto preswitch; + } + else + return BZ_PARAM_ERROR; + + case BZ_M_FLUSHING: + if (action != BZ_FLUSH) return BZ_SEQUENCE_ERROR; + if (s->avail_in_expect != s->strm->avail_in) + return BZ_SEQUENCE_ERROR; + progress = handle_compress ( strm ); + if (s->avail_in_expect > 0 || !isempty_RL(s) || + s->state_out_pos < s->numZ) return BZ_FLUSH_OK; + s->mode = BZ_M_RUNNING; + return BZ_RUN_OK; + + case BZ_M_FINISHING: + if (action != BZ_FINISH) return BZ_SEQUENCE_ERROR; + if (s->avail_in_expect != s->strm->avail_in) + return BZ_SEQUENCE_ERROR; + progress = handle_compress ( strm ); + if (!progress) return BZ_SEQUENCE_ERROR; + if (s->avail_in_expect > 0 || !isempty_RL(s) || + s->state_out_pos < s->numZ) return BZ_FINISH_OK; + s->mode = BZ_M_IDLE; + return BZ_STREAM_END; + } + return BZ_OK; /*--not reached--*/ +} + + +/*---------------------------------------------------*/ +int BZ_API(BZ2_bzCompressEnd) ( bz_stream *strm ) +{ + EState* s; + if (strm == NULL) return BZ_PARAM_ERROR; + s = strm->state; + if (s == NULL) return BZ_PARAM_ERROR; + if (s->strm != strm) return BZ_PARAM_ERROR; + + if (s->arr1 != NULL) BZFREE(s->arr1); + if (s->arr2 != NULL) BZFREE(s->arr2); + if (s->ftab != NULL) BZFREE(s->ftab); + BZFREE(strm->state); + + strm->state = NULL; + + return BZ_OK; +} + + +/*---------------------------------------------------*/ +/*--- Decompression stuff ---*/ +/*---------------------------------------------------*/ + +/*---------------------------------------------------*/ +int BZ_API(BZ2_bzDecompressInit) + ( bz_stream* strm, + int verbosity, + int small ) +{ + DState* s; + + if (!bz_config_ok()) return BZ_CONFIG_ERROR; + + if (strm == NULL) return BZ_PARAM_ERROR; + if (small != 0 && small != 1) return BZ_PARAM_ERROR; + if (verbosity < 0 || verbosity > 4) return BZ_PARAM_ERROR; + + if (strm->bzalloc == NULL) strm->bzalloc = default_bzalloc; + if (strm->bzfree == NULL) strm->bzfree = default_bzfree; + + s = BZALLOC( sizeof(DState) ); + if (s == NULL) return BZ_MEM_ERROR; + s->strm = strm; + strm->state = s; + s->state = BZ_X_MAGIC_1; + s->bsLive = 0; + s->bsBuff = 0; + s->calculatedCombinedCRC = 0; + strm->total_in_lo32 = 0; + strm->total_in_hi32 = 0; + strm->total_out_lo32 = 0; + strm->total_out_hi32 = 0; + s->smallDecompress = (Bool)small; + s->ll4 = NULL; + s->ll16 = NULL; + s->tt = NULL; + s->currBlockNo = 0; + s->verbosity = verbosity; + + return BZ_OK; +} + + +/*---------------------------------------------------*/ +/* Return True iff data corruption is discovered. + Returns False if there is no problem. +*/ +static +Bool unRLE_obuf_to_output_FAST ( DState* s ) +{ + UChar k1; + + if (s->blockRandomised) { + + while (True) { + /* try to finish existing run */ + while (True) { + if (s->strm->avail_out == 0) return False; + if (s->state_out_len == 0) break; + *( (UChar*)(s->strm->next_out) ) = s->state_out_ch; + BZ_UPDATE_CRC ( s->calculatedBlockCRC, s->state_out_ch ); + s->state_out_len--; + s->strm->next_out++; + s->strm->avail_out--; + s->strm->total_out_lo32++; + if (s->strm->total_out_lo32 == 0) s->strm->total_out_hi32++; + } + + /* can a new run be started? */ + if (s->nblock_used == s->save_nblock+1) return False; + + /* Only caused by corrupt data stream? */ + if (s->nblock_used > s->save_nblock+1) + return True; + + s->state_out_len = 1; + s->state_out_ch = s->k0; + BZ_GET_FAST(k1); BZ_RAND_UPD_MASK; + k1 ^= BZ_RAND_MASK; s->nblock_used++; + if (s->nblock_used == s->save_nblock+1) continue; + if (k1 != s->k0) { s->k0 = k1; continue; }; + + s->state_out_len = 2; + BZ_GET_FAST(k1); BZ_RAND_UPD_MASK; + k1 ^= BZ_RAND_MASK; s->nblock_used++; + if (s->nblock_used == s->save_nblock+1) continue; + if (k1 != s->k0) { s->k0 = k1; continue; }; + + s->state_out_len = 3; + BZ_GET_FAST(k1); BZ_RAND_UPD_MASK; + k1 ^= BZ_RAND_MASK; s->nblock_used++; + if (s->nblock_used == s->save_nblock+1) continue; + if (k1 != s->k0) { s->k0 = k1; continue; }; + + BZ_GET_FAST(k1); BZ_RAND_UPD_MASK; + k1 ^= BZ_RAND_MASK; s->nblock_used++; + s->state_out_len = ((Int32)k1) + 4; + BZ_GET_FAST(s->k0); BZ_RAND_UPD_MASK; + s->k0 ^= BZ_RAND_MASK; s->nblock_used++; + } + + } else { + + /* restore */ + UInt32 c_calculatedBlockCRC = s->calculatedBlockCRC; + UChar c_state_out_ch = s->state_out_ch; + Int32 c_state_out_len = s->state_out_len; + Int32 c_nblock_used = s->nblock_used; + Int32 c_k0 = s->k0; + UInt32* c_tt = s->tt; + UInt32 c_tPos = s->tPos; + char* cs_next_out = s->strm->next_out; + unsigned int cs_avail_out = s->strm->avail_out; + Int32 ro_blockSize100k = s->blockSize100k; + /* end restore */ + + UInt32 avail_out_INIT = cs_avail_out; + Int32 s_save_nblockPP = s->save_nblock+1; + unsigned int total_out_lo32_old; + + while (True) { + + /* try to finish existing run */ + if (c_state_out_len > 0) { + while (True) { + if (cs_avail_out == 0) goto return_notr; + if (c_state_out_len == 1) break; + *( (UChar*)(cs_next_out) ) = c_state_out_ch; + BZ_UPDATE_CRC ( c_calculatedBlockCRC, c_state_out_ch ); + c_state_out_len--; + cs_next_out++; + cs_avail_out--; + } + s_state_out_len_eq_one: + { + if (cs_avail_out == 0) { + c_state_out_len = 1; goto return_notr; + }; + *( (UChar*)(cs_next_out) ) = c_state_out_ch; + BZ_UPDATE_CRC ( c_calculatedBlockCRC, c_state_out_ch ); + cs_next_out++; + cs_avail_out--; + } + } + /* Only caused by corrupt data stream? */ + if (c_nblock_used > s_save_nblockPP) + return True; + + /* can a new run be started? */ + if (c_nblock_used == s_save_nblockPP) { + c_state_out_len = 0; goto return_notr; + }; + c_state_out_ch = c_k0; + BZ_GET_FAST_C(k1); c_nblock_used++; + if (k1 != c_k0) { + c_k0 = k1; goto s_state_out_len_eq_one; + }; + if (c_nblock_used == s_save_nblockPP) + goto s_state_out_len_eq_one; + + c_state_out_len = 2; + BZ_GET_FAST_C(k1); c_nblock_used++; + if (c_nblock_used == s_save_nblockPP) continue; + if (k1 != c_k0) { c_k0 = k1; continue; }; + + c_state_out_len = 3; + BZ_GET_FAST_C(k1); c_nblock_used++; + if (c_nblock_used == s_save_nblockPP) continue; + if (k1 != c_k0) { c_k0 = k1; continue; }; + + BZ_GET_FAST_C(k1); c_nblock_used++; + c_state_out_len = ((Int32)k1) + 4; + BZ_GET_FAST_C(c_k0); c_nblock_used++; + } + + return_notr: + total_out_lo32_old = s->strm->total_out_lo32; + s->strm->total_out_lo32 += (avail_out_INIT - cs_avail_out); + if (s->strm->total_out_lo32 < total_out_lo32_old) + s->strm->total_out_hi32++; + + /* save */ + s->calculatedBlockCRC = c_calculatedBlockCRC; + s->state_out_ch = c_state_out_ch; + s->state_out_len = c_state_out_len; + s->nblock_used = c_nblock_used; + s->k0 = c_k0; + s->tt = c_tt; + s->tPos = c_tPos; + s->strm->next_out = cs_next_out; + s->strm->avail_out = cs_avail_out; + /* end save */ + } + return False; +} + + + +/*---------------------------------------------------*/ +__inline__ Int32 BZ2_indexIntoF ( Int32 indx, Int32 *cftab ) +{ + Int32 nb, na, mid; + nb = 0; + na = 256; + do { + mid = (nb + na) >> 1; + if (indx >= cftab[mid]) nb = mid; else na = mid; + } + while (na - nb != 1); + return nb; +} + + +/*---------------------------------------------------*/ +/* Return True iff data corruption is discovered. + Returns False if there is no problem. +*/ +static +Bool unRLE_obuf_to_output_SMALL ( DState* s ) +{ + UChar k1; + + if (s->blockRandomised) { + + while (True) { + /* try to finish existing run */ + while (True) { + if (s->strm->avail_out == 0) return False; + if (s->state_out_len == 0) break; + *( (UChar*)(s->strm->next_out) ) = s->state_out_ch; + BZ_UPDATE_CRC ( s->calculatedBlockCRC, s->state_out_ch ); + s->state_out_len--; + s->strm->next_out++; + s->strm->avail_out--; + s->strm->total_out_lo32++; + if (s->strm->total_out_lo32 == 0) s->strm->total_out_hi32++; + } + + /* can a new run be started? */ + if (s->nblock_used == s->save_nblock+1) return False; + + /* Only caused by corrupt data stream? */ + if (s->nblock_used > s->save_nblock+1) + return True; + + s->state_out_len = 1; + s->state_out_ch = s->k0; + BZ_GET_SMALL(k1); BZ_RAND_UPD_MASK; + k1 ^= BZ_RAND_MASK; s->nblock_used++; + if (s->nblock_used == s->save_nblock+1) continue; + if (k1 != s->k0) { s->k0 = k1; continue; }; + + s->state_out_len = 2; + BZ_GET_SMALL(k1); BZ_RAND_UPD_MASK; + k1 ^= BZ_RAND_MASK; s->nblock_used++; + if (s->nblock_used == s->save_nblock+1) continue; + if (k1 != s->k0) { s->k0 = k1; continue; }; + + s->state_out_len = 3; + BZ_GET_SMALL(k1); BZ_RAND_UPD_MASK; + k1 ^= BZ_RAND_MASK; s->nblock_used++; + if (s->nblock_used == s->save_nblock+1) continue; + if (k1 != s->k0) { s->k0 = k1; continue; }; + + BZ_GET_SMALL(k1); BZ_RAND_UPD_MASK; + k1 ^= BZ_RAND_MASK; s->nblock_used++; + s->state_out_len = ((Int32)k1) + 4; + BZ_GET_SMALL(s->k0); BZ_RAND_UPD_MASK; + s->k0 ^= BZ_RAND_MASK; s->nblock_used++; + } + + } else { + + while (True) { + /* try to finish existing run */ + while (True) { + if (s->strm->avail_out == 0) return False; + if (s->state_out_len == 0) break; + *( (UChar*)(s->strm->next_out) ) = s->state_out_ch; + BZ_UPDATE_CRC ( s->calculatedBlockCRC, s->state_out_ch ); + s->state_out_len--; + s->strm->next_out++; + s->strm->avail_out--; + s->strm->total_out_lo32++; + if (s->strm->total_out_lo32 == 0) s->strm->total_out_hi32++; + } + + /* can a new run be started? */ + if (s->nblock_used == s->save_nblock+1) return False; + + /* Only caused by corrupt data stream? */ + if (s->nblock_used > s->save_nblock+1) + return True; + + s->state_out_len = 1; + s->state_out_ch = s->k0; + BZ_GET_SMALL(k1); s->nblock_used++; + if (s->nblock_used == s->save_nblock+1) continue; + if (k1 != s->k0) { s->k0 = k1; continue; }; + + s->state_out_len = 2; + BZ_GET_SMALL(k1); s->nblock_used++; + if (s->nblock_used == s->save_nblock+1) continue; + if (k1 != s->k0) { s->k0 = k1; continue; }; + + s->state_out_len = 3; + BZ_GET_SMALL(k1); s->nblock_used++; + if (s->nblock_used == s->save_nblock+1) continue; + if (k1 != s->k0) { s->k0 = k1; continue; }; + + BZ_GET_SMALL(k1); s->nblock_used++; + s->state_out_len = ((Int32)k1) + 4; + BZ_GET_SMALL(s->k0); s->nblock_used++; + } + + } +} + + +/*---------------------------------------------------*/ +int BZ_API(BZ2_bzDecompress) ( bz_stream *strm ) +{ + Bool corrupt; + DState* s; + if (strm == NULL) return BZ_PARAM_ERROR; + s = strm->state; + if (s == NULL) return BZ_PARAM_ERROR; + if (s->strm != strm) return BZ_PARAM_ERROR; + + while (True) { + if (s->state == BZ_X_IDLE) return BZ_SEQUENCE_ERROR; + if (s->state == BZ_X_OUTPUT) { + if (s->smallDecompress) + corrupt = unRLE_obuf_to_output_SMALL ( s ); else + corrupt = unRLE_obuf_to_output_FAST ( s ); + if (corrupt) return BZ_DATA_ERROR; + if (s->nblock_used == s->save_nblock+1 && s->state_out_len == 0) { + BZ_FINALISE_CRC ( s->calculatedBlockCRC ); + if (s->verbosity >= 3) + VPrintf2 ( " {0x%08x, 0x%08x}", s->storedBlockCRC, + s->calculatedBlockCRC ); + if (s->verbosity >= 2) VPrintf0 ( "]" ); + if (s->calculatedBlockCRC != s->storedBlockCRC) + return BZ_DATA_ERROR; + s->calculatedCombinedCRC + = (s->calculatedCombinedCRC << 1) | + (s->calculatedCombinedCRC >> 31); + s->calculatedCombinedCRC ^= s->calculatedBlockCRC; + s->state = BZ_X_BLKHDR_1; + } else { + return BZ_OK; + } + } + if (s->state >= BZ_X_MAGIC_1) { + Int32 r = BZ2_decompress ( s ); + if (r == BZ_STREAM_END) { + if (s->verbosity >= 3) + VPrintf2 ( "\n combined CRCs: stored = 0x%08x, computed = 0x%08x", + s->storedCombinedCRC, s->calculatedCombinedCRC ); + if (s->calculatedCombinedCRC != s->storedCombinedCRC) + return BZ_DATA_ERROR; + return r; + } + if (s->state != BZ_X_OUTPUT) return r; + } + } + + AssertH ( 0, 6001 ); + + return 0; /*NOTREACHED*/ +} + + +/*---------------------------------------------------*/ +int BZ_API(BZ2_bzDecompressEnd) ( bz_stream *strm ) +{ + DState* s; + if (strm == NULL) return BZ_PARAM_ERROR; + s = strm->state; + if (s == NULL) return BZ_PARAM_ERROR; + if (s->strm != strm) return BZ_PARAM_ERROR; + + if (s->tt != NULL) BZFREE(s->tt); + if (s->ll16 != NULL) BZFREE(s->ll16); + if (s->ll4 != NULL) BZFREE(s->ll4); + + BZFREE(strm->state); + strm->state = NULL; + + return BZ_OK; +} + + +#ifndef BZ_NO_STDIO +/*---------------------------------------------------*/ +/*--- File I/O stuff ---*/ +/*---------------------------------------------------*/ + +#define BZ_SETERR(eee) \ +{ \ + if (bzerror != NULL) *bzerror = eee; \ + if (bzf != NULL) bzf->lastErr = eee; \ +} + +typedef + struct { + FILE* handle; + Char buf[BZ_MAX_UNUSED]; + Int32 bufN; + Bool writing; + bz_stream strm; + Int32 lastErr; + Bool initialisedOk; + } + bzFile; + + +/*---------------------------------------------*/ +static Bool myfeof ( FILE* f ) +{ + Int32 c = fgetc ( f ); + if (c == EOF) return True; + ungetc ( c, f ); + return False; +} + + +/*---------------------------------------------------*/ +BZFILE* BZ_API(BZ2_bzWriteOpen) + ( int* bzerror, + FILE* f, + int blockSize100k, + int verbosity, + int workFactor ) +{ + Int32 ret; + bzFile* bzf = NULL; + + BZ_SETERR(BZ_OK); + + if (f == NULL || + (blockSize100k < 1 || blockSize100k > 9) || + (workFactor < 0 || workFactor > 250) || + (verbosity < 0 || verbosity > 4)) + { BZ_SETERR(BZ_PARAM_ERROR); return NULL; }; + + if (ferror(f)) + { BZ_SETERR(BZ_IO_ERROR); return NULL; }; + + bzf = malloc ( sizeof(bzFile) ); + if (bzf == NULL) + { BZ_SETERR(BZ_MEM_ERROR); return NULL; }; + + BZ_SETERR(BZ_OK); + bzf->initialisedOk = False; + bzf->bufN = 0; + bzf->handle = f; + bzf->writing = True; + bzf->strm.bzalloc = NULL; + bzf->strm.bzfree = NULL; + bzf->strm.opaque = NULL; + + if (workFactor == 0) workFactor = 30; + ret = BZ2_bzCompressInit ( &(bzf->strm), blockSize100k, + verbosity, workFactor ); + if (ret != BZ_OK) + { BZ_SETERR(ret); free(bzf); return NULL; }; + + bzf->strm.avail_in = 0; + bzf->initialisedOk = True; + return bzf; +} + + + +/*---------------------------------------------------*/ +void BZ_API(BZ2_bzWrite) + ( int* bzerror, + BZFILE* b, + void* buf, + int len ) +{ + Int32 n, n2, ret; + bzFile* bzf = (bzFile*)b; + + BZ_SETERR(BZ_OK); + if (bzf == NULL || buf == NULL || len < 0) + { BZ_SETERR(BZ_PARAM_ERROR); return; }; + if (!(bzf->writing)) + { BZ_SETERR(BZ_SEQUENCE_ERROR); return; }; + if (ferror(bzf->handle)) + { BZ_SETERR(BZ_IO_ERROR); return; }; + + if (len == 0) + { BZ_SETERR(BZ_OK); return; }; + + bzf->strm.avail_in = len; + bzf->strm.next_in = buf; + + while (True) { + bzf->strm.avail_out = BZ_MAX_UNUSED; + bzf->strm.next_out = bzf->buf; + ret = BZ2_bzCompress ( &(bzf->strm), BZ_RUN ); + if (ret != BZ_RUN_OK) + { BZ_SETERR(ret); return; }; + + if (bzf->strm.avail_out < BZ_MAX_UNUSED) { + n = BZ_MAX_UNUSED - bzf->strm.avail_out; + n2 = fwrite ( (void*)(bzf->buf), sizeof(UChar), + n, bzf->handle ); + if (n != n2 || ferror(bzf->handle)) + { BZ_SETERR(BZ_IO_ERROR); return; }; + } + + if (bzf->strm.avail_in == 0) + { BZ_SETERR(BZ_OK); return; }; + } +} + + +/*---------------------------------------------------*/ +void BZ_API(BZ2_bzWriteClose) + ( int* bzerror, + BZFILE* b, + int abandon, + unsigned int* nbytes_in, + unsigned int* nbytes_out ) +{ + BZ2_bzWriteClose64 ( bzerror, b, abandon, + nbytes_in, NULL, nbytes_out, NULL ); +} + + +void BZ_API(BZ2_bzWriteClose64) + ( int* bzerror, + BZFILE* b, + int abandon, + unsigned int* nbytes_in_lo32, + unsigned int* nbytes_in_hi32, + unsigned int* nbytes_out_lo32, + unsigned int* nbytes_out_hi32 ) +{ + Int32 n, n2, ret; + bzFile* bzf = (bzFile*)b; + + if (bzf == NULL) + { BZ_SETERR(BZ_OK); return; }; + if (!(bzf->writing)) + { BZ_SETERR(BZ_SEQUENCE_ERROR); return; }; + if (ferror(bzf->handle)) + { BZ_SETERR(BZ_IO_ERROR); return; }; + + if (nbytes_in_lo32 != NULL) *nbytes_in_lo32 = 0; + if (nbytes_in_hi32 != NULL) *nbytes_in_hi32 = 0; + if (nbytes_out_lo32 != NULL) *nbytes_out_lo32 = 0; + if (nbytes_out_hi32 != NULL) *nbytes_out_hi32 = 0; + + if ((!abandon) && bzf->lastErr == BZ_OK) { + while (True) { + bzf->strm.avail_out = BZ_MAX_UNUSED; + bzf->strm.next_out = bzf->buf; + ret = BZ2_bzCompress ( &(bzf->strm), BZ_FINISH ); + if (ret != BZ_FINISH_OK && ret != BZ_STREAM_END) + { BZ_SETERR(ret); return; }; + + if (bzf->strm.avail_out < BZ_MAX_UNUSED) { + n = BZ_MAX_UNUSED - bzf->strm.avail_out; + n2 = fwrite ( (void*)(bzf->buf), sizeof(UChar), + n, bzf->handle ); + if (n != n2 || ferror(bzf->handle)) + { BZ_SETERR(BZ_IO_ERROR); return; }; + } + + if (ret == BZ_STREAM_END) break; + } + } + + if ( !abandon && !ferror ( bzf->handle ) ) { + fflush ( bzf->handle ); + if (ferror(bzf->handle)) + { BZ_SETERR(BZ_IO_ERROR); return; }; + } + + if (nbytes_in_lo32 != NULL) + *nbytes_in_lo32 = bzf->strm.total_in_lo32; + if (nbytes_in_hi32 != NULL) + *nbytes_in_hi32 = bzf->strm.total_in_hi32; + if (nbytes_out_lo32 != NULL) + *nbytes_out_lo32 = bzf->strm.total_out_lo32; + if (nbytes_out_hi32 != NULL) + *nbytes_out_hi32 = bzf->strm.total_out_hi32; + + BZ_SETERR(BZ_OK); + BZ2_bzCompressEnd ( &(bzf->strm) ); + free ( bzf ); +} + + +/*---------------------------------------------------*/ +BZFILE* BZ_API(BZ2_bzReadOpen) + ( int* bzerror, + FILE* f, + int verbosity, + int small, + void* unused, + int nUnused ) +{ + bzFile* bzf = NULL; + int ret; + + BZ_SETERR(BZ_OK); + + if (f == NULL || + (small != 0 && small != 1) || + (verbosity < 0 || verbosity > 4) || + (unused == NULL && nUnused != 0) || + (unused != NULL && (nUnused < 0 || nUnused > BZ_MAX_UNUSED))) + { BZ_SETERR(BZ_PARAM_ERROR); return NULL; }; + + if (ferror(f)) + { BZ_SETERR(BZ_IO_ERROR); return NULL; }; + + bzf = malloc ( sizeof(bzFile) ); + if (bzf == NULL) + { BZ_SETERR(BZ_MEM_ERROR); return NULL; }; + + BZ_SETERR(BZ_OK); + + bzf->initialisedOk = False; + bzf->handle = f; + bzf->bufN = 0; + bzf->writing = False; + bzf->strm.bzalloc = NULL; + bzf->strm.bzfree = NULL; + bzf->strm.opaque = NULL; + + while (nUnused > 0) { + bzf->buf[bzf->bufN] = *((UChar*)(unused)); bzf->bufN++; + unused = ((void*)( 1 + ((UChar*)(unused)) )); + nUnused--; + } + + ret = BZ2_bzDecompressInit ( &(bzf->strm), verbosity, small ); + if (ret != BZ_OK) + { BZ_SETERR(ret); free(bzf); return NULL; }; + + bzf->strm.avail_in = bzf->bufN; + bzf->strm.next_in = bzf->buf; + + bzf->initialisedOk = True; + return bzf; +} + + +/*---------------------------------------------------*/ +void BZ_API(BZ2_bzReadClose) ( int *bzerror, BZFILE *b ) +{ + bzFile* bzf = (bzFile*)b; + + BZ_SETERR(BZ_OK); + if (bzf == NULL) + { BZ_SETERR(BZ_OK); return; }; + + if (bzf->writing) + { BZ_SETERR(BZ_SEQUENCE_ERROR); return; }; + + if (bzf->initialisedOk) + (void)BZ2_bzDecompressEnd ( &(bzf->strm) ); + free ( bzf ); +} + + +/*---------------------------------------------------*/ +int BZ_API(BZ2_bzRead) + ( int* bzerror, + BZFILE* b, + void* buf, + int len ) +{ + Int32 n, ret; + bzFile* bzf = (bzFile*)b; + + BZ_SETERR(BZ_OK); + + if (bzf == NULL || buf == NULL || len < 0) + { BZ_SETERR(BZ_PARAM_ERROR); return 0; }; + + if (bzf->writing) + { BZ_SETERR(BZ_SEQUENCE_ERROR); return 0; }; + + if (len == 0) + { BZ_SETERR(BZ_OK); return 0; }; + + bzf->strm.avail_out = len; + bzf->strm.next_out = buf; + + while (True) { + + if (ferror(bzf->handle)) + { BZ_SETERR(BZ_IO_ERROR); return 0; }; + + if (bzf->strm.avail_in == 0 && !myfeof(bzf->handle)) { + n = fread ( bzf->buf, sizeof(UChar), + BZ_MAX_UNUSED, bzf->handle ); + if (ferror(bzf->handle)) + { BZ_SETERR(BZ_IO_ERROR); return 0; }; + bzf->bufN = n; + bzf->strm.avail_in = bzf->bufN; + bzf->strm.next_in = bzf->buf; + } + + ret = BZ2_bzDecompress ( &(bzf->strm) ); + + if (ret != BZ_OK && ret != BZ_STREAM_END) + { BZ_SETERR(ret); return 0; }; + + if (ret == BZ_OK && myfeof(bzf->handle) && + bzf->strm.avail_in == 0 && bzf->strm.avail_out > 0) + { BZ_SETERR(BZ_UNEXPECTED_EOF); return 0; }; + + if (ret == BZ_STREAM_END) + { BZ_SETERR(BZ_STREAM_END); + return len - bzf->strm.avail_out; }; + if (bzf->strm.avail_out == 0) + { BZ_SETERR(BZ_OK); return len; }; + + } + + return 0; /*not reached*/ +} + + +/*---------------------------------------------------*/ +void BZ_API(BZ2_bzReadGetUnused) + ( int* bzerror, + BZFILE* b, + void** unused, + int* nUnused ) +{ + bzFile* bzf = (bzFile*)b; + if (bzf == NULL) + { BZ_SETERR(BZ_PARAM_ERROR); return; }; + if (bzf->lastErr != BZ_STREAM_END) + { BZ_SETERR(BZ_SEQUENCE_ERROR); return; }; + if (unused == NULL || nUnused == NULL) + { BZ_SETERR(BZ_PARAM_ERROR); return; }; + + BZ_SETERR(BZ_OK); + *nUnused = bzf->strm.avail_in; + *unused = bzf->strm.next_in; +} +#endif + + +/*---------------------------------------------------*/ +/*--- Misc convenience stuff ---*/ +/*---------------------------------------------------*/ + +/*---------------------------------------------------*/ +int BZ_API(BZ2_bzBuffToBuffCompress) + ( char* dest, + unsigned int* destLen, + char* source, + unsigned int sourceLen, + int blockSize100k, + int verbosity, + int workFactor ) +{ + bz_stream strm; + int ret; + + if (dest == NULL || destLen == NULL || + source == NULL || + blockSize100k < 1 || blockSize100k > 9 || + verbosity < 0 || verbosity > 4 || + workFactor < 0 || workFactor > 250) + return BZ_PARAM_ERROR; + + if (workFactor == 0) workFactor = 30; + strm.bzalloc = NULL; + strm.bzfree = NULL; + strm.opaque = NULL; + ret = BZ2_bzCompressInit ( &strm, blockSize100k, + verbosity, workFactor ); + if (ret != BZ_OK) return ret; + + strm.next_in = source; + strm.next_out = dest; + strm.avail_in = sourceLen; + strm.avail_out = *destLen; + + ret = BZ2_bzCompress ( &strm, BZ_FINISH ); + if (ret == BZ_FINISH_OK) goto output_overflow; + if (ret != BZ_STREAM_END) goto errhandler; + + /* normal termination */ + *destLen -= strm.avail_out; + BZ2_bzCompressEnd ( &strm ); + return BZ_OK; + + output_overflow: + BZ2_bzCompressEnd ( &strm ); + return BZ_OUTBUFF_FULL; + + errhandler: + BZ2_bzCompressEnd ( &strm ); + return ret; +} + + +/*---------------------------------------------------*/ +int BZ_API(BZ2_bzBuffToBuffDecompress) + ( char* dest, + unsigned int* destLen, + char* source, + unsigned int sourceLen, + int small, + int verbosity ) +{ + bz_stream strm; + int ret; + + if (dest == NULL || destLen == NULL || + source == NULL || + (small != 0 && small != 1) || + verbosity < 0 || verbosity > 4) + return BZ_PARAM_ERROR; + + strm.bzalloc = NULL; + strm.bzfree = NULL; + strm.opaque = NULL; + ret = BZ2_bzDecompressInit ( &strm, verbosity, small ); + if (ret != BZ_OK) return ret; + + strm.next_in = source; + strm.next_out = dest; + strm.avail_in = sourceLen; + strm.avail_out = *destLen; + + ret = BZ2_bzDecompress ( &strm ); + if (ret == BZ_OK) goto output_overflow_or_eof; + if (ret != BZ_STREAM_END) goto errhandler; + + /* normal termination */ + *destLen -= strm.avail_out; + BZ2_bzDecompressEnd ( &strm ); + return BZ_OK; + + output_overflow_or_eof: + if (strm.avail_out > 0) { + BZ2_bzDecompressEnd ( &strm ); + return BZ_UNEXPECTED_EOF; + } else { + BZ2_bzDecompressEnd ( &strm ); + return BZ_OUTBUFF_FULL; + }; + + errhandler: + BZ2_bzDecompressEnd ( &strm ); + return ret; +} + + +/*---------------------------------------------------*/ +/*-- + Code contributed by Yoshioka Tsuneo (tsuneo@rr.iij4u.or.jp) + to support better zlib compatibility. + This code is not _officially_ part of libbzip2 (yet); + I haven't tested it, documented it, or considered the + threading-safeness of it. + If this code breaks, please contact both Yoshioka and me. +--*/ +/*---------------------------------------------------*/ + +/*---------------------------------------------------*/ +/*-- + return version like "0.9.5d, 4-Sept-1999". +--*/ +const char * BZ_API(BZ2_bzlibVersion)(void) +{ + return BZ_VERSION; +} + + +#ifndef BZ_NO_STDIO +/*---------------------------------------------------*/ + +#if defined(_WIN32) || defined(OS2) || defined(MSDOS) +# include +# include +# define SET_BINARY_MODE(file) setmode(fileno(file),O_BINARY) +#else +# define SET_BINARY_MODE(file) +#endif +static +BZFILE * bzopen_or_bzdopen + ( const char *path, /* no use when bzdopen */ + int fd, /* no use when bzdopen */ + const char *mode, + int open_mode) /* bzopen: 0, bzdopen:1 */ +{ + int bzerr; + char unused[BZ_MAX_UNUSED]; + int blockSize100k = 9; + int writing = 0; + char mode2[10] = ""; + FILE *fp = NULL; + BZFILE *bzfp = NULL; + int verbosity = 0; + int workFactor = 30; + int smallMode = 0; + int nUnused = 0; + + if (mode == NULL) return NULL; + while (*mode) { + switch (*mode) { + case 'r': + writing = 0; break; + case 'w': + writing = 1; break; + case 's': + smallMode = 1; break; + default: + if (isdigit((int)(*mode))) { + blockSize100k = *mode-BZ_HDR_0; + } + } + mode++; + } + strcat(mode2, writing ? "w" : "r" ); + strcat(mode2,"b"); /* binary mode */ + + if (open_mode==0) { + if (path==NULL || strcmp(path,"")==0) { + fp = (writing ? stdout : stdin); + SET_BINARY_MODE(fp); + } else { + fp = fopen(path,mode2); + } + } else { +#ifdef BZ_STRICT_ANSI + fp = NULL; +#else + fp = fdopen(fd,mode2); +#endif + } + if (fp == NULL) return NULL; + + if (writing) { + /* Guard against total chaos and anarchy -- JRS */ + if (blockSize100k < 1) blockSize100k = 1; + if (blockSize100k > 9) blockSize100k = 9; + bzfp = BZ2_bzWriteOpen(&bzerr,fp,blockSize100k, + verbosity,workFactor); + } else { + bzfp = BZ2_bzReadOpen(&bzerr,fp,verbosity,smallMode, + unused,nUnused); + } + if (bzfp == NULL) { + if (fp != stdin && fp != stdout) fclose(fp); + return NULL; + } + return bzfp; +} + + +/*---------------------------------------------------*/ +/*-- + open file for read or write. + ex) bzopen("file","w9") + case path="" or NULL => use stdin or stdout. +--*/ +BZFILE * BZ_API(BZ2_bzopen) + ( const char *path, + const char *mode ) +{ + return bzopen_or_bzdopen(path,-1,mode,/*bzopen*/0); +} + + +/*---------------------------------------------------*/ +BZFILE * BZ_API(BZ2_bzdopen) + ( int fd, + const char *mode ) +{ + return bzopen_or_bzdopen(NULL,fd,mode,/*bzdopen*/1); +} + + +/*---------------------------------------------------*/ +int BZ_API(BZ2_bzread) (BZFILE* b, void* buf, int len ) +{ + int bzerr, nread; + if (((bzFile*)b)->lastErr == BZ_STREAM_END) return 0; + nread = BZ2_bzRead(&bzerr,b,buf,len); + if (bzerr == BZ_OK || bzerr == BZ_STREAM_END) { + return nread; + } else { + return -1; + } +} + + +/*---------------------------------------------------*/ +int BZ_API(BZ2_bzwrite) (BZFILE* b, void* buf, int len ) +{ + int bzerr; + + BZ2_bzWrite(&bzerr,b,buf,len); + if(bzerr == BZ_OK){ + return len; + }else{ + return -1; + } +} + + +/*---------------------------------------------------*/ +int BZ_API(BZ2_bzflush) (BZFILE *b) +{ + /* do nothing now... */ + return 0; +} + + +/*---------------------------------------------------*/ +void BZ_API(BZ2_bzclose) (BZFILE* b) +{ + int bzerr; + FILE *fp; + + if (b==NULL) {return;} + fp = ((bzFile *)b)->handle; + if(((bzFile*)b)->writing){ + BZ2_bzWriteClose(&bzerr,b,0,NULL,NULL); + if(bzerr != BZ_OK){ + BZ2_bzWriteClose(NULL,b,1,NULL,NULL); + } + }else{ + BZ2_bzReadClose(&bzerr,b); + } + if(fp!=stdin && fp!=stdout){ + fclose(fp); + } +} + + +/*---------------------------------------------------*/ +/*-- + return last error code +--*/ +static const char *bzerrorstrings[] = { + "OK" + ,"SEQUENCE_ERROR" + ,"PARAM_ERROR" + ,"MEM_ERROR" + ,"DATA_ERROR" + ,"DATA_ERROR_MAGIC" + ,"IO_ERROR" + ,"UNEXPECTED_EOF" + ,"OUTBUFF_FULL" + ,"CONFIG_ERROR" + ,"???" /* for future */ + ,"???" /* for future */ + ,"???" /* for future */ + ,"???" /* for future */ + ,"???" /* for future */ + ,"???" /* for future */ +}; + + +const char * BZ_API(BZ2_bzerror) (BZFILE *b, int *errnum) +{ + int err = ((bzFile *)b)->lastErr; + + if(err>0) err = 0; + *errnum = err; + return bzerrorstrings[err*-1]; +} +#endif + + +/*-------------------------------------------------------------*/ +/*--- end bzlib.c ---*/ +/*-------------------------------------------------------------*/ diff --git a/lib/libbz2/bzlib.h b/lib/libbz2/bzlib.h new file mode 100644 index 00000000..8966a6c5 --- /dev/null +++ b/lib/libbz2/bzlib.h @@ -0,0 +1,282 @@ + +/*-------------------------------------------------------------*/ +/*--- Public header file for the library. ---*/ +/*--- bzlib.h ---*/ +/*-------------------------------------------------------------*/ + +/* ------------------------------------------------------------------ + This file is part of bzip2/libbzip2, a program and library for + lossless, block-sorting data compression. + + bzip2/libbzip2 version 1.0.8 of 13 July 2019 + Copyright (C) 1996-2019 Julian Seward + + Please read the WARNING, DISCLAIMER and PATENTS sections in the + README file. + + This program is released under the terms of the license contained + in the file LICENSE. + ------------------------------------------------------------------ */ + + +#ifndef _BZLIB_H +#define _BZLIB_H + +#ifdef __cplusplus +extern "C" { +#endif + +#define BZ_RUN 0 +#define BZ_FLUSH 1 +#define BZ_FINISH 2 + +#define BZ_OK 0 +#define BZ_RUN_OK 1 +#define BZ_FLUSH_OK 2 +#define BZ_FINISH_OK 3 +#define BZ_STREAM_END 4 +#define BZ_SEQUENCE_ERROR (-1) +#define BZ_PARAM_ERROR (-2) +#define BZ_MEM_ERROR (-3) +#define BZ_DATA_ERROR (-4) +#define BZ_DATA_ERROR_MAGIC (-5) +#define BZ_IO_ERROR (-6) +#define BZ_UNEXPECTED_EOF (-7) +#define BZ_OUTBUFF_FULL (-8) +#define BZ_CONFIG_ERROR (-9) + +typedef + struct { + char *next_in; + unsigned int avail_in; + unsigned int total_in_lo32; + unsigned int total_in_hi32; + + char *next_out; + unsigned int avail_out; + unsigned int total_out_lo32; + unsigned int total_out_hi32; + + void *state; + + void *(*bzalloc)(void *,int,int); + void (*bzfree)(void *,void *); + void *opaque; + } + bz_stream; + + +#ifndef BZ_IMPORT +#define BZ_EXPORT +#endif + +#ifndef BZ_NO_STDIO +/* Need a definitition for FILE */ +#include +#endif + +#ifdef _WIN32 +# include +# ifdef small + /* windows.h define small to char */ +# undef small +# endif +# ifdef BZ_EXPORT +# define BZ_API(func) WINAPI func +# define BZ_EXTERN extern +# else + /* import windows dll dynamically */ +# define BZ_API(func) (WINAPI * func) +# define BZ_EXTERN +# endif +#else +# define BZ_API(func) func +# define BZ_EXTERN extern +#endif + + +/*-- Core (low-level) library functions --*/ + +BZ_EXTERN int BZ_API(BZ2_bzCompressInit) ( + bz_stream* strm, + int blockSize100k, + int verbosity, + int workFactor + ); + +BZ_EXTERN int BZ_API(BZ2_bzCompress) ( + bz_stream* strm, + int action + ); + +BZ_EXTERN int BZ_API(BZ2_bzCompressEnd) ( + bz_stream* strm + ); + +BZ_EXTERN int BZ_API(BZ2_bzDecompressInit) ( + bz_stream *strm, + int verbosity, + int small + ); + +BZ_EXTERN int BZ_API(BZ2_bzDecompress) ( + bz_stream* strm + ); + +BZ_EXTERN int BZ_API(BZ2_bzDecompressEnd) ( + bz_stream *strm + ); + + + +/*-- High(er) level library functions --*/ + +#ifndef BZ_NO_STDIO +#define BZ_MAX_UNUSED 5000 + +typedef void BZFILE; + +BZ_EXTERN BZFILE* BZ_API(BZ2_bzReadOpen) ( + int* bzerror, + FILE* f, + int verbosity, + int small, + void* unused, + int nUnused + ); + +BZ_EXTERN void BZ_API(BZ2_bzReadClose) ( + int* bzerror, + BZFILE* b + ); + +BZ_EXTERN void BZ_API(BZ2_bzReadGetUnused) ( + int* bzerror, + BZFILE* b, + void** unused, + int* nUnused + ); + +BZ_EXTERN int BZ_API(BZ2_bzRead) ( + int* bzerror, + BZFILE* b, + void* buf, + int len + ); + +BZ_EXTERN BZFILE* BZ_API(BZ2_bzWriteOpen) ( + int* bzerror, + FILE* f, + int blockSize100k, + int verbosity, + int workFactor + ); + +BZ_EXTERN void BZ_API(BZ2_bzWrite) ( + int* bzerror, + BZFILE* b, + void* buf, + int len + ); + +BZ_EXTERN void BZ_API(BZ2_bzWriteClose) ( + int* bzerror, + BZFILE* b, + int abandon, + unsigned int* nbytes_in, + unsigned int* nbytes_out + ); + +BZ_EXTERN void BZ_API(BZ2_bzWriteClose64) ( + int* bzerror, + BZFILE* b, + int abandon, + unsigned int* nbytes_in_lo32, + unsigned int* nbytes_in_hi32, + unsigned int* nbytes_out_lo32, + unsigned int* nbytes_out_hi32 + ); +#endif + + +/*-- Utility functions --*/ + +BZ_EXTERN int BZ_API(BZ2_bzBuffToBuffCompress) ( + char* dest, + unsigned int* destLen, + char* source, + unsigned int sourceLen, + int blockSize100k, + int verbosity, + int workFactor + ); + +BZ_EXTERN int BZ_API(BZ2_bzBuffToBuffDecompress) ( + char* dest, + unsigned int* destLen, + char* source, + unsigned int sourceLen, + int small, + int verbosity + ); + + +/*-- + Code contributed by Yoshioka Tsuneo (tsuneo@rr.iij4u.or.jp) + to support better zlib compatibility. + This code is not _officially_ part of libbzip2 (yet); + I haven't tested it, documented it, or considered the + threading-safeness of it. + If this code breaks, please contact both Yoshioka and me. +--*/ + +BZ_EXTERN const char * BZ_API(BZ2_bzlibVersion) ( + void + ); + +#ifndef BZ_NO_STDIO +BZ_EXTERN BZFILE * BZ_API(BZ2_bzopen) ( + const char *path, + const char *mode + ); + +BZ_EXTERN BZFILE * BZ_API(BZ2_bzdopen) ( + int fd, + const char *mode + ); + +BZ_EXTERN int BZ_API(BZ2_bzread) ( + BZFILE* b, + void* buf, + int len + ); + +BZ_EXTERN int BZ_API(BZ2_bzwrite) ( + BZFILE* b, + void* buf, + int len + ); + +BZ_EXTERN int BZ_API(BZ2_bzflush) ( + BZFILE* b + ); + +BZ_EXTERN void BZ_API(BZ2_bzclose) ( + BZFILE* b + ); + +BZ_EXTERN const char * BZ_API(BZ2_bzerror) ( + BZFILE *b, + int *errnum + ); +#endif + +#ifdef __cplusplus +} +#endif + +#endif + +/*-------------------------------------------------------------*/ +/*--- end bzlib.h ---*/ +/*-------------------------------------------------------------*/ diff --git a/lib/libbz2/bzlib_private.h b/lib/libbz2/bzlib_private.h new file mode 100644 index 00000000..3755a6f7 --- /dev/null +++ b/lib/libbz2/bzlib_private.h @@ -0,0 +1,509 @@ + +/*-------------------------------------------------------------*/ +/*--- Private header file for the library. ---*/ +/*--- bzlib_private.h ---*/ +/*-------------------------------------------------------------*/ + +/* ------------------------------------------------------------------ + This file is part of bzip2/libbzip2, a program and library for + lossless, block-sorting data compression. + + bzip2/libbzip2 version 1.0.8 of 13 July 2019 + Copyright (C) 1996-2019 Julian Seward + + Please read the WARNING, DISCLAIMER and PATENTS sections in the + README file. + + This program is released under the terms of the license contained + in the file LICENSE. + ------------------------------------------------------------------ */ + + +#ifndef _BZLIB_PRIVATE_H +#define _BZLIB_PRIVATE_H + +#include + +#ifndef BZ_NO_STDIO +#include +#include +#include +#endif + +#include "bzlib.h" + + + +/*-- General stuff. --*/ + +#define BZ_VERSION "1.0.8, 13-Jul-2019" + +typedef char Char; +typedef unsigned char Bool; +typedef unsigned char UChar; +typedef int Int32; +typedef unsigned int UInt32; +typedef short Int16; +typedef unsigned short UInt16; + +#define True ((Bool)1) +#define False ((Bool)0) + +#ifndef __GNUC__ +#define __inline__ /* */ +#endif + +#ifndef BZ_NO_STDIO + +extern void BZ2_bz__AssertH__fail ( int errcode ); +#define AssertH(cond,errcode) \ + { if (!(cond)) BZ2_bz__AssertH__fail ( errcode ); } + +#if BZ_DEBUG +#define AssertD(cond,msg) \ + { if (!(cond)) { \ + fprintf ( stderr, \ + "\n\nlibbzip2(debug build): internal error\n\t%s\n", msg );\ + exit(1); \ + }} +#else +#define AssertD(cond,msg) /* */ +#endif + +#define VPrintf0(zf) \ + fprintf(stderr,zf) +#define VPrintf1(zf,za1) \ + fprintf(stderr,zf,za1) +#define VPrintf2(zf,za1,za2) \ + fprintf(stderr,zf,za1,za2) +#define VPrintf3(zf,za1,za2,za3) \ + fprintf(stderr,zf,za1,za2,za3) +#define VPrintf4(zf,za1,za2,za3,za4) \ + fprintf(stderr,zf,za1,za2,za3,za4) +#define VPrintf5(zf,za1,za2,za3,za4,za5) \ + fprintf(stderr,zf,za1,za2,za3,za4,za5) + +#else + +extern void bz_internal_error ( int errcode ); +#define AssertH(cond,errcode) \ + { if (!(cond)) bz_internal_error ( errcode ); } +#define AssertD(cond,msg) do { } while (0) +#define VPrintf0(zf) do { } while (0) +#define VPrintf1(zf,za1) do { } while (0) +#define VPrintf2(zf,za1,za2) do { } while (0) +#define VPrintf3(zf,za1,za2,za3) do { } while (0) +#define VPrintf4(zf,za1,za2,za3,za4) do { } while (0) +#define VPrintf5(zf,za1,za2,za3,za4,za5) do { } while (0) + +#endif + + +#define BZALLOC(nnn) (strm->bzalloc)(strm->opaque,(nnn),1) +#define BZFREE(ppp) (strm->bzfree)(strm->opaque,(ppp)) + + +/*-- Header bytes. --*/ + +#define BZ_HDR_B 0x42 /* 'B' */ +#define BZ_HDR_Z 0x5a /* 'Z' */ +#define BZ_HDR_h 0x68 /* 'h' */ +#define BZ_HDR_0 0x30 /* '0' */ + +/*-- Constants for the back end. --*/ + +#define BZ_MAX_ALPHA_SIZE 258 +#define BZ_MAX_CODE_LEN 23 + +#define BZ_RUNA 0 +#define BZ_RUNB 1 + +#define BZ_N_GROUPS 6 +#define BZ_G_SIZE 50 +#define BZ_N_ITERS 4 + +#define BZ_MAX_SELECTORS (2 + (900000 / BZ_G_SIZE)) + + + +/*-- Stuff for randomising repetitive blocks. --*/ + +extern Int32 BZ2_rNums[512]; + +#define BZ_RAND_DECLS \ + Int32 rNToGo; \ + Int32 rTPos \ + +#define BZ_RAND_INIT_MASK \ + s->rNToGo = 0; \ + s->rTPos = 0 \ + +#define BZ_RAND_MASK ((s->rNToGo == 1) ? 1 : 0) + +#define BZ_RAND_UPD_MASK \ + if (s->rNToGo == 0) { \ + s->rNToGo = BZ2_rNums[s->rTPos]; \ + s->rTPos++; \ + if (s->rTPos == 512) s->rTPos = 0; \ + } \ + s->rNToGo--; + + + +/*-- Stuff for doing CRCs. --*/ + +extern UInt32 BZ2_crc32Table[256]; + +#define BZ_INITIALISE_CRC(crcVar) \ +{ \ + crcVar = 0xffffffffL; \ +} + +#define BZ_FINALISE_CRC(crcVar) \ +{ \ + crcVar = ~(crcVar); \ +} + +#define BZ_UPDATE_CRC(crcVar,cha) \ +{ \ + crcVar = (crcVar << 8) ^ \ + BZ2_crc32Table[(crcVar >> 24) ^ \ + ((UChar)cha)]; \ +} + + + +/*-- States and modes for compression. --*/ + +#define BZ_M_IDLE 1 +#define BZ_M_RUNNING 2 +#define BZ_M_FLUSHING 3 +#define BZ_M_FINISHING 4 + +#define BZ_S_OUTPUT 1 +#define BZ_S_INPUT 2 + +#define BZ_N_RADIX 2 +#define BZ_N_QSORT 12 +#define BZ_N_SHELL 18 +#define BZ_N_OVERSHOOT (BZ_N_RADIX + BZ_N_QSORT + BZ_N_SHELL + 2) + + + + +/*-- Structure holding all the compression-side stuff. --*/ + +typedef + struct { + /* pointer back to the struct bz_stream */ + bz_stream* strm; + + /* mode this stream is in, and whether inputting */ + /* or outputting data */ + Int32 mode; + Int32 state; + + /* remembers avail_in when flush/finish requested */ + UInt32 avail_in_expect; + + /* for doing the block sorting */ + UInt32* arr1; + UInt32* arr2; + UInt32* ftab; + Int32 origPtr; + + /* aliases for arr1 and arr2 */ + UInt32* ptr; + UChar* block; + UInt16* mtfv; + UChar* zbits; + + /* for deciding when to use the fallback sorting algorithm */ + Int32 workFactor; + + /* run-length-encoding of the input */ + UInt32 state_in_ch; + Int32 state_in_len; + BZ_RAND_DECLS; + + /* input and output limits and current posns */ + Int32 nblock; + Int32 nblockMAX; + Int32 numZ; + Int32 state_out_pos; + + /* map of bytes used in block */ + Int32 nInUse; + Bool inUse[256]; + UChar unseqToSeq[256]; + + /* the buffer for bit stream creation */ + UInt32 bsBuff; + Int32 bsLive; + + /* block and combined CRCs */ + UInt32 blockCRC; + UInt32 combinedCRC; + + /* misc administratium */ + Int32 verbosity; + Int32 blockNo; + Int32 blockSize100k; + + /* stuff for coding the MTF values */ + Int32 nMTF; + Int32 mtfFreq [BZ_MAX_ALPHA_SIZE]; + UChar selector [BZ_MAX_SELECTORS]; + UChar selectorMtf[BZ_MAX_SELECTORS]; + + UChar len [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE]; + Int32 code [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE]; + Int32 rfreq [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE]; + /* second dimension: only 3 needed; 4 makes index calculations faster */ + UInt32 len_pack[BZ_MAX_ALPHA_SIZE][4]; + + } + EState; + + + +/*-- externs for compression. --*/ + +extern void +BZ2_blockSort ( EState* ); + +extern void +BZ2_compressBlock ( EState*, Bool ); + +extern void +BZ2_bsInitWrite ( EState* ); + +extern void +BZ2_hbAssignCodes ( Int32*, UChar*, Int32, Int32, Int32 ); + +extern void +BZ2_hbMakeCodeLengths ( UChar*, Int32*, Int32, Int32 ); + + + +/*-- states for decompression. --*/ + +#define BZ_X_IDLE 1 +#define BZ_X_OUTPUT 2 + +#define BZ_X_MAGIC_1 10 +#define BZ_X_MAGIC_2 11 +#define BZ_X_MAGIC_3 12 +#define BZ_X_MAGIC_4 13 +#define BZ_X_BLKHDR_1 14 +#define BZ_X_BLKHDR_2 15 +#define BZ_X_BLKHDR_3 16 +#define BZ_X_BLKHDR_4 17 +#define BZ_X_BLKHDR_5 18 +#define BZ_X_BLKHDR_6 19 +#define BZ_X_BCRC_1 20 +#define BZ_X_BCRC_2 21 +#define BZ_X_BCRC_3 22 +#define BZ_X_BCRC_4 23 +#define BZ_X_RANDBIT 24 +#define BZ_X_ORIGPTR_1 25 +#define BZ_X_ORIGPTR_2 26 +#define BZ_X_ORIGPTR_3 27 +#define BZ_X_MAPPING_1 28 +#define BZ_X_MAPPING_2 29 +#define BZ_X_SELECTOR_1 30 +#define BZ_X_SELECTOR_2 31 +#define BZ_X_SELECTOR_3 32 +#define BZ_X_CODING_1 33 +#define BZ_X_CODING_2 34 +#define BZ_X_CODING_3 35 +#define BZ_X_MTF_1 36 +#define BZ_X_MTF_2 37 +#define BZ_X_MTF_3 38 +#define BZ_X_MTF_4 39 +#define BZ_X_MTF_5 40 +#define BZ_X_MTF_6 41 +#define BZ_X_ENDHDR_2 42 +#define BZ_X_ENDHDR_3 43 +#define BZ_X_ENDHDR_4 44 +#define BZ_X_ENDHDR_5 45 +#define BZ_X_ENDHDR_6 46 +#define BZ_X_CCRC_1 47 +#define BZ_X_CCRC_2 48 +#define BZ_X_CCRC_3 49 +#define BZ_X_CCRC_4 50 + + + +/*-- Constants for the fast MTF decoder. --*/ + +#define MTFA_SIZE 4096 +#define MTFL_SIZE 16 + + + +/*-- Structure holding all the decompression-side stuff. --*/ + +typedef + struct { + /* pointer back to the struct bz_stream */ + bz_stream* strm; + + /* state indicator for this stream */ + Int32 state; + + /* for doing the final run-length decoding */ + UChar state_out_ch; + Int32 state_out_len; + Bool blockRandomised; + BZ_RAND_DECLS; + + /* the buffer for bit stream reading */ + UInt32 bsBuff; + Int32 bsLive; + + /* misc administratium */ + Int32 blockSize100k; + Bool smallDecompress; + Int32 currBlockNo; + Int32 verbosity; + + /* for undoing the Burrows-Wheeler transform */ + Int32 origPtr; + UInt32 tPos; + Int32 k0; + Int32 unzftab[256]; + Int32 nblock_used; + Int32 cftab[257]; + Int32 cftabCopy[257]; + + /* for undoing the Burrows-Wheeler transform (FAST) */ + UInt32 *tt; + + /* for undoing the Burrows-Wheeler transform (SMALL) */ + UInt16 *ll16; + UChar *ll4; + + /* stored and calculated CRCs */ + UInt32 storedBlockCRC; + UInt32 storedCombinedCRC; + UInt32 calculatedBlockCRC; + UInt32 calculatedCombinedCRC; + + /* map of bytes used in block */ + Int32 nInUse; + Bool inUse[256]; + Bool inUse16[16]; + UChar seqToUnseq[256]; + + /* for decoding the MTF values */ + UChar mtfa [MTFA_SIZE]; + Int32 mtfbase[256 / MTFL_SIZE]; + UChar selector [BZ_MAX_SELECTORS]; + UChar selectorMtf[BZ_MAX_SELECTORS]; + UChar len [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE]; + + Int32 limit [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE]; + Int32 base [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE]; + Int32 perm [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE]; + Int32 minLens[BZ_N_GROUPS]; + + /* save area for scalars in the main decompress code */ + Int32 save_i; + Int32 save_j; + Int32 save_t; + Int32 save_alphaSize; + Int32 save_nGroups; + Int32 save_nSelectors; + Int32 save_EOB; + Int32 save_groupNo; + Int32 save_groupPos; + Int32 save_nextSym; + Int32 save_nblockMAX; + Int32 save_nblock; + Int32 save_es; + Int32 save_N; + Int32 save_curr; + Int32 save_zt; + Int32 save_zn; + Int32 save_zvec; + Int32 save_zj; + Int32 save_gSel; + Int32 save_gMinlen; + Int32* save_gLimit; + Int32* save_gBase; + Int32* save_gPerm; + + } + DState; + + + +/*-- Macros for decompression. --*/ + +#define BZ_GET_FAST(cccc) \ + /* c_tPos is unsigned, hence test < 0 is pointless. */ \ + if (s->tPos >= (UInt32)100000 * (UInt32)s->blockSize100k) return True; \ + s->tPos = s->tt[s->tPos]; \ + cccc = (UChar)(s->tPos & 0xff); \ + s->tPos >>= 8; + +#define BZ_GET_FAST_C(cccc) \ + /* c_tPos is unsigned, hence test < 0 is pointless. */ \ + if (c_tPos >= (UInt32)100000 * (UInt32)ro_blockSize100k) return True; \ + c_tPos = c_tt[c_tPos]; \ + cccc = (UChar)(c_tPos & 0xff); \ + c_tPos >>= 8; + +#define SET_LL4(i,n) \ + { if (((i) & 0x1) == 0) \ + s->ll4[(i) >> 1] = (s->ll4[(i) >> 1] & 0xf0) | (n); else \ + s->ll4[(i) >> 1] = (s->ll4[(i) >> 1] & 0x0f) | ((n) << 4); \ + } + +#define GET_LL4(i) \ + ((((UInt32)(s->ll4[(i) >> 1])) >> (((i) << 2) & 0x4)) & 0xF) + +#define SET_LL(i,n) \ + { s->ll16[i] = (UInt16)(n & 0x0000ffff); \ + SET_LL4(i, n >> 16); \ + } + +#define GET_LL(i) \ + (((UInt32)s->ll16[i]) | (GET_LL4(i) << 16)) + +#define BZ_GET_SMALL(cccc) \ + /* c_tPos is unsigned, hence test < 0 is pointless. */ \ + if (s->tPos >= (UInt32)100000 * (UInt32)s->blockSize100k) return True; \ + cccc = BZ2_indexIntoF ( s->tPos, s->cftab ); \ + s->tPos = GET_LL(s->tPos); + + +/*-- externs for decompression. --*/ + +extern Int32 +BZ2_indexIntoF ( Int32, Int32* ); + +extern Int32 +BZ2_decompress ( DState* ); + +extern void +BZ2_hbCreateDecodeTables ( Int32*, Int32*, Int32*, UChar*, + Int32, Int32, Int32 ); + + +#endif + + +/*-- BZ_NO_STDIO seems to make NULL disappear on some platforms. --*/ + +#ifdef BZ_NO_STDIO +#ifndef NULL +#define NULL 0 +#endif +#endif + + +/*-------------------------------------------------------------*/ +/*--- end bzlib_private.h ---*/ +/*-------------------------------------------------------------*/ diff --git a/lib/libbz2/compress.c b/lib/libbz2/compress.c new file mode 100644 index 00000000..5dfa0023 --- /dev/null +++ b/lib/libbz2/compress.c @@ -0,0 +1,672 @@ + +/*-------------------------------------------------------------*/ +/*--- Compression machinery (not incl block sorting) ---*/ +/*--- compress.c ---*/ +/*-------------------------------------------------------------*/ + +/* ------------------------------------------------------------------ + This file is part of bzip2/libbzip2, a program and library for + lossless, block-sorting data compression. + + bzip2/libbzip2 version 1.0.8 of 13 July 2019 + Copyright (C) 1996-2019 Julian Seward + + Please read the WARNING, DISCLAIMER and PATENTS sections in the + README file. + + This program is released under the terms of the license contained + in the file LICENSE. + ------------------------------------------------------------------ */ + + +/* CHANGES + 0.9.0 -- original version. + 0.9.0a/b -- no changes in this file. + 0.9.0c -- changed setting of nGroups in sendMTFValues() + so as to do a bit better on small files +*/ + +#include "bzlib_private.h" + + +/*---------------------------------------------------*/ +/*--- Bit stream I/O ---*/ +/*---------------------------------------------------*/ + +/*---------------------------------------------------*/ +void BZ2_bsInitWrite ( EState* s ) +{ + s->bsLive = 0; + s->bsBuff = 0; +} + + +/*---------------------------------------------------*/ +static +void bsFinishWrite ( EState* s ) +{ + while (s->bsLive > 0) { + s->zbits[s->numZ] = (UChar)(s->bsBuff >> 24); + s->numZ++; + s->bsBuff <<= 8; + s->bsLive -= 8; + } +} + + +/*---------------------------------------------------*/ +#define bsNEEDW(nz) \ +{ \ + while (s->bsLive >= 8) { \ + s->zbits[s->numZ] \ + = (UChar)(s->bsBuff >> 24); \ + s->numZ++; \ + s->bsBuff <<= 8; \ + s->bsLive -= 8; \ + } \ +} + + +/*---------------------------------------------------*/ +static +__inline__ +void bsW ( EState* s, Int32 n, UInt32 v ) +{ + bsNEEDW ( n ); + s->bsBuff |= (v << (32 - s->bsLive - n)); + s->bsLive += n; +} + + +/*---------------------------------------------------*/ +static +void bsPutUInt32 ( EState* s, UInt32 u ) +{ + bsW ( s, 8, (u >> 24) & 0xffL ); + bsW ( s, 8, (u >> 16) & 0xffL ); + bsW ( s, 8, (u >> 8) & 0xffL ); + bsW ( s, 8, u & 0xffL ); +} + + +/*---------------------------------------------------*/ +static +void bsPutUChar ( EState* s, UChar c ) +{ + bsW( s, 8, (UInt32)c ); +} + + +/*---------------------------------------------------*/ +/*--- The back end proper ---*/ +/*---------------------------------------------------*/ + +/*---------------------------------------------------*/ +static +void makeMaps_e ( EState* s ) +{ + Int32 i; + s->nInUse = 0; + for (i = 0; i < 256; i++) + if (s->inUse[i]) { + s->unseqToSeq[i] = s->nInUse; + s->nInUse++; + } +} + + +/*---------------------------------------------------*/ +static +void generateMTFValues ( EState* s ) +{ + UChar yy[256]; + Int32 i, j; + Int32 zPend; + Int32 wr; + Int32 EOB; + + /* + After sorting (eg, here), + s->arr1 [ 0 .. s->nblock-1 ] holds sorted order, + and + ((UChar*)s->arr2) [ 0 .. s->nblock-1 ] + holds the original block data. + + The first thing to do is generate the MTF values, + and put them in + ((UInt16*)s->arr1) [ 0 .. s->nblock-1 ]. + Because there are strictly fewer or equal MTF values + than block values, ptr values in this area are overwritten + with MTF values only when they are no longer needed. + + The final compressed bitstream is generated into the + area starting at + (UChar*) (&((UChar*)s->arr2)[s->nblock]) + + These storage aliases are set up in bzCompressInit(), + except for the last one, which is arranged in + compressBlock(). + */ + UInt32* ptr = s->ptr; + UChar* block = s->block; + UInt16* mtfv = s->mtfv; + + makeMaps_e ( s ); + EOB = s->nInUse+1; + + for (i = 0; i <= EOB; i++) s->mtfFreq[i] = 0; + + wr = 0; + zPend = 0; + for (i = 0; i < s->nInUse; i++) yy[i] = (UChar) i; + + for (i = 0; i < s->nblock; i++) { + UChar ll_i; + AssertD ( wr <= i, "generateMTFValues(1)" ); + j = ptr[i]-1; if (j < 0) j += s->nblock; + ll_i = s->unseqToSeq[block[j]]; + AssertD ( ll_i < s->nInUse, "generateMTFValues(2a)" ); + + if (yy[0] == ll_i) { + zPend++; + } else { + + if (zPend > 0) { + zPend--; + while (True) { + if (zPend & 1) { + mtfv[wr] = BZ_RUNB; wr++; + s->mtfFreq[BZ_RUNB]++; + } else { + mtfv[wr] = BZ_RUNA; wr++; + s->mtfFreq[BZ_RUNA]++; + } + if (zPend < 2) break; + zPend = (zPend - 2) / 2; + }; + zPend = 0; + } + { + register UChar rtmp; + register UChar* ryy_j; + register UChar rll_i; + rtmp = yy[1]; + yy[1] = yy[0]; + ryy_j = &(yy[1]); + rll_i = ll_i; + while ( rll_i != rtmp ) { + register UChar rtmp2; + ryy_j++; + rtmp2 = rtmp; + rtmp = *ryy_j; + *ryy_j = rtmp2; + }; + yy[0] = rtmp; + j = ryy_j - &(yy[0]); + mtfv[wr] = j+1; wr++; s->mtfFreq[j+1]++; + } + + } + } + + if (zPend > 0) { + zPend--; + while (True) { + if (zPend & 1) { + mtfv[wr] = BZ_RUNB; wr++; + s->mtfFreq[BZ_RUNB]++; + } else { + mtfv[wr] = BZ_RUNA; wr++; + s->mtfFreq[BZ_RUNA]++; + } + if (zPend < 2) break; + zPend = (zPend - 2) / 2; + }; + zPend = 0; + } + + mtfv[wr] = EOB; wr++; s->mtfFreq[EOB]++; + + s->nMTF = wr; +} + + +/*---------------------------------------------------*/ +#define BZ_LESSER_ICOST 0 +#define BZ_GREATER_ICOST 15 + +static +void sendMTFValues ( EState* s ) +{ + Int32 v, t, i, j, gs, ge, totc, bt, bc, iter; + Int32 nSelectors, alphaSize, minLen, maxLen, selCtr; + Int32 nGroups, nBytes; + + /*-- + UChar len [BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE]; + is a global since the decoder also needs it. + + Int32 code[BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE]; + Int32 rfreq[BZ_N_GROUPS][BZ_MAX_ALPHA_SIZE]; + are also globals only used in this proc. + Made global to keep stack frame size small. + --*/ + + + UInt16 cost[BZ_N_GROUPS]; + Int32 fave[BZ_N_GROUPS]; + + UInt16* mtfv = s->mtfv; + + if (s->verbosity >= 3) + VPrintf3( " %d in block, %d after MTF & 1-2 coding, " + "%d+2 syms in use\n", + s->nblock, s->nMTF, s->nInUse ); + + alphaSize = s->nInUse+2; + for (t = 0; t < BZ_N_GROUPS; t++) + for (v = 0; v < alphaSize; v++) + s->len[t][v] = BZ_GREATER_ICOST; + + /*--- Decide how many coding tables to use ---*/ + AssertH ( s->nMTF > 0, 3001 ); + if (s->nMTF < 200) nGroups = 2; else + if (s->nMTF < 600) nGroups = 3; else + if (s->nMTF < 1200) nGroups = 4; else + if (s->nMTF < 2400) nGroups = 5; else + nGroups = 6; + + /*--- Generate an initial set of coding tables ---*/ + { + Int32 nPart, remF, tFreq, aFreq; + + nPart = nGroups; + remF = s->nMTF; + gs = 0; + while (nPart > 0) { + tFreq = remF / nPart; + ge = gs-1; + aFreq = 0; + while (aFreq < tFreq && ge < alphaSize-1) { + ge++; + aFreq += s->mtfFreq[ge]; + } + + if (ge > gs + && nPart != nGroups && nPart != 1 + && ((nGroups-nPart) % 2 == 1)) { + aFreq -= s->mtfFreq[ge]; + ge--; + } + + if (s->verbosity >= 3) + VPrintf5( " initial group %d, [%d .. %d], " + "has %d syms (%4.1f%%)\n", + nPart, gs, ge, aFreq, + (100.0 * (float)aFreq) / (float)(s->nMTF) ); + + for (v = 0; v < alphaSize; v++) + if (v >= gs && v <= ge) + s->len[nPart-1][v] = BZ_LESSER_ICOST; else + s->len[nPart-1][v] = BZ_GREATER_ICOST; + + nPart--; + gs = ge+1; + remF -= aFreq; + } + } + + /*--- + Iterate up to BZ_N_ITERS times to improve the tables. + ---*/ + for (iter = 0; iter < BZ_N_ITERS; iter++) { + + for (t = 0; t < nGroups; t++) fave[t] = 0; + + for (t = 0; t < nGroups; t++) + for (v = 0; v < alphaSize; v++) + s->rfreq[t][v] = 0; + + /*--- + Set up an auxiliary length table which is used to fast-track + the common case (nGroups == 6). + ---*/ + if (nGroups == 6) { + for (v = 0; v < alphaSize; v++) { + s->len_pack[v][0] = (s->len[1][v] << 16) | s->len[0][v]; + s->len_pack[v][1] = (s->len[3][v] << 16) | s->len[2][v]; + s->len_pack[v][2] = (s->len[5][v] << 16) | s->len[4][v]; + } + } + + nSelectors = 0; + totc = 0; + gs = 0; + while (True) { + + /*--- Set group start & end marks. --*/ + if (gs >= s->nMTF) break; + ge = gs + BZ_G_SIZE - 1; + if (ge >= s->nMTF) ge = s->nMTF-1; + + /*-- + Calculate the cost of this group as coded + by each of the coding tables. + --*/ + for (t = 0; t < nGroups; t++) cost[t] = 0; + + if (nGroups == 6 && 50 == ge-gs+1) { + /*--- fast track the common case ---*/ + register UInt32 cost01, cost23, cost45; + register UInt16 icv; + cost01 = cost23 = cost45 = 0; + +# define BZ_ITER(nn) \ + icv = mtfv[gs+(nn)]; \ + cost01 += s->len_pack[icv][0]; \ + cost23 += s->len_pack[icv][1]; \ + cost45 += s->len_pack[icv][2]; \ + + BZ_ITER(0); BZ_ITER(1); BZ_ITER(2); BZ_ITER(3); BZ_ITER(4); + BZ_ITER(5); BZ_ITER(6); BZ_ITER(7); BZ_ITER(8); BZ_ITER(9); + BZ_ITER(10); BZ_ITER(11); BZ_ITER(12); BZ_ITER(13); BZ_ITER(14); + BZ_ITER(15); BZ_ITER(16); BZ_ITER(17); BZ_ITER(18); BZ_ITER(19); + BZ_ITER(20); BZ_ITER(21); BZ_ITER(22); BZ_ITER(23); BZ_ITER(24); + BZ_ITER(25); BZ_ITER(26); BZ_ITER(27); BZ_ITER(28); BZ_ITER(29); + BZ_ITER(30); BZ_ITER(31); BZ_ITER(32); BZ_ITER(33); BZ_ITER(34); + BZ_ITER(35); BZ_ITER(36); BZ_ITER(37); BZ_ITER(38); BZ_ITER(39); + BZ_ITER(40); BZ_ITER(41); BZ_ITER(42); BZ_ITER(43); BZ_ITER(44); + BZ_ITER(45); BZ_ITER(46); BZ_ITER(47); BZ_ITER(48); BZ_ITER(49); + +# undef BZ_ITER + + cost[0] = cost01 & 0xffff; cost[1] = cost01 >> 16; + cost[2] = cost23 & 0xffff; cost[3] = cost23 >> 16; + cost[4] = cost45 & 0xffff; cost[5] = cost45 >> 16; + + } else { + /*--- slow version which correctly handles all situations ---*/ + for (i = gs; i <= ge; i++) { + UInt16 icv = mtfv[i]; + for (t = 0; t < nGroups; t++) cost[t] += s->len[t][icv]; + } + } + + /*-- + Find the coding table which is best for this group, + and record its identity in the selector table. + --*/ + bc = 999999999; bt = -1; + for (t = 0; t < nGroups; t++) + if (cost[t] < bc) { bc = cost[t]; bt = t; }; + totc += bc; + fave[bt]++; + s->selector[nSelectors] = bt; + nSelectors++; + + /*-- + Increment the symbol frequencies for the selected table. + --*/ + if (nGroups == 6 && 50 == ge-gs+1) { + /*--- fast track the common case ---*/ + +# define BZ_ITUR(nn) s->rfreq[bt][ mtfv[gs+(nn)] ]++ + + BZ_ITUR(0); BZ_ITUR(1); BZ_ITUR(2); BZ_ITUR(3); BZ_ITUR(4); + BZ_ITUR(5); BZ_ITUR(6); BZ_ITUR(7); BZ_ITUR(8); BZ_ITUR(9); + BZ_ITUR(10); BZ_ITUR(11); BZ_ITUR(12); BZ_ITUR(13); BZ_ITUR(14); + BZ_ITUR(15); BZ_ITUR(16); BZ_ITUR(17); BZ_ITUR(18); BZ_ITUR(19); + BZ_ITUR(20); BZ_ITUR(21); BZ_ITUR(22); BZ_ITUR(23); BZ_ITUR(24); + BZ_ITUR(25); BZ_ITUR(26); BZ_ITUR(27); BZ_ITUR(28); BZ_ITUR(29); + BZ_ITUR(30); BZ_ITUR(31); BZ_ITUR(32); BZ_ITUR(33); BZ_ITUR(34); + BZ_ITUR(35); BZ_ITUR(36); BZ_ITUR(37); BZ_ITUR(38); BZ_ITUR(39); + BZ_ITUR(40); BZ_ITUR(41); BZ_ITUR(42); BZ_ITUR(43); BZ_ITUR(44); + BZ_ITUR(45); BZ_ITUR(46); BZ_ITUR(47); BZ_ITUR(48); BZ_ITUR(49); + +# undef BZ_ITUR + + } else { + /*--- slow version which correctly handles all situations ---*/ + for (i = gs; i <= ge; i++) + s->rfreq[bt][ mtfv[i] ]++; + } + + gs = ge+1; + } + if (s->verbosity >= 3) { + VPrintf2 ( " pass %d: size is %d, grp uses are ", + iter+1, totc/8 ); + for (t = 0; t < nGroups; t++) + VPrintf1 ( "%d ", fave[t] ); + VPrintf0 ( "\n" ); + } + + /*-- + Recompute the tables based on the accumulated frequencies. + --*/ + /* maxLen was changed from 20 to 17 in bzip2-1.0.3. See + comment in huffman.c for details. */ + for (t = 0; t < nGroups; t++) + BZ2_hbMakeCodeLengths ( &(s->len[t][0]), &(s->rfreq[t][0]), + alphaSize, 17 /*20*/ ); + } + + + AssertH( nGroups < 8, 3002 ); + AssertH( nSelectors < 32768 && + nSelectors <= BZ_MAX_SELECTORS, + 3003 ); + + + /*--- Compute MTF values for the selectors. ---*/ + { + UChar pos[BZ_N_GROUPS], ll_i, tmp2, tmp; + for (i = 0; i < nGroups; i++) pos[i] = i; + for (i = 0; i < nSelectors; i++) { + ll_i = s->selector[i]; + j = 0; + tmp = pos[j]; + while ( ll_i != tmp ) { + j++; + tmp2 = tmp; + tmp = pos[j]; + pos[j] = tmp2; + }; + pos[0] = tmp; + s->selectorMtf[i] = j; + } + }; + + /*--- Assign actual codes for the tables. --*/ + for (t = 0; t < nGroups; t++) { + minLen = 32; + maxLen = 0; + for (i = 0; i < alphaSize; i++) { + if (s->len[t][i] > maxLen) maxLen = s->len[t][i]; + if (s->len[t][i] < minLen) minLen = s->len[t][i]; + } + AssertH ( !(maxLen > 17 /*20*/ ), 3004 ); + AssertH ( !(minLen < 1), 3005 ); + BZ2_hbAssignCodes ( &(s->code[t][0]), &(s->len[t][0]), + minLen, maxLen, alphaSize ); + } + + /*--- Transmit the mapping table. ---*/ + { + Bool inUse16[16]; + for (i = 0; i < 16; i++) { + inUse16[i] = False; + for (j = 0; j < 16; j++) + if (s->inUse[i * 16 + j]) inUse16[i] = True; + } + + nBytes = s->numZ; + for (i = 0; i < 16; i++) + if (inUse16[i]) bsW(s,1,1); else bsW(s,1,0); + + for (i = 0; i < 16; i++) + if (inUse16[i]) + for (j = 0; j < 16; j++) { + if (s->inUse[i * 16 + j]) bsW(s,1,1); else bsW(s,1,0); + } + + if (s->verbosity >= 3) + VPrintf1( " bytes: mapping %d, ", s->numZ-nBytes ); + } + + /*--- Now the selectors. ---*/ + nBytes = s->numZ; + bsW ( s, 3, nGroups ); + bsW ( s, 15, nSelectors ); + for (i = 0; i < nSelectors; i++) { + for (j = 0; j < s->selectorMtf[i]; j++) bsW(s,1,1); + bsW(s,1,0); + } + if (s->verbosity >= 3) + VPrintf1( "selectors %d, ", s->numZ-nBytes ); + + /*--- Now the coding tables. ---*/ + nBytes = s->numZ; + + for (t = 0; t < nGroups; t++) { + Int32 curr = s->len[t][0]; + bsW ( s, 5, curr ); + for (i = 0; i < alphaSize; i++) { + while (curr < s->len[t][i]) { bsW(s,2,2); curr++; /* 10 */ }; + while (curr > s->len[t][i]) { bsW(s,2,3); curr--; /* 11 */ }; + bsW ( s, 1, 0 ); + } + } + + if (s->verbosity >= 3) + VPrintf1 ( "code lengths %d, ", s->numZ-nBytes ); + + /*--- And finally, the block data proper ---*/ + nBytes = s->numZ; + selCtr = 0; + gs = 0; + while (True) { + if (gs >= s->nMTF) break; + ge = gs + BZ_G_SIZE - 1; + if (ge >= s->nMTF) ge = s->nMTF-1; + AssertH ( s->selector[selCtr] < nGroups, 3006 ); + + if (nGroups == 6 && 50 == ge-gs+1) { + /*--- fast track the common case ---*/ + UInt16 mtfv_i; + UChar* s_len_sel_selCtr + = &(s->len[s->selector[selCtr]][0]); + Int32* s_code_sel_selCtr + = &(s->code[s->selector[selCtr]][0]); + +# define BZ_ITAH(nn) \ + mtfv_i = mtfv[gs+(nn)]; \ + bsW ( s, \ + s_len_sel_selCtr[mtfv_i], \ + s_code_sel_selCtr[mtfv_i] ) + + BZ_ITAH(0); BZ_ITAH(1); BZ_ITAH(2); BZ_ITAH(3); BZ_ITAH(4); + BZ_ITAH(5); BZ_ITAH(6); BZ_ITAH(7); BZ_ITAH(8); BZ_ITAH(9); + BZ_ITAH(10); BZ_ITAH(11); BZ_ITAH(12); BZ_ITAH(13); BZ_ITAH(14); + BZ_ITAH(15); BZ_ITAH(16); BZ_ITAH(17); BZ_ITAH(18); BZ_ITAH(19); + BZ_ITAH(20); BZ_ITAH(21); BZ_ITAH(22); BZ_ITAH(23); BZ_ITAH(24); + BZ_ITAH(25); BZ_ITAH(26); BZ_ITAH(27); BZ_ITAH(28); BZ_ITAH(29); + BZ_ITAH(30); BZ_ITAH(31); BZ_ITAH(32); BZ_ITAH(33); BZ_ITAH(34); + BZ_ITAH(35); BZ_ITAH(36); BZ_ITAH(37); BZ_ITAH(38); BZ_ITAH(39); + BZ_ITAH(40); BZ_ITAH(41); BZ_ITAH(42); BZ_ITAH(43); BZ_ITAH(44); + BZ_ITAH(45); BZ_ITAH(46); BZ_ITAH(47); BZ_ITAH(48); BZ_ITAH(49); + +# undef BZ_ITAH + + } else { + /*--- slow version which correctly handles all situations ---*/ + for (i = gs; i <= ge; i++) { + bsW ( s, + s->len [s->selector[selCtr]] [mtfv[i]], + s->code [s->selector[selCtr]] [mtfv[i]] ); + } + } + + + gs = ge+1; + selCtr++; + } + AssertH( selCtr == nSelectors, 3007 ); + + if (s->verbosity >= 3) + VPrintf1( "codes %d\n", s->numZ-nBytes ); +} + + +/*---------------------------------------------------*/ +void BZ2_compressBlock ( EState* s, Bool is_last_block ) +{ + if (s->nblock > 0) { + + BZ_FINALISE_CRC ( s->blockCRC ); + s->combinedCRC = (s->combinedCRC << 1) | (s->combinedCRC >> 31); + s->combinedCRC ^= s->blockCRC; + if (s->blockNo > 1) s->numZ = 0; + + if (s->verbosity >= 2) + VPrintf4( " block %d: crc = 0x%08x, " + "combined CRC = 0x%08x, size = %d\n", + s->blockNo, s->blockCRC, s->combinedCRC, s->nblock ); + + BZ2_blockSort ( s ); + } + + s->zbits = (UChar*) (&((UChar*)s->arr2)[s->nblock]); + + /*-- If this is the first block, create the stream header. --*/ + if (s->blockNo == 1) { + BZ2_bsInitWrite ( s ); + bsPutUChar ( s, BZ_HDR_B ); + bsPutUChar ( s, BZ_HDR_Z ); + bsPutUChar ( s, BZ_HDR_h ); + bsPutUChar ( s, (UChar)(BZ_HDR_0 + s->blockSize100k) ); + } + + if (s->nblock > 0) { + + bsPutUChar ( s, 0x31 ); bsPutUChar ( s, 0x41 ); + bsPutUChar ( s, 0x59 ); bsPutUChar ( s, 0x26 ); + bsPutUChar ( s, 0x53 ); bsPutUChar ( s, 0x59 ); + + /*-- Now the block's CRC, so it is in a known place. --*/ + bsPutUInt32 ( s, s->blockCRC ); + + /*-- + Now a single bit indicating (non-)randomisation. + As of version 0.9.5, we use a better sorting algorithm + which makes randomisation unnecessary. So always set + the randomised bit to 'no'. Of course, the decoder + still needs to be able to handle randomised blocks + so as to maintain backwards compatibility with + older versions of bzip2. + --*/ + bsW(s,1,0); + + bsW ( s, 24, s->origPtr ); + generateMTFValues ( s ); + sendMTFValues ( s ); + } + + + /*-- If this is the last block, add the stream trailer. --*/ + if (is_last_block) { + + bsPutUChar ( s, 0x17 ); bsPutUChar ( s, 0x72 ); + bsPutUChar ( s, 0x45 ); bsPutUChar ( s, 0x38 ); + bsPutUChar ( s, 0x50 ); bsPutUChar ( s, 0x90 ); + bsPutUInt32 ( s, s->combinedCRC ); + if (s->verbosity >= 2) + VPrintf1( " final combined CRC = 0x%08x\n ", s->combinedCRC ); + bsFinishWrite ( s ); + } +} + + +/*-------------------------------------------------------------*/ +/*--- end compress.c ---*/ +/*-------------------------------------------------------------*/ diff --git a/lib/libbz2/crctable.c b/lib/libbz2/crctable.c new file mode 100644 index 00000000..2b33c253 --- /dev/null +++ b/lib/libbz2/crctable.c @@ -0,0 +1,104 @@ + +/*-------------------------------------------------------------*/ +/*--- Table for doing CRCs ---*/ +/*--- crctable.c ---*/ +/*-------------------------------------------------------------*/ + +/* ------------------------------------------------------------------ + This file is part of bzip2/libbzip2, a program and library for + lossless, block-sorting data compression. + + bzip2/libbzip2 version 1.0.8 of 13 July 2019 + Copyright (C) 1996-2019 Julian Seward + + Please read the WARNING, DISCLAIMER and PATENTS sections in the + README file. + + This program is released under the terms of the license contained + in the file LICENSE. + ------------------------------------------------------------------ */ + + +#include "bzlib_private.h" + +/*-- + I think this is an implementation of the AUTODIN-II, + Ethernet & FDDI 32-bit CRC standard. Vaguely derived + from code by Rob Warnock, in Section 51 of the + comp.compression FAQ. +--*/ + +UInt32 BZ2_crc32Table[256] = { + + /*-- Ugly, innit? --*/ + + 0x00000000L, 0x04c11db7L, 0x09823b6eL, 0x0d4326d9L, + 0x130476dcL, 0x17c56b6bL, 0x1a864db2L, 0x1e475005L, + 0x2608edb8L, 0x22c9f00fL, 0x2f8ad6d6L, 0x2b4bcb61L, + 0x350c9b64L, 0x31cd86d3L, 0x3c8ea00aL, 0x384fbdbdL, + 0x4c11db70L, 0x48d0c6c7L, 0x4593e01eL, 0x4152fda9L, + 0x5f15adacL, 0x5bd4b01bL, 0x569796c2L, 0x52568b75L, + 0x6a1936c8L, 0x6ed82b7fL, 0x639b0da6L, 0x675a1011L, + 0x791d4014L, 0x7ddc5da3L, 0x709f7b7aL, 0x745e66cdL, + 0x9823b6e0L, 0x9ce2ab57L, 0x91a18d8eL, 0x95609039L, + 0x8b27c03cL, 0x8fe6dd8bL, 0x82a5fb52L, 0x8664e6e5L, + 0xbe2b5b58L, 0xbaea46efL, 0xb7a96036L, 0xb3687d81L, + 0xad2f2d84L, 0xa9ee3033L, 0xa4ad16eaL, 0xa06c0b5dL, + 0xd4326d90L, 0xd0f37027L, 0xddb056feL, 0xd9714b49L, + 0xc7361b4cL, 0xc3f706fbL, 0xceb42022L, 0xca753d95L, + 0xf23a8028L, 0xf6fb9d9fL, 0xfbb8bb46L, 0xff79a6f1L, + 0xe13ef6f4L, 0xe5ffeb43L, 0xe8bccd9aL, 0xec7dd02dL, + 0x34867077L, 0x30476dc0L, 0x3d044b19L, 0x39c556aeL, + 0x278206abL, 0x23431b1cL, 0x2e003dc5L, 0x2ac12072L, + 0x128e9dcfL, 0x164f8078L, 0x1b0ca6a1L, 0x1fcdbb16L, + 0x018aeb13L, 0x054bf6a4L, 0x0808d07dL, 0x0cc9cdcaL, + 0x7897ab07L, 0x7c56b6b0L, 0x71159069L, 0x75d48ddeL, + 0x6b93dddbL, 0x6f52c06cL, 0x6211e6b5L, 0x66d0fb02L, + 0x5e9f46bfL, 0x5a5e5b08L, 0x571d7dd1L, 0x53dc6066L, + 0x4d9b3063L, 0x495a2dd4L, 0x44190b0dL, 0x40d816baL, + 0xaca5c697L, 0xa864db20L, 0xa527fdf9L, 0xa1e6e04eL, + 0xbfa1b04bL, 0xbb60adfcL, 0xb6238b25L, 0xb2e29692L, + 0x8aad2b2fL, 0x8e6c3698L, 0x832f1041L, 0x87ee0df6L, + 0x99a95df3L, 0x9d684044L, 0x902b669dL, 0x94ea7b2aL, + 0xe0b41de7L, 0xe4750050L, 0xe9362689L, 0xedf73b3eL, + 0xf3b06b3bL, 0xf771768cL, 0xfa325055L, 0xfef34de2L, + 0xc6bcf05fL, 0xc27dede8L, 0xcf3ecb31L, 0xcbffd686L, + 0xd5b88683L, 0xd1799b34L, 0xdc3abdedL, 0xd8fba05aL, + 0x690ce0eeL, 0x6dcdfd59L, 0x608edb80L, 0x644fc637L, + 0x7a089632L, 0x7ec98b85L, 0x738aad5cL, 0x774bb0ebL, + 0x4f040d56L, 0x4bc510e1L, 0x46863638L, 0x42472b8fL, + 0x5c007b8aL, 0x58c1663dL, 0x558240e4L, 0x51435d53L, + 0x251d3b9eL, 0x21dc2629L, 0x2c9f00f0L, 0x285e1d47L, + 0x36194d42L, 0x32d850f5L, 0x3f9b762cL, 0x3b5a6b9bL, + 0x0315d626L, 0x07d4cb91L, 0x0a97ed48L, 0x0e56f0ffL, + 0x1011a0faL, 0x14d0bd4dL, 0x19939b94L, 0x1d528623L, + 0xf12f560eL, 0xf5ee4bb9L, 0xf8ad6d60L, 0xfc6c70d7L, + 0xe22b20d2L, 0xe6ea3d65L, 0xeba91bbcL, 0xef68060bL, + 0xd727bbb6L, 0xd3e6a601L, 0xdea580d8L, 0xda649d6fL, + 0xc423cd6aL, 0xc0e2d0ddL, 0xcda1f604L, 0xc960ebb3L, + 0xbd3e8d7eL, 0xb9ff90c9L, 0xb4bcb610L, 0xb07daba7L, + 0xae3afba2L, 0xaafbe615L, 0xa7b8c0ccL, 0xa379dd7bL, + 0x9b3660c6L, 0x9ff77d71L, 0x92b45ba8L, 0x9675461fL, + 0x8832161aL, 0x8cf30badL, 0x81b02d74L, 0x857130c3L, + 0x5d8a9099L, 0x594b8d2eL, 0x5408abf7L, 0x50c9b640L, + 0x4e8ee645L, 0x4a4ffbf2L, 0x470cdd2bL, 0x43cdc09cL, + 0x7b827d21L, 0x7f436096L, 0x7200464fL, 0x76c15bf8L, + 0x68860bfdL, 0x6c47164aL, 0x61043093L, 0x65c52d24L, + 0x119b4be9L, 0x155a565eL, 0x18197087L, 0x1cd86d30L, + 0x029f3d35L, 0x065e2082L, 0x0b1d065bL, 0x0fdc1becL, + 0x3793a651L, 0x3352bbe6L, 0x3e119d3fL, 0x3ad08088L, + 0x2497d08dL, 0x2056cd3aL, 0x2d15ebe3L, 0x29d4f654L, + 0xc5a92679L, 0xc1683bceL, 0xcc2b1d17L, 0xc8ea00a0L, + 0xd6ad50a5L, 0xd26c4d12L, 0xdf2f6bcbL, 0xdbee767cL, + 0xe3a1cbc1L, 0xe760d676L, 0xea23f0afL, 0xeee2ed18L, + 0xf0a5bd1dL, 0xf464a0aaL, 0xf9278673L, 0xfde69bc4L, + 0x89b8fd09L, 0x8d79e0beL, 0x803ac667L, 0x84fbdbd0L, + 0x9abc8bd5L, 0x9e7d9662L, 0x933eb0bbL, 0x97ffad0cL, + 0xafb010b1L, 0xab710d06L, 0xa6322bdfL, 0xa2f33668L, + 0xbcb4666dL, 0xb8757bdaL, 0xb5365d03L, 0xb1f740b4L +}; + + +/*-------------------------------------------------------------*/ +/*--- end crctable.c ---*/ +/*-------------------------------------------------------------*/ diff --git a/lib/libbz2/decompress.c b/lib/libbz2/decompress.c new file mode 100644 index 00000000..a1a0bac8 --- /dev/null +++ b/lib/libbz2/decompress.c @@ -0,0 +1,652 @@ + +/*-------------------------------------------------------------*/ +/*--- Decompression machinery ---*/ +/*--- decompress.c ---*/ +/*-------------------------------------------------------------*/ + +/* ------------------------------------------------------------------ + This file is part of bzip2/libbzip2, a program and library for + lossless, block-sorting data compression. + + bzip2/libbzip2 version 1.0.8 of 13 July 2019 + Copyright (C) 1996-2019 Julian Seward + + Please read the WARNING, DISCLAIMER and PATENTS sections in the + README file. + + This program is released under the terms of the license contained + in the file LICENSE. + ------------------------------------------------------------------ */ + + +#include "bzlib_private.h" + + +/*---------------------------------------------------*/ +static +void makeMaps_d ( DState* s ) +{ + Int32 i; + s->nInUse = 0; + for (i = 0; i < 256; i++) + if (s->inUse[i]) { + s->seqToUnseq[s->nInUse] = i; + s->nInUse++; + } +} + + +/*---------------------------------------------------*/ +#define RETURN(rrr) \ + { retVal = rrr; goto save_state_and_return; }; + +#define GET_BITS(lll,vvv,nnn) \ + case lll: s->state = lll; \ + while (True) { \ + if (s->bsLive >= nnn) { \ + UInt32 v; \ + v = (s->bsBuff >> \ + (s->bsLive-nnn)) & ((1 << nnn)-1); \ + s->bsLive -= nnn; \ + vvv = v; \ + break; \ + } \ + if (s->strm->avail_in == 0) RETURN(BZ_OK); \ + s->bsBuff \ + = (s->bsBuff << 8) | \ + ((UInt32) \ + (*((UChar*)(s->strm->next_in)))); \ + s->bsLive += 8; \ + s->strm->next_in++; \ + s->strm->avail_in--; \ + s->strm->total_in_lo32++; \ + if (s->strm->total_in_lo32 == 0) \ + s->strm->total_in_hi32++; \ + } + +#define GET_UCHAR(lll,uuu) \ + GET_BITS(lll,uuu,8) + +#define GET_BIT(lll,uuu) \ + GET_BITS(lll,uuu,1) + +/*---------------------------------------------------*/ +#define GET_MTF_VAL(label1,label2,lval) \ +{ \ + if (groupPos == 0) { \ + groupNo++; \ + if (groupNo >= nSelectors) \ + RETURN(BZ_DATA_ERROR); \ + groupPos = BZ_G_SIZE; \ + gSel = s->selector[groupNo]; \ + gMinlen = s->minLens[gSel]; \ + gLimit = &(s->limit[gSel][0]); \ + gPerm = &(s->perm[gSel][0]); \ + gBase = &(s->base[gSel][0]); \ + } \ + groupPos--; \ + zn = gMinlen; \ + GET_BITS(label1, zvec, zn); \ + while (1) { \ + if (zn > 20 /* the longest code */) \ + RETURN(BZ_DATA_ERROR); \ + if (zvec <= gLimit[zn]) break; \ + zn++; \ + GET_BIT(label2, zj); \ + zvec = (zvec << 1) | zj; \ + }; \ + if (zvec - gBase[zn] < 0 \ + || zvec - gBase[zn] >= BZ_MAX_ALPHA_SIZE) \ + RETURN(BZ_DATA_ERROR); \ + lval = gPerm[zvec - gBase[zn]]; \ +} + + +/*---------------------------------------------------*/ +Int32 BZ2_decompress ( DState* s ) +{ + UChar uc; + Int32 retVal; + Int32 minLen, maxLen; + bz_stream* strm = s->strm; + + /* stuff that needs to be saved/restored */ + Int32 i; + Int32 j; + Int32 t; + Int32 alphaSize; + Int32 nGroups; + Int32 nSelectors; + Int32 EOB; + Int32 groupNo; + Int32 groupPos; + Int32 nextSym; + Int32 nblockMAX; + Int32 nblock; + Int32 es; + Int32 N; + Int32 curr; + Int32 zt; + Int32 zn; + Int32 zvec; + Int32 zj; + Int32 gSel; + Int32 gMinlen; + Int32* gLimit; + Int32* gBase; + Int32* gPerm; + + if (s->state == BZ_X_MAGIC_1) { + /*initialise the save area*/ + s->save_i = 0; + s->save_j = 0; + s->save_t = 0; + s->save_alphaSize = 0; + s->save_nGroups = 0; + s->save_nSelectors = 0; + s->save_EOB = 0; + s->save_groupNo = 0; + s->save_groupPos = 0; + s->save_nextSym = 0; + s->save_nblockMAX = 0; + s->save_nblock = 0; + s->save_es = 0; + s->save_N = 0; + s->save_curr = 0; + s->save_zt = 0; + s->save_zn = 0; + s->save_zvec = 0; + s->save_zj = 0; + s->save_gSel = 0; + s->save_gMinlen = 0; + s->save_gLimit = NULL; + s->save_gBase = NULL; + s->save_gPerm = NULL; + } + + /*restore from the save area*/ + i = s->save_i; + j = s->save_j; + t = s->save_t; + alphaSize = s->save_alphaSize; + nGroups = s->save_nGroups; + nSelectors = s->save_nSelectors; + EOB = s->save_EOB; + groupNo = s->save_groupNo; + groupPos = s->save_groupPos; + nextSym = s->save_nextSym; + nblockMAX = s->save_nblockMAX; + nblock = s->save_nblock; + es = s->save_es; + N = s->save_N; + curr = s->save_curr; + zt = s->save_zt; + zn = s->save_zn; + zvec = s->save_zvec; + zj = s->save_zj; + gSel = s->save_gSel; + gMinlen = s->save_gMinlen; + gLimit = s->save_gLimit; + gBase = s->save_gBase; + gPerm = s->save_gPerm; + + retVal = BZ_OK; + + switch (s->state) { + + GET_UCHAR(BZ_X_MAGIC_1, uc); + if (uc != BZ_HDR_B) RETURN(BZ_DATA_ERROR_MAGIC); + + GET_UCHAR(BZ_X_MAGIC_2, uc); + if (uc != BZ_HDR_Z) RETURN(BZ_DATA_ERROR_MAGIC); + + GET_UCHAR(BZ_X_MAGIC_3, uc) + if (uc != BZ_HDR_h) RETURN(BZ_DATA_ERROR_MAGIC); + + GET_BITS(BZ_X_MAGIC_4, s->blockSize100k, 8) + if (s->blockSize100k < (BZ_HDR_0 + 1) || + s->blockSize100k > (BZ_HDR_0 + 9)) RETURN(BZ_DATA_ERROR_MAGIC); + s->blockSize100k -= BZ_HDR_0; + + if (s->smallDecompress) { + s->ll16 = BZALLOC( s->blockSize100k * 100000 * sizeof(UInt16) ); + s->ll4 = BZALLOC( + ((1 + s->blockSize100k * 100000) >> 1) * sizeof(UChar) + ); + if (s->ll16 == NULL || s->ll4 == NULL) RETURN(BZ_MEM_ERROR); + } else { + s->tt = BZALLOC( s->blockSize100k * 100000 * sizeof(Int32) ); + if (s->tt == NULL) RETURN(BZ_MEM_ERROR); + } + + GET_UCHAR(BZ_X_BLKHDR_1, uc); + + if (uc == 0x17) goto endhdr_2; + if (uc != 0x31) RETURN(BZ_DATA_ERROR); + GET_UCHAR(BZ_X_BLKHDR_2, uc); + if (uc != 0x41) RETURN(BZ_DATA_ERROR); + GET_UCHAR(BZ_X_BLKHDR_3, uc); + if (uc != 0x59) RETURN(BZ_DATA_ERROR); + GET_UCHAR(BZ_X_BLKHDR_4, uc); + if (uc != 0x26) RETURN(BZ_DATA_ERROR); + GET_UCHAR(BZ_X_BLKHDR_5, uc); + if (uc != 0x53) RETURN(BZ_DATA_ERROR); + GET_UCHAR(BZ_X_BLKHDR_6, uc); + if (uc != 0x59) RETURN(BZ_DATA_ERROR); + + s->currBlockNo++; + if (s->verbosity >= 2) + VPrintf1 ( "\n [%d: huff+mtf ", s->currBlockNo ); + + s->storedBlockCRC = 0; + GET_UCHAR(BZ_X_BCRC_1, uc); + s->storedBlockCRC = (s->storedBlockCRC << 8) | ((UInt32)uc); + GET_UCHAR(BZ_X_BCRC_2, uc); + s->storedBlockCRC = (s->storedBlockCRC << 8) | ((UInt32)uc); + GET_UCHAR(BZ_X_BCRC_3, uc); + s->storedBlockCRC = (s->storedBlockCRC << 8) | ((UInt32)uc); + GET_UCHAR(BZ_X_BCRC_4, uc); + s->storedBlockCRC = (s->storedBlockCRC << 8) | ((UInt32)uc); + + GET_BITS(BZ_X_RANDBIT, s->blockRandomised, 1); + + s->origPtr = 0; + GET_UCHAR(BZ_X_ORIGPTR_1, uc); + s->origPtr = (s->origPtr << 8) | ((Int32)uc); + GET_UCHAR(BZ_X_ORIGPTR_2, uc); + s->origPtr = (s->origPtr << 8) | ((Int32)uc); + GET_UCHAR(BZ_X_ORIGPTR_3, uc); + s->origPtr = (s->origPtr << 8) | ((Int32)uc); + + if (s->origPtr < 0) + RETURN(BZ_DATA_ERROR); + if (s->origPtr > 10 + 100000*s->blockSize100k) + RETURN(BZ_DATA_ERROR); + + /*--- Receive the mapping table ---*/ + for (i = 0; i < 16; i++) { + GET_BIT(BZ_X_MAPPING_1, uc); + if (uc == 1) + s->inUse16[i] = True; else + s->inUse16[i] = False; + } + + for (i = 0; i < 256; i++) s->inUse[i] = False; + + for (i = 0; i < 16; i++) + if (s->inUse16[i]) + for (j = 0; j < 16; j++) { + GET_BIT(BZ_X_MAPPING_2, uc); + if (uc == 1) s->inUse[i * 16 + j] = True; + } + makeMaps_d ( s ); + if (s->nInUse == 0) RETURN(BZ_DATA_ERROR); + alphaSize = s->nInUse+2; + + /*--- Now the selectors ---*/ + GET_BITS(BZ_X_SELECTOR_1, nGroups, 3); + if (nGroups < 2 || nGroups > BZ_N_GROUPS) RETURN(BZ_DATA_ERROR); + GET_BITS(BZ_X_SELECTOR_2, nSelectors, 15); + if (nSelectors < 1) RETURN(BZ_DATA_ERROR); + for (i = 0; i < nSelectors; i++) { + j = 0; + while (True) { + GET_BIT(BZ_X_SELECTOR_3, uc); + if (uc == 0) break; + j++; + if (j >= nGroups) RETURN(BZ_DATA_ERROR); + } + /* Having more than BZ_MAX_SELECTORS doesn't make much sense + since they will never be used, but some implementations might + "round up" the number of selectors, so just ignore those. */ + if (i < BZ_MAX_SELECTORS) + s->selectorMtf[i] = j; + } + if (nSelectors > BZ_MAX_SELECTORS) + nSelectors = BZ_MAX_SELECTORS; + + /*--- Undo the MTF values for the selectors. ---*/ + { + UChar pos[BZ_N_GROUPS], tmp, v; + for (v = 0; v < nGroups; v++) pos[v] = v; + + for (i = 0; i < nSelectors; i++) { + v = s->selectorMtf[i]; + tmp = pos[v]; + while (v > 0) { pos[v] = pos[v-1]; v--; } + pos[0] = tmp; + s->selector[i] = tmp; + } + } + + /*--- Now the coding tables ---*/ + for (t = 0; t < nGroups; t++) { + GET_BITS(BZ_X_CODING_1, curr, 5); + for (i = 0; i < alphaSize; i++) { + while (True) { + if (curr < 1 || curr > 20) RETURN(BZ_DATA_ERROR); + GET_BIT(BZ_X_CODING_2, uc); + if (uc == 0) break; + GET_BIT(BZ_X_CODING_3, uc); + if (uc == 0) curr++; else curr--; + } + s->len[t][i] = curr; + } + } + + /*--- Create the Huffman decoding tables ---*/ + for (t = 0; t < nGroups; t++) { + minLen = 32; + maxLen = 0; + for (i = 0; i < alphaSize; i++) { + if (s->len[t][i] > maxLen) maxLen = s->len[t][i]; + if (s->len[t][i] < minLen) minLen = s->len[t][i]; + } + BZ2_hbCreateDecodeTables ( + &(s->limit[t][0]), + &(s->base[t][0]), + &(s->perm[t][0]), + &(s->len[t][0]), + minLen, maxLen, alphaSize + ); + s->minLens[t] = minLen; + } + + /*--- Now the MTF values ---*/ + + EOB = s->nInUse+1; + nblockMAX = 100000 * s->blockSize100k; + groupNo = -1; + groupPos = 0; + + for (i = 0; i <= 255; i++) s->unzftab[i] = 0; + + /*-- MTF init --*/ + { + Int32 ii, jj, kk; + kk = MTFA_SIZE-1; + for (ii = 256 / MTFL_SIZE - 1; ii >= 0; ii--) { + for (jj = MTFL_SIZE-1; jj >= 0; jj--) { + s->mtfa[kk] = (UChar)(ii * MTFL_SIZE + jj); + kk--; + } + s->mtfbase[ii] = kk + 1; + } + } + /*-- end MTF init --*/ + + nblock = 0; + GET_MTF_VAL(BZ_X_MTF_1, BZ_X_MTF_2, nextSym); + + while (True) { + + if (nextSym == EOB) break; + + if (nextSym == BZ_RUNA || nextSym == BZ_RUNB) { + + es = -1; + N = 1; + do { + /* Check that N doesn't get too big, so that es doesn't + go negative. The maximum value that can be + RUNA/RUNB encoded is equal to the block size (post + the initial RLE), viz, 900k, so bounding N at 2 + million should guard against overflow without + rejecting any legitimate inputs. */ + if (N >= 2*1024*1024) RETURN(BZ_DATA_ERROR); + if (nextSym == BZ_RUNA) es = es + (0+1) * N; else + if (nextSym == BZ_RUNB) es = es + (1+1) * N; + N = N * 2; + GET_MTF_VAL(BZ_X_MTF_3, BZ_X_MTF_4, nextSym); + } + while (nextSym == BZ_RUNA || nextSym == BZ_RUNB); + + es++; + uc = s->seqToUnseq[ s->mtfa[s->mtfbase[0]] ]; + s->unzftab[uc] += es; + + if (s->smallDecompress) + while (es > 0) { + if (nblock >= nblockMAX) RETURN(BZ_DATA_ERROR); + s->ll16[nblock] = (UInt16)uc; + nblock++; + es--; + } + else + while (es > 0) { + if (nblock >= nblockMAX) RETURN(BZ_DATA_ERROR); + s->tt[nblock] = (UInt32)uc; + nblock++; + es--; + }; + + continue; + + } else { + + if (nblock >= nblockMAX) RETURN(BZ_DATA_ERROR); + + /*-- uc = MTF ( nextSym-1 ) --*/ + { + Int32 ii, jj, kk, pp, lno, off; + UInt32 nn; + nn = (UInt32)(nextSym - 1); + + if (nn < MTFL_SIZE) { + /* avoid general-case expense */ + pp = s->mtfbase[0]; + uc = s->mtfa[pp+nn]; + while (nn > 3) { + Int32 z = pp+nn; + s->mtfa[(z) ] = s->mtfa[(z)-1]; + s->mtfa[(z)-1] = s->mtfa[(z)-2]; + s->mtfa[(z)-2] = s->mtfa[(z)-3]; + s->mtfa[(z)-3] = s->mtfa[(z)-4]; + nn -= 4; + } + while (nn > 0) { + s->mtfa[(pp+nn)] = s->mtfa[(pp+nn)-1]; nn--; + }; + s->mtfa[pp] = uc; + } else { + /* general case */ + lno = nn / MTFL_SIZE; + off = nn % MTFL_SIZE; + pp = s->mtfbase[lno] + off; + uc = s->mtfa[pp]; + while (pp > s->mtfbase[lno]) { + s->mtfa[pp] = s->mtfa[pp-1]; pp--; + }; + s->mtfbase[lno]++; + while (lno > 0) { + s->mtfbase[lno]--; + s->mtfa[s->mtfbase[lno]] + = s->mtfa[s->mtfbase[lno-1] + MTFL_SIZE - 1]; + lno--; + } + s->mtfbase[0]--; + s->mtfa[s->mtfbase[0]] = uc; + if (s->mtfbase[0] == 0) { + kk = MTFA_SIZE-1; + for (ii = 256 / MTFL_SIZE-1; ii >= 0; ii--) { + for (jj = MTFL_SIZE-1; jj >= 0; jj--) { + s->mtfa[kk] = s->mtfa[s->mtfbase[ii] + jj]; + kk--; + } + s->mtfbase[ii] = kk + 1; + } + } + } + } + /*-- end uc = MTF ( nextSym-1 ) --*/ + + s->unzftab[s->seqToUnseq[uc]]++; + if (s->smallDecompress) + s->ll16[nblock] = (UInt16)(s->seqToUnseq[uc]); else + s->tt[nblock] = (UInt32)(s->seqToUnseq[uc]); + nblock++; + + GET_MTF_VAL(BZ_X_MTF_5, BZ_X_MTF_6, nextSym); + continue; + } + } + + /* Now we know what nblock is, we can do a better sanity + check on s->origPtr. + */ + if (s->origPtr < 0 || s->origPtr >= nblock) + RETURN(BZ_DATA_ERROR); + + /*-- Set up cftab to facilitate generation of T^(-1) --*/ + /* Check: unzftab entries in range. */ + for (i = 0; i <= 255; i++) { + if (s->unzftab[i] < 0 || s->unzftab[i] > nblock) + RETURN(BZ_DATA_ERROR); + } + /* Actually generate cftab. */ + s->cftab[0] = 0; + for (i = 1; i <= 256; i++) s->cftab[i] = s->unzftab[i-1]; + for (i = 1; i <= 256; i++) s->cftab[i] += s->cftab[i-1]; + /* Check: cftab entries in range. */ + for (i = 0; i <= 256; i++) { + if (s->cftab[i] < 0 || s->cftab[i] > nblock) { + /* s->cftab[i] can legitimately be == nblock */ + RETURN(BZ_DATA_ERROR); + } + } + /* Check: cftab entries non-descending. */ + for (i = 1; i <= 256; i++) { + if (s->cftab[i-1] > s->cftab[i]) { + RETURN(BZ_DATA_ERROR); + } + } + + s->state_out_len = 0; + s->state_out_ch = 0; + BZ_INITIALISE_CRC ( s->calculatedBlockCRC ); + s->state = BZ_X_OUTPUT; + if (s->verbosity >= 2) VPrintf0 ( "rt+rld" ); + + if (s->smallDecompress) { + + /*-- Make a copy of cftab, used in generation of T --*/ + for (i = 0; i <= 256; i++) s->cftabCopy[i] = s->cftab[i]; + + /*-- compute the T vector --*/ + for (i = 0; i < nblock; i++) { + uc = (UChar)(s->ll16[i]); + SET_LL(i, s->cftabCopy[uc]); + s->cftabCopy[uc]++; + } + + /*-- Compute T^(-1) by pointer reversal on T --*/ + i = s->origPtr; + j = GET_LL(i); + do { + Int32 tmp = GET_LL(j); + SET_LL(j, i); + i = j; + j = tmp; + } + while (i != s->origPtr); + + s->tPos = s->origPtr; + s->nblock_used = 0; + if (s->blockRandomised) { + BZ_RAND_INIT_MASK; + BZ_GET_SMALL(s->k0); s->nblock_used++; + BZ_RAND_UPD_MASK; s->k0 ^= BZ_RAND_MASK; + } else { + BZ_GET_SMALL(s->k0); s->nblock_used++; + } + + } else { + + /*-- compute the T^(-1) vector --*/ + for (i = 0; i < nblock; i++) { + uc = (UChar)(s->tt[i] & 0xff); + s->tt[s->cftab[uc]] |= (i << 8); + s->cftab[uc]++; + } + + s->tPos = s->tt[s->origPtr] >> 8; + s->nblock_used = 0; + if (s->blockRandomised) { + BZ_RAND_INIT_MASK; + BZ_GET_FAST(s->k0); s->nblock_used++; + BZ_RAND_UPD_MASK; s->k0 ^= BZ_RAND_MASK; + } else { + BZ_GET_FAST(s->k0); s->nblock_used++; + } + + } + + RETURN(BZ_OK); + + + + endhdr_2: + + GET_UCHAR(BZ_X_ENDHDR_2, uc); + if (uc != 0x72) RETURN(BZ_DATA_ERROR); + GET_UCHAR(BZ_X_ENDHDR_3, uc); + if (uc != 0x45) RETURN(BZ_DATA_ERROR); + GET_UCHAR(BZ_X_ENDHDR_4, uc); + if (uc != 0x38) RETURN(BZ_DATA_ERROR); + GET_UCHAR(BZ_X_ENDHDR_5, uc); + if (uc != 0x50) RETURN(BZ_DATA_ERROR); + GET_UCHAR(BZ_X_ENDHDR_6, uc); + if (uc != 0x90) RETURN(BZ_DATA_ERROR); + + s->storedCombinedCRC = 0; + GET_UCHAR(BZ_X_CCRC_1, uc); + s->storedCombinedCRC = (s->storedCombinedCRC << 8) | ((UInt32)uc); + GET_UCHAR(BZ_X_CCRC_2, uc); + s->storedCombinedCRC = (s->storedCombinedCRC << 8) | ((UInt32)uc); + GET_UCHAR(BZ_X_CCRC_3, uc); + s->storedCombinedCRC = (s->storedCombinedCRC << 8) | ((UInt32)uc); + GET_UCHAR(BZ_X_CCRC_4, uc); + s->storedCombinedCRC = (s->storedCombinedCRC << 8) | ((UInt32)uc); + + s->state = BZ_X_IDLE; + RETURN(BZ_STREAM_END); + + default: AssertH ( False, 4001 ); + } + + AssertH ( False, 4002 ); + + save_state_and_return: + + s->save_i = i; + s->save_j = j; + s->save_t = t; + s->save_alphaSize = alphaSize; + s->save_nGroups = nGroups; + s->save_nSelectors = nSelectors; + s->save_EOB = EOB; + s->save_groupNo = groupNo; + s->save_groupPos = groupPos; + s->save_nextSym = nextSym; + s->save_nblockMAX = nblockMAX; + s->save_nblock = nblock; + s->save_es = es; + s->save_N = N; + s->save_curr = curr; + s->save_zt = zt; + s->save_zn = zn; + s->save_zvec = zvec; + s->save_zj = zj; + s->save_gSel = gSel; + s->save_gMinlen = gMinlen; + s->save_gLimit = gLimit; + s->save_gBase = gBase; + s->save_gPerm = gPerm; + + return retVal; +} + + +/*-------------------------------------------------------------*/ +/*--- end decompress.c ---*/ +/*-------------------------------------------------------------*/ diff --git a/lib/libbz2/huffman.c b/lib/libbz2/huffman.c new file mode 100644 index 00000000..43a1899e --- /dev/null +++ b/lib/libbz2/huffman.c @@ -0,0 +1,205 @@ + +/*-------------------------------------------------------------*/ +/*--- Huffman coding low-level stuff ---*/ +/*--- huffman.c ---*/ +/*-------------------------------------------------------------*/ + +/* ------------------------------------------------------------------ + This file is part of bzip2/libbzip2, a program and library for + lossless, block-sorting data compression. + + bzip2/libbzip2 version 1.0.8 of 13 July 2019 + Copyright (C) 1996-2019 Julian Seward + + Please read the WARNING, DISCLAIMER and PATENTS sections in the + README file. + + This program is released under the terms of the license contained + in the file LICENSE. + ------------------------------------------------------------------ */ + + +#include "bzlib_private.h" + +/*---------------------------------------------------*/ +#define WEIGHTOF(zz0) ((zz0) & 0xffffff00) +#define DEPTHOF(zz1) ((zz1) & 0x000000ff) +#define MYMAX(zz2,zz3) ((zz2) > (zz3) ? (zz2) : (zz3)) + +#define ADDWEIGHTS(zw1,zw2) \ + (WEIGHTOF(zw1)+WEIGHTOF(zw2)) | \ + (1 + MYMAX(DEPTHOF(zw1),DEPTHOF(zw2))) + +#define UPHEAP(z) \ +{ \ + Int32 zz, tmp; \ + zz = z; tmp = heap[zz]; \ + while (weight[tmp] < weight[heap[zz >> 1]]) { \ + heap[zz] = heap[zz >> 1]; \ + zz >>= 1; \ + } \ + heap[zz] = tmp; \ +} + +#define DOWNHEAP(z) \ +{ \ + Int32 zz, yy, tmp; \ + zz = z; tmp = heap[zz]; \ + while (True) { \ + yy = zz << 1; \ + if (yy > nHeap) break; \ + if (yy < nHeap && \ + weight[heap[yy+1]] < weight[heap[yy]]) \ + yy++; \ + if (weight[tmp] < weight[heap[yy]]) break; \ + heap[zz] = heap[yy]; \ + zz = yy; \ + } \ + heap[zz] = tmp; \ +} + + +/*---------------------------------------------------*/ +void BZ2_hbMakeCodeLengths ( UChar *len, + Int32 *freq, + Int32 alphaSize, + Int32 maxLen ) +{ + /*-- + Nodes and heap entries run from 1. Entry 0 + for both the heap and nodes is a sentinel. + --*/ + Int32 nNodes, nHeap, n1, n2, i, j, k; + Bool tooLong; + + Int32 heap [ BZ_MAX_ALPHA_SIZE + 2 ]; + Int32 weight [ BZ_MAX_ALPHA_SIZE * 2 ]; + Int32 parent [ BZ_MAX_ALPHA_SIZE * 2 ]; + + for (i = 0; i < alphaSize; i++) + weight[i+1] = (freq[i] == 0 ? 1 : freq[i]) << 8; + + while (True) { + + nNodes = alphaSize; + nHeap = 0; + + heap[0] = 0; + weight[0] = 0; + parent[0] = -2; + + for (i = 1; i <= alphaSize; i++) { + parent[i] = -1; + nHeap++; + heap[nHeap] = i; + UPHEAP(nHeap); + } + + AssertH( nHeap < (BZ_MAX_ALPHA_SIZE+2), 2001 ); + + while (nHeap > 1) { + n1 = heap[1]; heap[1] = heap[nHeap]; nHeap--; DOWNHEAP(1); + n2 = heap[1]; heap[1] = heap[nHeap]; nHeap--; DOWNHEAP(1); + nNodes++; + parent[n1] = parent[n2] = nNodes; + weight[nNodes] = ADDWEIGHTS(weight[n1], weight[n2]); + parent[nNodes] = -1; + nHeap++; + heap[nHeap] = nNodes; + UPHEAP(nHeap); + } + + AssertH( nNodes < (BZ_MAX_ALPHA_SIZE * 2), 2002 ); + + tooLong = False; + for (i = 1; i <= alphaSize; i++) { + j = 0; + k = i; + while (parent[k] >= 0) { k = parent[k]; j++; } + len[i-1] = j; + if (j > maxLen) tooLong = True; + } + + if (! tooLong) break; + + /* 17 Oct 04: keep-going condition for the following loop used + to be 'i < alphaSize', which missed the last element, + theoretically leading to the possibility of the compressor + looping. However, this count-scaling step is only needed if + one of the generated Huffman code words is longer than + maxLen, which up to and including version 1.0.2 was 20 bits, + which is extremely unlikely. In version 1.0.3 maxLen was + changed to 17 bits, which has minimal effect on compression + ratio, but does mean this scaling step is used from time to + time, enough to verify that it works. + + This means that bzip2-1.0.3 and later will only produce + Huffman codes with a maximum length of 17 bits. However, in + order to preserve backwards compatibility with bitstreams + produced by versions pre-1.0.3, the decompressor must still + handle lengths of up to 20. */ + + for (i = 1; i <= alphaSize; i++) { + j = weight[i] >> 8; + j = 1 + (j / 2); + weight[i] = j << 8; + } + } +} + + +/*---------------------------------------------------*/ +void BZ2_hbAssignCodes ( Int32 *code, + UChar *length, + Int32 minLen, + Int32 maxLen, + Int32 alphaSize ) +{ + Int32 n, vec, i; + + vec = 0; + for (n = minLen; n <= maxLen; n++) { + for (i = 0; i < alphaSize; i++) + if (length[i] == n) { code[i] = vec; vec++; }; + vec <<= 1; + } +} + + +/*---------------------------------------------------*/ +void BZ2_hbCreateDecodeTables ( Int32 *limit, + Int32 *base, + Int32 *perm, + UChar *length, + Int32 minLen, + Int32 maxLen, + Int32 alphaSize ) +{ + Int32 pp, i, j, vec; + + pp = 0; + for (i = minLen; i <= maxLen; i++) + for (j = 0; j < alphaSize; j++) + if (length[j] == i) { perm[pp] = j; pp++; }; + + for (i = 0; i < BZ_MAX_CODE_LEN; i++) base[i] = 0; + for (i = 0; i < alphaSize; i++) base[length[i]+1]++; + + for (i = 1; i < BZ_MAX_CODE_LEN; i++) base[i] += base[i-1]; + + for (i = 0; i < BZ_MAX_CODE_LEN; i++) limit[i] = 0; + vec = 0; + + for (i = minLen; i <= maxLen; i++) { + vec += (base[i+1] - base[i]); + limit[i] = vec-1; + vec <<= 1; + } + for (i = minLen + 1; i <= maxLen; i++) + base[i] = ((limit[i-1] + 1) << 1) - base[i]; +} + + +/*-------------------------------------------------------------*/ +/*--- end huffman.c ---*/ +/*-------------------------------------------------------------*/ diff --git a/lib/libbz2/library.json b/lib/libbz2/library.json new file mode 100644 index 00000000..c0c4966b --- /dev/null +++ b/lib/libbz2/library.json @@ -0,0 +1,22 @@ +{ + "name": "libbz2", + "version": "1.0.8", + "description": "bzip2 compression library for embedded systems", + "keywords": "compression, bzip2, bz2", + "license": "BSD-4-Clause", + "build": { + "flags": [ + "-DBZ_NO_STDIO" + ], + "srcFilter": [ + "+", + "+", + "+", + "+", + "+", + "+", + "+", + "+" + ] + } +} diff --git a/lib/libbz2/randtable.c b/lib/libbz2/randtable.c new file mode 100644 index 00000000..bdc6d4a4 --- /dev/null +++ b/lib/libbz2/randtable.c @@ -0,0 +1,84 @@ + +/*-------------------------------------------------------------*/ +/*--- Table for randomising repetitive blocks ---*/ +/*--- randtable.c ---*/ +/*-------------------------------------------------------------*/ + +/* ------------------------------------------------------------------ + This file is part of bzip2/libbzip2, a program and library for + lossless, block-sorting data compression. + + bzip2/libbzip2 version 1.0.8 of 13 July 2019 + Copyright (C) 1996-2019 Julian Seward + + Please read the WARNING, DISCLAIMER and PATENTS sections in the + README file. + + This program is released under the terms of the license contained + in the file LICENSE. + ------------------------------------------------------------------ */ + + +#include "bzlib_private.h" + + +/*---------------------------------------------*/ +Int32 BZ2_rNums[512] = { + 619, 720, 127, 481, 931, 816, 813, 233, 566, 247, + 985, 724, 205, 454, 863, 491, 741, 242, 949, 214, + 733, 859, 335, 708, 621, 574, 73, 654, 730, 472, + 419, 436, 278, 496, 867, 210, 399, 680, 480, 51, + 878, 465, 811, 169, 869, 675, 611, 697, 867, 561, + 862, 687, 507, 283, 482, 129, 807, 591, 733, 623, + 150, 238, 59, 379, 684, 877, 625, 169, 643, 105, + 170, 607, 520, 932, 727, 476, 693, 425, 174, 647, + 73, 122, 335, 530, 442, 853, 695, 249, 445, 515, + 909, 545, 703, 919, 874, 474, 882, 500, 594, 612, + 641, 801, 220, 162, 819, 984, 589, 513, 495, 799, + 161, 604, 958, 533, 221, 400, 386, 867, 600, 782, + 382, 596, 414, 171, 516, 375, 682, 485, 911, 276, + 98, 553, 163, 354, 666, 933, 424, 341, 533, 870, + 227, 730, 475, 186, 263, 647, 537, 686, 600, 224, + 469, 68, 770, 919, 190, 373, 294, 822, 808, 206, + 184, 943, 795, 384, 383, 461, 404, 758, 839, 887, + 715, 67, 618, 276, 204, 918, 873, 777, 604, 560, + 951, 160, 578, 722, 79, 804, 96, 409, 713, 940, + 652, 934, 970, 447, 318, 353, 859, 672, 112, 785, + 645, 863, 803, 350, 139, 93, 354, 99, 820, 908, + 609, 772, 154, 274, 580, 184, 79, 626, 630, 742, + 653, 282, 762, 623, 680, 81, 927, 626, 789, 125, + 411, 521, 938, 300, 821, 78, 343, 175, 128, 250, + 170, 774, 972, 275, 999, 639, 495, 78, 352, 126, + 857, 956, 358, 619, 580, 124, 737, 594, 701, 612, + 669, 112, 134, 694, 363, 992, 809, 743, 168, 974, + 944, 375, 748, 52, 600, 747, 642, 182, 862, 81, + 344, 805, 988, 739, 511, 655, 814, 334, 249, 515, + 897, 955, 664, 981, 649, 113, 974, 459, 893, 228, + 433, 837, 553, 268, 926, 240, 102, 654, 459, 51, + 686, 754, 806, 760, 493, 403, 415, 394, 687, 700, + 946, 670, 656, 610, 738, 392, 760, 799, 887, 653, + 978, 321, 576, 617, 626, 502, 894, 679, 243, 440, + 680, 879, 194, 572, 640, 724, 926, 56, 204, 700, + 707, 151, 457, 449, 797, 195, 791, 558, 945, 679, + 297, 59, 87, 824, 713, 663, 412, 693, 342, 606, + 134, 108, 571, 364, 631, 212, 174, 643, 304, 329, + 343, 97, 430, 751, 497, 314, 983, 374, 822, 928, + 140, 206, 73, 263, 980, 736, 876, 478, 430, 305, + 170, 514, 364, 692, 829, 82, 855, 953, 676, 246, + 369, 970, 294, 750, 807, 827, 150, 790, 288, 923, + 804, 378, 215, 828, 592, 281, 565, 555, 710, 82, + 896, 831, 547, 261, 524, 462, 293, 465, 502, 56, + 661, 821, 976, 991, 658, 869, 905, 758, 745, 193, + 768, 550, 608, 933, 378, 286, 215, 979, 792, 961, + 61, 688, 793, 644, 986, 403, 106, 366, 905, 644, + 372, 567, 466, 434, 645, 210, 389, 550, 919, 135, + 780, 773, 635, 389, 707, 100, 626, 958, 165, 504, + 920, 176, 193, 713, 857, 265, 203, 50, 668, 108, + 645, 990, 626, 197, 510, 357, 358, 850, 858, 364, + 936, 638 +}; + + +/*-------------------------------------------------------------*/ +/*--- end randtable.c ---*/ +/*-------------------------------------------------------------*/ diff --git a/platformio.ini b/platformio.ini index a7f8778e..c23bf164 100644 --- a/platformio.ini +++ b/platformio.ini @@ -42,7 +42,8 @@ lib_deps = ble_interface lxst_audio sh123/esp32_codec2@^1.0.7 - symlink://${PROJECT_DIR}/deps/microReticulum/lib/libbz2 + libbz2 + https://github.com/attermann/microStore.git ; Library dependency finder mode (deep search) lib_ldf_mode = deep+ @@ -61,8 +62,12 @@ build_flags = -DARDUINO_USB_MODE=1 -DLV_CONF_INCLUDE_SIMPLE -Ilib - -Ideps/microReticulum/lib/libbz2 + -Ilib/libbz2 -Ideps/microReticulum/src + -Isrc-shim + -Isrc-shim/Cryptography + -Isrc-shim/Instrumentation + -Isrc-shim/Utilities -DBZ_NO_STDIO -DUSE_BLUEDROID -Os @@ -128,7 +133,8 @@ lib_deps = ble_interface lxst_audio sh123/esp32_codec2@^1.0.7 - symlink://${PROJECT_DIR}/deps/microReticulum/lib/libbz2 + libbz2 + https://github.com/attermann/microStore.git ; Build configuration build_flags = @@ -142,8 +148,12 @@ build_flags = -DARDUINO_USB_MODE=1 -DLV_CONF_INCLUDE_SIMPLE -Ilib - -Ideps/microReticulum/lib/libbz2 + -Ilib/libbz2 -Ideps/microReticulum/src + -Isrc-shim + -Isrc-shim/Cryptography + -Isrc-shim/Instrumentation + -Isrc-shim/Utilities -DBZ_NO_STDIO -DUSE_NIMBLE -DCONFIG_BT_NIMBLE_MEM_ALLOC_MODE_EXTERNAL=1 diff --git a/src-shim/Buffer.cpp b/src-shim/Buffer.cpp new file mode 100644 index 00000000..78e26482 --- /dev/null +++ b/src-shim/Buffer.cpp @@ -0,0 +1,404 @@ +#include "Buffer.h" +#include "ChannelData.h" // Required for Channel template instantiation +#include "Cryptography/BZ2.h" + +#include + +namespace RNS { + +//============================================================================== +// RawChannelReaderData - Internal data for RawChannelReader +//============================================================================== + +class RawChannelReaderData { +public: + RawChannelReaderData(uint16_t stream_id, Channel& channel) + : _stream_id(stream_id), _channel(&channel), _eof(false), _closed(false) { + MEM("RawChannelReaderData object created"); + } + + ~RawChannelReaderData() { + if (!_closed) { + close(); + } + MEM("RawChannelReaderData object destroyed"); + } + + bool handle_message(MessageBase& msg) { + if (_closed) return false; + + // Only handle StreamDataMessage + if (msg.msgtype() != StreamDataMessage::MSGTYPE) { + return false; + } + + StreamDataMessage& stream_msg = static_cast(msg); + + // Filter by stream_id + if (stream_msg.stream_id != _stream_id) { + return false; + } + + DEBUGF("RawChannelReader: Received %zu bytes on stream %u (eof=%d)", + stream_msg.data.size(), _stream_id, stream_msg.eof); + + // Accumulate data + if (stream_msg.data) { + _buffer += stream_msg.data; + } + + // Set EOF flag + if (stream_msg.eof) { + _eof = true; + DEBUG("RawChannelReader: EOF received"); + } + + // Notify callbacks + notify_ready(); + + return true; // Message consumed + } + + void notify_ready() { + size_t avail = _buffer.size(); + if (avail == 0 && !_eof) return; + + // Make a copy of callbacks in case they modify the list + auto callbacks_copy = _ready_callbacks; + for (auto& callback : callbacks_copy) { + try { + callback(avail); + } catch (...) { + ERROR("RawChannelReader: Callback threw exception"); + } + } + } + + void close() { + _closed = true; + _ready_callbacks.clear(); + DEBUG("RawChannelReader: Closed"); + } + + uint16_t _stream_id; + Channel* _channel; + Bytes _buffer; + bool _eof; + bool _closed; + std::vector _ready_callbacks; +}; + +//============================================================================== +// StreamDataMessage +//============================================================================== + +StreamDataMessage::StreamDataMessage(uint16_t stream_id, const Bytes& data, + bool eof, bool compressed) + : stream_id(stream_id), data(data), eof(eof), compressed(compressed) { +} + +Bytes StreamDataMessage::pack() const { + // Build 2-byte header: EOF(bit15) | COMP(bit14) | STREAM_ID(bits13-0) + uint16_t header = (stream_id & Type::Buffer::STREAM_ID_MASK); + if (eof) header |= Type::Buffer::FLAG_EOF; + if (compressed) header |= Type::Buffer::FLAG_COMPRESSED; + + Bytes result; + result.reserve(Type::Buffer::STREAM_OVERHEAD + data.size()); + + // Big-endian header (matching Python struct.pack(">H", header)) + result.append(static_cast((header >> 8) & 0xFF)); + result.append(static_cast(header & 0xFF)); + + // Data payload (already compressed if compressed flag is set) + if (data) { + result += data; + } + + return result; +} + +void StreamDataMessage::unpack(const Bytes& raw) { + if (raw.size() < Type::Buffer::STREAM_OVERHEAD) { + ERROR("StreamDataMessage::unpack: Data too short"); + return; + } + + // Read big-endian header + uint16_t header = (static_cast(raw[0]) << 8) | raw[1]; + + eof = (header & Type::Buffer::FLAG_EOF) != 0; + compressed = (header & Type::Buffer::FLAG_COMPRESSED) != 0; + stream_id = header & Type::Buffer::STREAM_ID_MASK; + + // Extract data payload + if (raw.size() > Type::Buffer::STREAM_OVERHEAD) { + data = raw.mid(Type::Buffer::STREAM_OVERHEAD); + + // Decompress if needed + if (compressed && data.size() > 0) { + data = Cryptography::bz2_decompress(data); + } + } else { + data = Bytes::NONE; + } +} + +//============================================================================== +// RawChannelReader - Uses shared_ptr to RawChannelReaderData +//============================================================================== + +RawChannelReader::RawChannelReader(uint16_t stream_id, Channel& channel) + : _object(std::make_shared(stream_id, channel)) { + + // Register StreamDataMessage type (as system message) + _object->_channel->register_message_type(true); + + // Add message handler - capture shared_ptr to keep data alive + // and use weak_ptr to avoid preventing destruction + std::weak_ptr weak_obj = _object; + _object->_channel->add_message_handler([weak_obj](MessageBase& msg) -> bool { + if (auto obj = weak_obj.lock()) { + return obj->handle_message(msg); + } + return false; // Object destroyed, don't handle + }); + + DEBUGF("RawChannelReader: Created for stream_id=%u", stream_id); +} + +RawChannelReader::~RawChannelReader() { + // shared_ptr handles cleanup automatically + MEM("RawChannelReader object destroyed"); +} + +Bytes RawChannelReader::read(size_t max_bytes) { + if (!_object || _object->_buffer.size() == 0) { + return Bytes::NONE; + } + + size_t to_read = (max_bytes == 0) ? _object->_buffer.size() + : std::min(max_bytes, _object->_buffer.size()); + Bytes result = _object->_buffer.left(to_read); + _object->_buffer = _object->_buffer.mid(to_read); + + DEBUGF("RawChannelReader: Read %zu bytes, %zu remaining", to_read, _object->_buffer.size()); + return result; +} + +Bytes RawChannelReader::readline() { + if (!_object) return Bytes::NONE; + + // Search for newline + for (size_t i = 0; i < _object->_buffer.size(); i++) { + if (_object->_buffer[i] == '\n') { + // Return including the newline + Bytes line = _object->_buffer.left(i + 1); + _object->_buffer = _object->_buffer.mid(i + 1); + return line; + } + } + + // No complete line available + // If EOF, return remaining data as final line (no newline) + if (_object->_eof && _object->_buffer.size() > 0) { + Bytes line = _object->_buffer; + _object->_buffer = Bytes::NONE; + return line; + } + + return Bytes::NONE; +} + +size_t RawChannelReader::available() const { + if (!_object) return 0; + return _object->_buffer.size(); +} + +bool RawChannelReader::eof() const { + if (!_object) return true; + return _object->_eof && _object->_buffer.size() == 0; +} + +void RawChannelReader::add_ready_callback(ReadyCallback callback) { + if (!_object) return; + _object->_ready_callbacks.push_back(callback); +} + +void RawChannelReader::remove_ready_callback(ReadyCallback callback) { + // Note: Function comparison is tricky in C++ + // This is a simplified implementation + // For production, consider using callback IDs +} + +void RawChannelReader::close() { + if (!_object) return; + _object->close(); +} + +//============================================================================== +// RawChannelWriter +//============================================================================== + +RawChannelWriter::RawChannelWriter(uint16_t stream_id, Channel& channel) + : _stream_id(stream_id), _channel(&channel), _eof_sent(false) { + + // Calculate max data length (Channel MDU minus stream header) + _max_data_len = _channel->mdu() - Type::Buffer::STREAM_OVERHEAD; + + // Register StreamDataMessage type (as system message) + // This may already be registered by a reader, but that's OK + _channel->register_message_type(true); + + DEBUGF("RawChannelWriter: Created for stream_id=%u, max_data_len=%zu", + _stream_id, _max_data_len); +} + +RawChannelWriter::~RawChannelWriter() { + if (!_eof_sent && _channel) { + close(); + } +} + +RawChannelWriter::RawChannelWriter(RawChannelWriter&& other) noexcept + : _stream_id(other._stream_id), + _channel(other._channel), + _max_data_len(other._max_data_len), + _eof_sent(other._eof_sent) { + other._channel = nullptr; + other._eof_sent = true; +} + +RawChannelWriter& RawChannelWriter::operator=(RawChannelWriter&& other) noexcept { + if (this != &other) { + if (!_eof_sent && _channel) { + close(); + } + _stream_id = other._stream_id; + _channel = other._channel; + _max_data_len = other._max_data_len; + _eof_sent = other._eof_sent; + other._channel = nullptr; + other._eof_sent = true; + } + return *this; +} + +size_t RawChannelWriter::write(const Bytes& data) { + if (!_channel || _eof_sent) { + ERROR("RawChannelWriter: Cannot write after close"); + return 0; + } + + if (!data || data.size() == 0) { + return 0; + } + + size_t chunk_len = std::min(data.size(), Type::Buffer::MAX_CHUNK_LEN); + Bytes chunk = data.left(chunk_len); + + bool use_compression = false; + Bytes send_data; + size_t processed = 0; + + // Try compression at decreasing chunk sizes (matching Python Buffer.py) + if (chunk_len > Type::Buffer::COMPRESSION_MIN_SIZE) { + for (size_t try_num = 1; try_num <= Type::Buffer::COMPRESSION_TRIES; try_num++) { + size_t segment_len = chunk_len / try_num; + if (segment_len == 0) break; + + Bytes segment = chunk.left(segment_len); + Bytes compressed = Cryptography::bz2_compress(segment); + + if (compressed.size() < _max_data_len && + compressed.size() < segment_len) { + use_compression = true; + send_data = compressed; + processed = segment_len; + DEBUGF("RawChannelWriter: Compression succeeded: %zu -> %zu bytes", + segment_len, compressed.size()); + break; + } + } + } + + // If compression didn't help, send uncompressed + if (!use_compression) { + send_data = chunk.left(_max_data_len); + processed = send_data.size(); + } + + // Build and send message + StreamDataMessage msg; + msg.stream_id = _stream_id; + msg.data = send_data; + msg.eof = false; + msg.compressed = use_compression; + + _channel->send(msg); + + DEBUGF("RawChannelWriter: Sent %zu bytes on stream %u (compressed=%d)", + processed, _stream_id, use_compression); + + return processed; +} + +void RawChannelWriter::flush() { + // In the current implementation, write() sends immediately + // This is a no-op but provided for API compatibility + DEBUG("RawChannelWriter: Flush called (no-op)"); +} + +void RawChannelWriter::close() { + if (_eof_sent) return; + + if (_channel) { + // Send EOF message + StreamDataMessage msg; + msg.stream_id = _stream_id; + msg.data = Bytes::NONE; + msg.eof = true; + msg.compressed = false; + + _channel->send(msg); + + DEBUGF("RawChannelWriter: Sent EOF on stream %u", _stream_id); + } + + _eof_sent = true; +} + +//============================================================================== +// Buffer namespace factory functions +//============================================================================== + +namespace Buffer { + +RawChannelReader create_reader(uint16_t stream_id, Channel& channel, + RawChannelReader::ReadyCallback callback) { + RawChannelReader reader(stream_id, channel); + if (callback) { + reader.add_ready_callback(callback); + } + return reader; +} + +RawChannelWriter create_writer(uint16_t stream_id, Channel& channel) { + return RawChannelWriter(stream_id, channel); +} + +std::pair +create_bidirectional_buffer(uint16_t rx_stream_id, uint16_t tx_stream_id, + Channel& channel, + RawChannelReader::ReadyCallback callback) { + RawChannelReader reader(rx_stream_id, channel); + if (callback) { + reader.add_ready_callback(callback); + } + RawChannelWriter writer(tx_stream_id, channel); + return std::make_pair(std::move(reader), std::move(writer)); +} + +} // namespace Buffer + +} // namespace RNS diff --git a/src-shim/Buffer.h b/src-shim/Buffer.h new file mode 100644 index 00000000..b5016c35 --- /dev/null +++ b/src-shim/Buffer.h @@ -0,0 +1,163 @@ +#pragma once + +#ifndef RNS_BUFFER_H +#define RNS_BUFFER_H + +#include "Bytes.h" +#include "Channel.h" +#include "MessageBase.h" +#include "Type.h" +#include "Log.h" + +#include +#include +#include +#include + +namespace RNS { + +/** + * StreamDataMessage - Wire format for Buffer data over Channel + * + * Header (2 bytes, big-endian): + * Bit 15: EOF flag + * Bit 14: Compression flag + * Bits 13-0: Stream ID (max 16383) + * + * Followed by optional data payload (BZ2 compressed if flag set) + */ +class StreamDataMessage : public MessageBase { +public: + static constexpr uint16_t MSGTYPE = Type::Channel::SMT_STREAM_DATA; + + // Fields + uint16_t stream_id = 0; + Bytes data; + bool eof = false; + bool compressed = false; + + // Constructors + StreamDataMessage() = default; + StreamDataMessage(uint16_t stream_id, const Bytes& data = Bytes::NONE, + bool eof = false, bool compressed = false); + + // MessageBase interface + uint16_t msgtype() const override { return MSGTYPE; } + Bytes pack() const override; + void unpack(const Bytes& raw) override; +}; + +// Forward declaration for internal data +class RawChannelReaderData; + +/** + * RawChannelReader - Read stream data from a Channel + * + * Uses shared_ptr internally so that move operations preserve callback validity. + */ +class RawChannelReader { +public: + using ReadyCallback = std::function; + + RawChannelReader(uint16_t stream_id, Channel& channel); + RawChannelReader(Type::NoneConstructor none) { MEM("RawChannelReader NONE object created"); } + ~RawChannelReader(); + + // Copy/move - shallow copy of shared_ptr (like other RNS types) + RawChannelReader(const RawChannelReader& other) : _object(other._object) { + MEM("RawChannelReader object copy created"); + } + RawChannelReader(RawChannelReader&& other) noexcept : _object(std::move(other._object)) { + MEM("RawChannelReader object moved"); + } + RawChannelReader& operator=(const RawChannelReader& other) { + _object = other._object; + return *this; + } + RawChannelReader& operator=(RawChannelReader&& other) noexcept { + _object = std::move(other._object); + return *this; + } + + // Validity check + operator bool() const { return _object.get() != nullptr; } + + // Reading interface + Bytes read(size_t max_bytes = 0); // 0 = read all available + Bytes readline(); + size_t available() const; + bool eof() const; + + // Callback management + void add_ready_callback(ReadyCallback callback); + void remove_ready_callback(ReadyCallback callback); + + // Cleanup + void close(); + +private: + std::shared_ptr _object; +}; + +/** + * RawChannelWriter - Write stream data to a Channel + */ +class RawChannelWriter { +public: + RawChannelWriter(uint16_t stream_id, Channel& channel); + ~RawChannelWriter(); + + // Disable copy + RawChannelWriter(const RawChannelWriter&) = delete; + RawChannelWriter& operator=(const RawChannelWriter&) = delete; + + // Move support + RawChannelWriter(RawChannelWriter&& other) noexcept; + RawChannelWriter& operator=(RawChannelWriter&& other) noexcept; + + // Writing interface + size_t write(const Bytes& data); + void flush(); + void close(); // Send EOF + +private: + uint16_t _stream_id; + Channel* _channel; + size_t _max_data_len; + bool _eof_sent = false; +}; + +/** + * Buffer namespace - Factory functions for creating readers/writers + */ +namespace Buffer { + + /** + * Create a reader for receiving stream data + */ + RawChannelReader create_reader(uint16_t stream_id, Channel& channel, + RawChannelReader::ReadyCallback callback = nullptr); + + /** + * Create a writer for sending stream data + */ + RawChannelWriter create_writer(uint16_t stream_id, Channel& channel); + + /** + * Create bidirectional buffer pair + * @param rx_stream_id Stream ID for receiving data + * @param tx_stream_id Stream ID for sending data + * @param channel The Channel to use + * @param callback Optional callback when data is ready to read + * @return Pair of (reader, writer) + */ + std::pair + create_bidirectional_buffer(uint16_t rx_stream_id, uint16_t tx_stream_id, + Channel& channel, + RawChannelReader::ReadyCallback callback = nullptr); + +} // namespace Buffer + +} // namespace RNS + +#endif // RNS_BUFFER_H diff --git a/src-shim/BytesPool.h b/src-shim/BytesPool.h new file mode 100644 index 00000000..b05767d3 --- /dev/null +++ b/src-shim/BytesPool.h @@ -0,0 +1,356 @@ +#pragma once + +/** + * BytesPool.h - Pool for Bytes Data objects to prevent heap fragmentation + * + * Problem: Bytes uses shared_ptr where Data = vector. + * Each make_shared allocates a control block (24 bytes) + vector metadata. + * In long-running firmware, this causes heap fragmentation. + * + * Solution: Pool the Data objects (vectors) themselves. When a Bytes object + * needs storage for common sizes (<=1024 bytes), it gets a pre-allocated + * Data from the pool with capacity already reserved. When the shared_ptr + * refcount hits 0, a custom deleter returns the Data to the pool instead + * of destroying it. + * + * The pool has four tiers sized for Reticulum packet processing: + * - 64 bytes (512 slots): hashes (16-32 bytes), small fields - highest traffic + * - 256 bytes (24 slots): keys, small announces + * - 512 bytes (16 slots): standard packets (MTU=500 + margin) + * - 1024 bytes (16 slots): resource advertisements, large packets + * + * Thread-safe via FreeRTOS spinlock (ESP32) or std::mutex (native). + * + * Usage (in Bytes.cpp): + * auto [data, tier] = BytesPool::instance().acquire(capacity); + * if (data) { + * // Got pooled Data - use with custom deleter + * _data = SharedData(data, BytesPoolDeleter{tier}); + * } else { + * // Pool exhausted - fall back to make_shared + * _data = std::make_shared(); + * } + */ + +#include "PSRAMAllocator.h" +#include "Log.h" + +#include +#include +#include +#include + +// FreeRTOS spinlock support - only on ESP32 +#if defined(ESP_PLATFORM) || defined(ARDUINO) +#include "freertos/FreeRTOS.h" +#include "freertos/portmacro.h" +#define BYTESPOOL_USE_SPINLOCK 1 +#else +// Native build - use std::mutex instead +#include +#define BYTESPOOL_USE_SPINLOCK 0 +#endif + +namespace RNS { + +// Pool configuration - sized for Reticulum packet processing +// MTU=500, most packets fit in 512 bytes, large resources may need 1024 +namespace BytesPoolConfig { + static constexpr size_t TIER_TINY = 64; // Hashes (16-32 bytes), small fields + static constexpr size_t TIER_SMALL = 256; // Small packets, keys + static constexpr size_t TIER_MEDIUM = 512; // Standard packets + static constexpr size_t TIER_LARGE = 1024; // Large packets, resource ads + + // Slot counts per tier - tuned from runtime observation (2026-01-24) + // NOTE: Each slot uses ~16 bytes internal RAM for vector metadata + stack pointer + // 512 slots = ~8KB internal RAM overhead (conservative for busy networks) + static constexpr size_t TINY_SLOTS = 512; // High traffic tier (balanced for memory) + static constexpr size_t SMALL_SLOTS = 8; // Low traffic (peak 1) + static constexpr size_t MEDIUM_SLOTS = 8; // Rare (packets, keep headroom) + static constexpr size_t LARGE_SLOTS = 8; // Rare (resource ads, keep headroom) + + // Tier identifiers for deleter + enum Tier : uint8_t { + TIER_NONE = 0, // Not from pool (fallback allocation) + TIER_64 = 1, + TIER_256 = 2, + TIER_512 = 3, + TIER_1024 = 4 + }; +} + +// Forward declaration - Data is vector with PSRAMAllocator +using PooledData = std::vector>; + +/** + * Pool for Bytes Data objects (vectors). + * + * Each tier maintains a stack of pre-allocated vectors with capacity reserved. + * Vectors are cleared (size=0) but capacity preserved when returned to pool. + * + * This eliminates: + * - Repeated vector construction/destruction + * - Repeated capacity reservation allocations + * - shared_ptr control block allocations (via make_shared replacement) + * + * Memory footprint (tuned 2026-01-24): + * - Tiny: 512 slots x 64 bytes = 32KB backing + ~8KB metadata (internal RAM) + * - Small: 8 slots x 256 bytes = 2KB backing + ~128B metadata + * - Medium: 8 slots x 512 bytes = 4KB backing + ~128B metadata + * - Large: 8 slots x 1024 bytes = 8KB backing + ~128B metadata + * - Total: ~46KB PSRAM backing + ~8.5KB internal RAM metadata + */ +class BytesPool { +public: + // Singleton access - pool must be global for custom deleter + static BytesPool& instance() { + static BytesPool pool; + return pool; + } + + /** + * Acquire a Data object from pool. + * Returns {pointer, tier} or {nullptr, TIER_NONE} if pool exhausted. + * + * The returned Data is empty (size=0) but has capacity >= requested. + * Caller must use the tier value to construct BytesPoolDeleter. + */ + std::pair acquire(size_t requested_capacity) { + _total_requests++; + +#if BYTESPOOL_USE_SPINLOCK + portENTER_CRITICAL(&_mux); +#else + std::lock_guard lock(_mutex); +#endif + + PooledData* result = nullptr; + BytesPoolConfig::Tier tier = BytesPoolConfig::TIER_NONE; + + // Try smallest tier that fits + if (requested_capacity <= BytesPoolConfig::TIER_TINY) { + if (_tiny_count > 0) { + result = _tiny_stack[--_tiny_count]; + tier = BytesPoolConfig::TIER_64; + _pool_hits++; + } + } + else if (requested_capacity <= BytesPoolConfig::TIER_SMALL) { + if (_small_count > 0) { + result = _small_stack[--_small_count]; + tier = BytesPoolConfig::TIER_256; + _pool_hits++; + } + } + else if (requested_capacity <= BytesPoolConfig::TIER_MEDIUM) { + if (_medium_count > 0) { + result = _medium_stack[--_medium_count]; + tier = BytesPoolConfig::TIER_512; + _pool_hits++; + } + } + else if (requested_capacity <= BytesPoolConfig::TIER_LARGE) { + if (_large_count > 0) { + result = _large_stack[--_large_count]; + tier = BytesPoolConfig::TIER_1024; + _pool_hits++; + } + } + // Oversized requests fall through with nullptr + + if (!result) { + _pool_misses++; + } + +#if BYTESPOOL_USE_SPINLOCK + portEXIT_CRITICAL(&_mux); +#endif + + return {result, tier}; + } + + /** + * Release a Data object back to pool. + * Called by BytesPoolDeleter when shared_ptr refcount hits 0. + * + * The Data is cleared (preserving capacity) and pushed to tier stack. + */ + void release(PooledData* data, BytesPoolConfig::Tier tier) { + if (!data || tier == BytesPoolConfig::TIER_NONE) { + // Not from pool - should not happen, but defensive + return; + } + + // Clear but preserve capacity + data->clear(); + +#if BYTESPOOL_USE_SPINLOCK + portENTER_CRITICAL(&_mux); +#else + std::lock_guard lock(_mutex); +#endif + + switch (tier) { + case BytesPoolConfig::TIER_64: + if (_tiny_count < BytesPoolConfig::TINY_SLOTS) { + _tiny_stack[_tiny_count++] = data; + } + // else pool full - data leaks (shouldn't happen in normal operation) + break; + case BytesPoolConfig::TIER_256: + if (_small_count < BytesPoolConfig::SMALL_SLOTS) { + _small_stack[_small_count++] = data; + } + break; + case BytesPoolConfig::TIER_512: + if (_medium_count < BytesPoolConfig::MEDIUM_SLOTS) { + _medium_stack[_medium_count++] = data; + } + break; + case BytesPoolConfig::TIER_1024: + if (_large_count < BytesPoolConfig::LARGE_SLOTS) { + _large_stack[_large_count++] = data; + } + break; + default: + break; + } + +#if BYTESPOOL_USE_SPINLOCK + portEXIT_CRITICAL(&_mux); +#endif + } + + // Instrumentation + size_t total_requests() const { return _total_requests; } + size_t pool_hits() const { return _pool_hits; } + size_t pool_misses() const { return _pool_misses; } + size_t fallback_count() const { return _fallback_count; } + float hit_rate() const { + return _total_requests > 0 ? (float)_pool_hits / _total_requests : 0.0f; + } + + /** + * Record a fallback to heap allocation and log WARNING. + * Called by Bytes.cpp when pool is exhausted but fallback succeeds. + */ + void recordFallback(size_t requested_size) { + _fallback_count++; + WARNINGF("BytesPool: exhausted, falling back to heap (requested=%zu bytes, " + "tiny=%zu/%zu small=%zu/%zu med=%zu/%zu large=%zu/%zu)", + requested_size, + tiny_in_use(), BytesPoolConfig::TINY_SLOTS, + small_in_use(), BytesPoolConfig::SMALL_SLOTS, + medium_in_use(), BytesPoolConfig::MEDIUM_SLOTS, + large_in_use(), BytesPoolConfig::LARGE_SLOTS); + } + + // Current pool state + size_t tiny_available() const { return _tiny_count; } + size_t small_available() const { return _small_count; } + size_t medium_available() const { return _medium_count; } + size_t large_available() const { return _large_count; } + size_t tiny_in_use() const { return BytesPoolConfig::TINY_SLOTS - _tiny_count; } + size_t small_in_use() const { return BytesPoolConfig::SMALL_SLOTS - _small_count; } + size_t medium_in_use() const { return BytesPoolConfig::MEDIUM_SLOTS - _medium_count; } + size_t large_in_use() const { return BytesPoolConfig::LARGE_SLOTS - _large_count; } + + // Log statistics for tuning + void logStats() const { + INFOF("BytesPool: requests=%zu hits=%zu misses=%zu fallbacks=%zu hit_rate=%d%% " + "tiny=%zu/%zu small=%zu/%zu med=%zu/%zu large=%zu/%zu", + _total_requests, _pool_hits, _pool_misses, _fallback_count, + (int)(hit_rate() * 100), + tiny_in_use(), BytesPoolConfig::TINY_SLOTS, + small_in_use(), BytesPoolConfig::SMALL_SLOTS, + medium_in_use(), BytesPoolConfig::MEDIUM_SLOTS, + large_in_use(), BytesPoolConfig::LARGE_SLOTS); + } + +private: + BytesPool() { +#if BYTESPOOL_USE_SPINLOCK + portMUX_INITIALIZE(&_mux); +#endif + // Pre-allocate all pool entries + // This is done at construction (startup) to front-load allocations + initializeTier(_tiny_storage, _tiny_stack, _tiny_count, + BytesPoolConfig::TIER_TINY, BytesPoolConfig::TINY_SLOTS); + initializeTier(_small_storage, _small_stack, _small_count, + BytesPoolConfig::TIER_SMALL, BytesPoolConfig::SMALL_SLOTS); + initializeTier(_medium_storage, _medium_stack, _medium_count, + BytesPoolConfig::TIER_MEDIUM, BytesPoolConfig::MEDIUM_SLOTS); + initializeTier(_large_storage, _large_stack, _large_count, + BytesPoolConfig::TIER_LARGE, BytesPoolConfig::LARGE_SLOTS); + } + + // Non-copyable + BytesPool(const BytesPool&) = delete; + BytesPool& operator=(const BytesPool&) = delete; + + // Initialize a tier with pre-allocated vectors + void initializeTier(PooledData* storage, PooledData** stack, size_t& count, + size_t capacity, size_t slots) { + for (size_t i = 0; i < slots; i++) { + // Placement new to construct in storage array + new (&storage[i]) PooledData(); + storage[i].reserve(capacity); + stack[i] = &storage[i]; + } + count = slots; + } + + // Storage for pooled vectors (fixed arrays avoid dynamic allocation) + PooledData _tiny_storage[BytesPoolConfig::TINY_SLOTS]; + PooledData _small_storage[BytesPoolConfig::SMALL_SLOTS]; + PooledData _medium_storage[BytesPoolConfig::MEDIUM_SLOTS]; + PooledData _large_storage[BytesPoolConfig::LARGE_SLOTS]; + + // Stacks of available vectors (indices into storage arrays) + PooledData* _tiny_stack[BytesPoolConfig::TINY_SLOTS]; + PooledData* _small_stack[BytesPoolConfig::SMALL_SLOTS]; + PooledData* _medium_stack[BytesPoolConfig::MEDIUM_SLOTS]; + PooledData* _large_stack[BytesPoolConfig::LARGE_SLOTS]; + + // Stack counts (how many available in each tier) + size_t _tiny_count = 0; + size_t _small_count = 0; + size_t _medium_count = 0; + size_t _large_count = 0; + + // Instrumentation counters + size_t _total_requests = 0; + size_t _pool_hits = 0; + size_t _pool_misses = 0; + size_t _fallback_count = 0; // Heap fallbacks due to pool exhaustion + +#if BYTESPOOL_USE_SPINLOCK + portMUX_TYPE _mux; +#else + std::mutex _mutex; +#endif +}; + +/** + * Custom deleter for shared_ptr that returns Data to pool. + * + * When shared_ptr refcount hits 0, this deleter is called instead of delete. + * The Data is returned to the appropriate tier in BytesPool. + */ +struct BytesPoolDeleter { + BytesPoolConfig::Tier tier; + + explicit BytesPoolDeleter(BytesPoolConfig::Tier t = BytesPoolConfig::TIER_NONE) + : tier(t) {} + + void operator()(PooledData* data) const { + if (data && tier != BytesPoolConfig::TIER_NONE) { + // Return to pool instead of destroying + BytesPool::instance().release(data, tier); + } + // Note: We never delete the data - it lives in BytesPool's storage arrays + // If tier is TIER_NONE, this is a fallback allocation and won't be called + } +}; + +} // namespace RNS diff --git a/src-shim/ChannelData.h b/src-shim/ChannelData.h new file mode 100644 index 00000000..c6a9170e --- /dev/null +++ b/src-shim/ChannelData.h @@ -0,0 +1,369 @@ +#pragma once + +#ifndef RNS_CHANNEL_DATA_H +#define RNS_CHANNEL_DATA_H + +#include "Bytes.h" +#include "Link.h" +#include "Packet.h" +#include "MessageBase.h" +#include "Type.h" +#include "Log.h" + +#include +#include +#include +#include +#include // for std::move + +namespace RNS { + +// Forward declaration +class Channel; + +// Envelope wraps a message with protocol metadata +class Envelope { +public: + Envelope() = default; + Envelope(uint16_t msgtype, uint16_t sequence, const Bytes& raw) + : _msgtype(msgtype), _sequence(sequence), _raw(raw) {} + + // Move semantics (unique_ptr makes class non-copyable) + Envelope(Envelope&&) = default; + Envelope& operator=(Envelope&&) = default; + + // Delete copy operations + Envelope(const Envelope&) = delete; + Envelope& operator=(const Envelope&) = delete; + + uint16_t msgtype() const { return _msgtype; } + uint16_t sequence() const { return _sequence; } + const Bytes& raw() const { return _raw; } + + // For TX tracking + Packet packet() const { return _packet; } + void set_packet(const Packet& packet) { _packet = packet; } + uint8_t tries() const { return _tries; } + void increment_tries() { _tries++; } + double timestamp() const { return _timestamp; } + void set_timestamp(double ts) { _timestamp = ts; } + bool tracked() const { return _tracked; } + void set_tracked(bool tracked) { _tracked = tracked; } + + // Message instance (for RX) + std::unique_ptr& message() { return _message; } + void set_message(std::unique_ptr msg) { _message = std::move(msg); } + + // Pack envelope to wire format (big-endian) + Bytes pack() const { + // Wire format: MSGTYPE(2) + SEQUENCE(2) + LENGTH(2) + DATA(N) + // All values big-endian + Bytes result; + + // Allocate exact size needed + size_t data_len = _raw.size(); + result.reserve(6 + data_len); + + // MSGTYPE (2 bytes, big-endian) + result.append((uint8_t)((_msgtype >> 8) & 0xFF)); + result.append((uint8_t)(_msgtype & 0xFF)); + + // SEQUENCE (2 bytes, big-endian) + result.append((uint8_t)((_sequence >> 8) & 0xFF)); + result.append((uint8_t)(_sequence & 0xFF)); + + // LENGTH (2 bytes, big-endian) + result.append((uint8_t)((data_len >> 8) & 0xFF)); + result.append((uint8_t)(data_len & 0xFF)); + + // DATA + result += _raw; + + return result; + } + + // Unpack envelope from wire format (big-endian) + static bool unpack(const Bytes& wire_data, Envelope& out) { + // Need at least 6 bytes for header + if (wire_data.size() < 6) { + return false; + } + + const uint8_t* data = wire_data.data(); + + // MSGTYPE (2 bytes, big-endian) + uint16_t msgtype = (static_cast(data[0]) << 8) | data[1]; + + // SEQUENCE (2 bytes, big-endian) + uint16_t sequence = (static_cast(data[2]) << 8) | data[3]; + + // LENGTH (2 bytes, big-endian) + uint16_t length = (static_cast(data[4]) << 8) | data[5]; + + // Validate length + if (wire_data.size() < 6 + static_cast(length)) { + return false; + } + + // Extract data payload + Bytes raw; + if (length > 0) { + raw = wire_data.mid(6, length); + } + + out = Envelope(msgtype, sequence, raw); + return true; + } + +private: + uint16_t _msgtype = 0; + uint16_t _sequence = 0; + Bytes _raw; + Packet _packet = {Type::NONE}; + uint8_t _tries = 0; + double _timestamp = 0.0; + bool _tracked = false; + std::unique_ptr _message; +}; + +// Internal Channel data structure +class ChannelData { +public: + enum class WindowTier { FAST, MEDIUM, SLOW, VERY_SLOW }; + + // Fixed ring buffer sizes + static constexpr size_t RX_RING_SIZE = 16; + static constexpr size_t TX_RING_SIZE = 16; + + ChannelData() { MEM("ChannelData object created"); } + ChannelData(const Link& link) : _link(link) { MEM("ChannelData object created with link"); } + virtual ~ChannelData() { MEM("ChannelData object destroyed"); } + + // RX Ring buffer operations (ordered by sequence) + bool rx_ring_empty() const { return _rx_ring_count == 0; } + size_t rx_ring_size() const { return _rx_ring_count; } + bool rx_ring_full() const { return _rx_ring_count >= RX_RING_SIZE; } + + Envelope& rx_ring_front() { + return _rx_ring_pool[_rx_ring_head]; + } + + void rx_ring_pop_front() { + if (_rx_ring_count > 0) { + // Reset the envelope being removed + _rx_ring_pool[_rx_ring_head] = Envelope(); + _rx_ring_head = (_rx_ring_head + 1) % RX_RING_SIZE; + _rx_ring_count--; + } + } + + void rx_ring_clear() { + // Reset all envelopes + for (size_t i = 0; i < RX_RING_SIZE; i++) { + _rx_ring_pool[i] = Envelope(); + } + _rx_ring_head = 0; + _rx_ring_tail = 0; + _rx_ring_count = 0; + } + + // Insert envelope in sequence order (for reordering) + // Returns false if ring is full + bool rx_ring_insert_ordered(Envelope&& envelope) { + if (_rx_ring_count >= RX_RING_SIZE) { + return false; + } + + uint16_t new_seq = envelope.sequence(); + + // If empty, just add at head + if (_rx_ring_count == 0) { + _rx_ring_pool[_rx_ring_head] = std::move(envelope); + _rx_ring_tail = (_rx_ring_head + 1) % RX_RING_SIZE; + _rx_ring_count = 1; + return true; + } + + // Find insertion position by iterating through valid entries + // We need to shift elements to make room + size_t insert_pos = _rx_ring_count; // Default: insert at end + + for (size_t i = 0; i < _rx_ring_count; i++) { + size_t idx = (_rx_ring_head + i) % RX_RING_SIZE; + uint16_t existing_seq = _rx_ring_pool[idx].sequence(); + + // Calculate relative position (handling wraparound) + int32_t diff = static_cast(new_seq) - static_cast(existing_seq); + if (diff >= static_cast(Type::Channel::SEQ_MODULUS / 2)) { + diff -= Type::Channel::SEQ_MODULUS; + } else if (diff < -static_cast(Type::Channel::SEQ_MODULUS / 2)) { + diff += Type::Channel::SEQ_MODULUS; + } + + if (diff < 0) { + // Insert before this position + insert_pos = i; + break; + } + } + + // Shift elements from insert_pos to end to make room + // We shift by moving tail back and shifting elements + for (size_t i = _rx_ring_count; i > insert_pos; i--) { + size_t dst_idx = (_rx_ring_head + i) % RX_RING_SIZE; + size_t src_idx = (_rx_ring_head + i - 1) % RX_RING_SIZE; + _rx_ring_pool[dst_idx] = std::move(_rx_ring_pool[src_idx]); + } + + // Insert the new envelope + size_t actual_idx = (_rx_ring_head + insert_pos) % RX_RING_SIZE; + _rx_ring_pool[actual_idx] = std::move(envelope); + _rx_ring_tail = (_rx_ring_tail + 1) % RX_RING_SIZE; + _rx_ring_count++; + + return true; + } + + // Check if sequence exists in RX ring + bool rx_ring_contains_sequence(uint16_t sequence) const { + for (size_t i = 0; i < _rx_ring_count; i++) { + size_t idx = (_rx_ring_head + i) % RX_RING_SIZE; + if (_rx_ring_pool[idx].sequence() == sequence) { + return true; + } + } + return false; + } + + // TX Ring buffer operations (simple FIFO with removal by packet match) + bool tx_ring_empty() const { return _tx_ring_count == 0; } + size_t tx_ring_size() const { return _tx_ring_count; } + bool tx_ring_full() const { return _tx_ring_count >= TX_RING_SIZE; } + + bool tx_ring_push_back(Envelope&& envelope) { + if (_tx_ring_count >= TX_RING_SIZE) { + return false; + } + _tx_ring_pool[_tx_ring_tail] = std::move(envelope); + _tx_ring_tail = (_tx_ring_tail + 1) % TX_RING_SIZE; + _tx_ring_count++; + return true; + } + + void tx_ring_clear() { + for (size_t i = 0; i < TX_RING_SIZE; i++) { + _tx_ring_pool[i] = Envelope(); + } + _tx_ring_head = 0; + _tx_ring_tail = 0; + _tx_ring_count = 0; + } + + // Remove envelope matching packet, returns true if found and removed + bool tx_ring_remove_by_packet(const Packet& packet) { + for (size_t i = 0; i < _tx_ring_count; i++) { + size_t idx = (_tx_ring_head + i) % TX_RING_SIZE; + if (_tx_ring_pool[idx].packet() == packet) { + // Shift remaining elements forward + for (size_t j = i; j < _tx_ring_count - 1; j++) { + size_t dst_idx = (_tx_ring_head + j) % TX_RING_SIZE; + size_t src_idx = (_tx_ring_head + j + 1) % TX_RING_SIZE; + _tx_ring_pool[dst_idx] = std::move(_tx_ring_pool[src_idx]); + } + // Clear the last slot + size_t last_idx = (_tx_ring_head + _tx_ring_count - 1) % TX_RING_SIZE; + _tx_ring_pool[last_idx] = Envelope(); + _tx_ring_count--; + return true; + } + } + return false; + } + + // Find envelope by packet (returns nullptr if not found) + Envelope* tx_ring_find_by_packet(const Packet& packet) { + for (size_t i = 0; i < _tx_ring_count; i++) { + size_t idx = (_tx_ring_head + i) % TX_RING_SIZE; + if (_tx_ring_pool[idx].packet() == packet) { + return &_tx_ring_pool[idx]; + } + } + return nullptr; + } + + // Find envelope by sequence (returns nullptr if not found) + Envelope* tx_ring_find_by_sequence(uint16_t sequence) { + for (size_t i = 0; i < _tx_ring_count; i++) { + size_t idx = (_tx_ring_head + i) % TX_RING_SIZE; + if (_tx_ring_pool[idx].sequence() == sequence) { + return &_tx_ring_pool[idx]; + } + } + return nullptr; + } + + // Iterate over TX ring (for counting outstanding, checking timeouts) + template + void tx_ring_foreach(Func&& func) { + for (size_t i = 0; i < _tx_ring_count; i++) { + size_t idx = (_tx_ring_head + i) % TX_RING_SIZE; + func(_tx_ring_pool[idx]); + } + } + + template + void tx_ring_foreach(Func&& func) const { + for (size_t i = 0; i < _tx_ring_count; i++) { + size_t idx = (_tx_ring_head + i) % TX_RING_SIZE; + func(_tx_ring_pool[idx]); + } + } + +private: + friend class Channel; + + // Link reference + Link _link = {Type::NONE}; + + // Sequencing + uint16_t _next_sequence = 0; + uint16_t _next_rx_sequence = 0; + + // RX Ring buffer (fixed-size circular buffer, ordered by sequence) + Envelope _rx_ring_pool[RX_RING_SIZE]; + size_t _rx_ring_head = 0; + size_t _rx_ring_tail = 0; + size_t _rx_ring_count = 0; + + // TX Ring buffer (fixed-size circular buffer) + Envelope _tx_ring_pool[TX_RING_SIZE]; + size_t _tx_ring_head = 0; + size_t _tx_ring_tail = 0; + size_t _tx_ring_count = 0; + + // Message dispatch + // Factory: msgtype -> function that creates a new message instance + // Note: These are only set up once at initialization, so map is acceptable + std::map()>> _message_factories; + // Handlers: list of callbacks, first returning true stops dispatch + // Note: These are only set up once at initialization, so vector is acceptable + std::vector> _message_callbacks; + + // Window management + uint16_t _window = Type::Channel::WINDOW_INITIAL; + uint16_t _window_min = Type::Channel::WINDOW_MIN; + uint16_t _window_max = Type::Channel::WINDOW_MAX; + uint16_t _fast_rate_rounds = 0; + + // Timing/RTT + double _rtt = 0.0; + uint8_t _max_tries = Type::Channel::MAX_TRIES; + WindowTier _current_tier = WindowTier::MEDIUM; + + // State + bool _ready = false; +}; + +} // namespace RNS + +#endif // RNS_CHANNEL_DATA_H diff --git a/src-shim/Cryptography/BZ2.cpp b/src-shim/Cryptography/BZ2.cpp new file mode 100644 index 00000000..718429e9 --- /dev/null +++ b/src-shim/Cryptography/BZ2.cpp @@ -0,0 +1,179 @@ +#include "BZ2.h" + +#if defined(NATIVE) +#include +#elif defined(ARDUINO) +#include "bzlib.h" +#endif + +#include +#include +#include + +#ifdef ARDUINO +#include +#endif + +namespace RNS { namespace Cryptography { + +const Bytes bz2_decompress(const Bytes& data) { +#if defined(NATIVE) || defined(ARDUINO) + if (data.empty()) { + return Bytes(); + } + +#ifdef ARDUINO + // ESP32: Use smaller buffers to fit in memory, grow if needed + // Start with 64KB and grow up to 512KB max + const size_t MIN_OUTPUT_SIZE = 64 * 1024; // 64KB initial + const size_t MAX_OUTPUT_SIZE = 512 * 1024; // 512KB max +#else + // Native: Use larger buffers for efficiency + const size_t MIN_OUTPUT_SIZE = 2 * 1024 * 1024; // 2MB minimum + const size_t MAX_OUTPUT_SIZE = 16 * 1024 * 1024; // 16MB max +#endif + size_t output_size = std::min(std::max(data.size() * 100, MIN_OUTPUT_SIZE), MAX_OUTPUT_SIZE); + +#ifdef ARDUINO + // Allocate from PSRAM if available + char* output = (char*)heap_caps_malloc(output_size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (!output) { + // Fall back to regular heap + output = (char*)malloc(output_size); + } + if (!output) { + ERROR("bz2_decompress: Failed to allocate output buffer"); + return Bytes(); + } +#else + std::vector output_vec(output_size); + char* output = output_vec.data(); +#endif + + bz_stream stream; + memset(&stream, 0, sizeof(stream)); + + int ret = BZ2_bzDecompressInit(&stream, 0, 0); + if (ret != BZ_OK) { +#ifdef ARDUINO + free(output); +#endif + return Bytes(); + } + + stream.next_in = const_cast(reinterpret_cast(data.data())); + stream.avail_in = data.size(); + stream.next_out = output; + stream.avail_out = output_size; + + Bytes result; + int iteration = 0; + do { + ret = BZ2_bzDecompress(&stream); + iteration++; + + if (ret == BZ_OK || ret == BZ_STREAM_END) { + // Append decompressed data + size_t decompressed = output_size - stream.avail_out; + result.append(reinterpret_cast(output), decompressed); + + DEBUGF("bz2_decompress: iter=%d, ret=%d, decompressed=%zu, total=%zu, avail_in=%u", + iteration, ret, decompressed, result.size(), stream.avail_in); + + if (ret == BZ_STREAM_END) { + break; + } + + // Reset output buffer for more data + stream.next_out = output; + stream.avail_out = output_size; + } else { + DEBUGF("bz2_decompress: iter=%d, FAILED ret=%d", iteration, ret); + BZ2_bzDecompressEnd(&stream); +#ifdef ARDUINO + free(output); +#endif + return Bytes(); + } + } while (stream.avail_in > 0 || ret != BZ_STREAM_END); + + BZ2_bzDecompressEnd(&stream); +#ifdef ARDUINO + free(output); +#endif + DEBUGF("bz2_decompress: final output size=%zu", result.size()); + return result; +#else + ERROR("bz2_decompress: BZ2 support not available on this platform"); + return Bytes(); +#endif +} + +const Bytes bz2_compress(const Bytes& data) { +#if defined(NATIVE) || defined(ARDUINO) + if (data.empty()) { + return Bytes(); + } + + // Compressed size is at most input size + 1% + 600 bytes + size_t output_size = data.size() + data.size() / 100 + 600; + +#ifdef ARDUINO + // Allocate from PSRAM if available + char* output = (char*)heap_caps_malloc(output_size, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (!output) { + output = (char*)malloc(output_size); + } + if (!output) { + ERROR("bz2_compress: Failed to allocate output buffer"); + return Bytes(); + } + // Use smaller block size (1 = 100k) on ESP32 to reduce memory usage + const int block_size = 1; +#else + std::vector output_vec(output_size); + char* output = output_vec.data(); + // Use largest block size (9 = 900k) on native for best compression + const int block_size = 9; +#endif + + bz_stream stream; + memset(&stream, 0, sizeof(stream)); + + int ret = BZ2_bzCompressInit(&stream, block_size, 0, 0); + if (ret != BZ_OK) { +#ifdef ARDUINO + free(output); +#endif + return Bytes(); + } + + stream.next_in = const_cast(reinterpret_cast(data.data())); + stream.avail_in = data.size(); + stream.next_out = output; + stream.avail_out = output_size; + + ret = BZ2_bzCompress(&stream, BZ_FINISH); + if (ret != BZ_STREAM_END) { + BZ2_bzCompressEnd(&stream); +#ifdef ARDUINO + free(output); +#endif + return Bytes(); + } + + size_t compressed_size = output_size - stream.avail_out; + BZ2_bzCompressEnd(&stream); + + Bytes result(reinterpret_cast(output), compressed_size); +#ifdef ARDUINO + free(output); +#endif + return result; +#else + ERROR("bz2_compress: BZ2 support not available on this platform"); + return Bytes(); +#endif +} + +} } diff --git a/src-shim/Cryptography/BZ2.h b/src-shim/Cryptography/BZ2.h new file mode 100644 index 00000000..2050d594 --- /dev/null +++ b/src-shim/Cryptography/BZ2.h @@ -0,0 +1,10 @@ +#pragma once + +#include + +namespace RNS { namespace Cryptography { + + const Bytes bz2_decompress(const Bytes& data); + const Bytes bz2_compress(const Bytes& data); + +} } diff --git a/src-shim/Cryptography/Ratchet.cpp b/src-shim/Cryptography/Ratchet.cpp new file mode 100644 index 00000000..fed34215 --- /dev/null +++ b/src-shim/Cryptography/Ratchet.cpp @@ -0,0 +1,193 @@ +#include "Ratchet.h" +#include "Fernet.h" +#include + +#include +#include + +#include +#include + +using namespace RNS; +using namespace RNS::Cryptography; + +// Constructor from existing key material +Ratchet::Ratchet(const Bytes& private_key, const Bytes& public_key, double created_at) + : _private_key(private_key), _public_key(public_key), _created_at(created_at) +{ + if (private_key.size() != RATCHET_LENGTH) { + throw std::invalid_argument("Ratchet private key must be exactly 32 bytes"); + } + if (public_key.size() != RATCHET_LENGTH) { + throw std::invalid_argument("Ratchet public key must be exactly 32 bytes"); + } + if (_created_at == 0.0) { + _created_at = Utilities::OS::time(); + } +} + +// Copy constructor +Ratchet::Ratchet(const Ratchet& ratchet) + : _private_key(ratchet._private_key) + , _public_key(ratchet._public_key) + , _created_at(ratchet._created_at) +{ +} + +// Assignment operator +Ratchet& Ratchet::operator=(const Ratchet& ratchet) { + if (this != &ratchet) { + _private_key = ratchet._private_key; + _public_key = ratchet._public_key; + _created_at = ratchet._created_at; + } + return *this; +} + +// Generate a new ratchet with fresh X25519 keypair +Ratchet Ratchet::generate() { + Bytes private_key; + Bytes public_key; + + // Generate random X25519 keypair using Curve25519::dh1 + // This matches X25519PrivateKey::generate() pattern + Curve25519::dh1(public_key.writable(RATCHET_LENGTH), private_key.writable(RATCHET_LENGTH)); + + double created_at = Utilities::OS::time(); + + TRACE("Ratchet::generate: Generated new ratchet"); + DEBUG(" Private key: " + private_key.toHex()); + DEBUG(" Public key: " + public_key.toHex()); + DEBUG(" Created at: " + std::to_string(created_at)); + + return Ratchet(private_key, public_key, created_at); +} + +// Derive ratchet ID from public key bytes +Bytes Ratchet::get_ratchet_id(const Bytes& public_bytes) { + if (public_bytes.size() != RATCHET_LENGTH) { + throw std::invalid_argument("Ratchet public key must be exactly 32 bytes"); + } + + // Ratchet ID is first 10 bytes of SHA-256(public_key) + // This matches Python implementation + Bytes hash = sha256(public_bytes); + Bytes ratchet_id = hash.left(RATCHET_ID_LENGTH); + + DEBUG("Ratchet::get_ratchet_id: " + ratchet_id.toHex() + " from pubkey " + public_bytes.toHex()); + + return ratchet_id; +} + +// Derive ratchet ID from X25519PublicKey object +Bytes Ratchet::get_ratchet_id(X25519PublicKey& public_key) { + return get_ratchet_id(public_key.public_bytes()); +} + +// Get public key bytes for announcement +Bytes Ratchet::public_bytes() const { + return _public_key; +} + +// Get private key bytes (use with caution!) +Bytes Ratchet::private_bytes() const { + return _private_key; +} + +// Get ratchet ID for this ratchet +Bytes Ratchet::get_id() const { + return get_ratchet_id(_public_key); +} + +// Derive shared secret with peer's ratchet +Bytes Ratchet::derive_shared_secret(const Bytes& peer_public_key) const { + if (!_private_key || !_public_key) { + throw std::runtime_error("Cannot derive shared secret from empty ratchet"); + } + if (peer_public_key.size() != RATCHET_LENGTH) { + throw std::invalid_argument("Peer public key must be exactly 32 bytes"); + } + + // Perform X25519 ECDH: shared = ECDH(my_private, peer_public) + // This uses Curve25519::eval() like X25519PrivateKey::exchange() + Bytes shared_secret; + if (!Curve25519::eval(shared_secret.writable(RATCHET_LENGTH), _private_key.data(), peer_public_key.data())) { + throw std::runtime_error("Peer ratchet key is invalid"); + } + + DEBUG("Ratchet::derive_shared_secret:"); + DEBUG(" My private: " + _private_key.toHex()); + DEBUG(" My public: " + _public_key.toHex()); + DEBUG(" Peer public: " + peer_public_key.toHex()); + DEBUG(" Shared: " + shared_secret.toHex()); + + return shared_secret; +} + +// Derive encryption/decryption key from shared secret +Bytes Ratchet::derive_key(const Bytes& shared_secret) const { + if (shared_secret.size() != RATCHET_LENGTH) { + throw std::invalid_argument("Shared secret must be exactly 32 bytes"); + } + + // Use HKDF to derive a Fernet-compatible key from the shared secret + // Fernet requires a 32-byte key (256 bits) + // Python implementation uses HKDF with no salt or context for ratchet keys + Bytes derived_key = hkdf(32, shared_secret); + + DEBUG("Ratchet::derive_key:"); + DEBUG(" Shared secret: " + shared_secret.toHex()); + DEBUG(" Derived key: " + derived_key.toHex()); + + return derived_key; +} + +// Encrypt plaintext using this ratchet and peer's public key +Bytes Ratchet::encrypt(const Bytes& plaintext, const Bytes& peer_public_key) const { + if (!_private_key || !_public_key) { + throw std::runtime_error("Cannot encrypt with empty ratchet"); + } + + // 1. Perform X25519 ECDH to get shared secret + Bytes shared_secret = derive_shared_secret(peer_public_key); + + // 2. Derive encryption key using HKDF + Bytes encryption_key = derive_key(shared_secret); + + // 3. Encrypt with Fernet + Fernet fernet(encryption_key); + Bytes ciphertext = fernet.encrypt(plaintext); + + DEBUG("Ratchet::encrypt: Encrypted " + std::to_string(plaintext.size()) + + " bytes to " + std::to_string(ciphertext.size()) + " bytes"); + + return ciphertext; +} + +// Decrypt ciphertext using this ratchet and peer's public key +Bytes Ratchet::decrypt(const Bytes& ciphertext, const Bytes& peer_public_key) const { + if (!_private_key || !_public_key) { + throw std::runtime_error("Cannot decrypt with empty ratchet"); + } + + try { + // 1. Perform X25519 ECDH to get shared secret + Bytes shared_secret = derive_shared_secret(peer_public_key); + + // 2. Derive decryption key using HKDF + Bytes decryption_key = derive_key(shared_secret); + + // 3. Decrypt with Fernet + Fernet fernet(decryption_key); + Bytes plaintext = fernet.decrypt(ciphertext); + + DEBUG("Ratchet::decrypt: Decrypted " + std::to_string(ciphertext.size()) + + " bytes to " + std::to_string(plaintext.size()) + " bytes"); + + return plaintext; + } + catch (const std::exception& e) { + ERROR("Ratchet::decrypt failed: " + std::string(e.what())); + throw; + } +} diff --git a/src-shim/Cryptography/Ratchet.h b/src-shim/Cryptography/Ratchet.h new file mode 100644 index 00000000..e3f89cca --- /dev/null +++ b/src-shim/Cryptography/Ratchet.h @@ -0,0 +1,181 @@ +#pragma once + +#include +#include +#include +#include "X25519.h" +#include "HKDF.h" +#include "Fernet.h" +#include "Hashes.h" + +#include +#include + +namespace RNS { namespace Cryptography { + + /** + * @brief Represents a single ratchet (X25519 keypair) used for forward secrecy + * + * Ratchets provide forward secrecy by rotating encryption keys at regular intervals. + * Each ratchet consists of an X25519 keypair. The public key is announced, while + * the private key is used to derive shared secrets with peer ratchets. + */ + class Ratchet { + + public: + using Ptr = std::shared_ptr; + + // Constants from Python implementation + static const uint8_t RATCHET_LENGTH = 32; // X25519 key length + static const uint8_t RATCHET_ID_LENGTH = 10; // Truncated hash length + static const uint16_t MAX_RATCHETS = 128; // Maximum ratchets to store + static const uint32_t DEFAULT_RATCHET_INTERVAL = 1800; // 30 minutes in seconds + + public: + /** + * @brief Default constructor creates an empty ratchet + */ + Ratchet() : _created_at(0.0) {} + + /** + * @brief Constructs a ratchet from existing key material + * @param private_key The X25519 private key (32 bytes) + * @param public_key The X25519 public key (32 bytes) + * @param created_at Timestamp when ratchet was created + */ + Ratchet(const Bytes& private_key, const Bytes& public_key, double created_at = 0.0); + + /** + * @brief Copy constructor + */ + Ratchet(const Ratchet& ratchet); + + /** + * @brief Assignment operator + */ + Ratchet& operator=(const Ratchet& ratchet); + + /** + * @brief Destructor + */ + ~Ratchet() {} + + /** + * @brief Validity check + */ + inline operator bool() const { + return _private_key && _public_key; + } + + public: + /** + * @brief Generates a new ratchet with fresh X25519 keypair + * @return A new Ratchet instance + */ + static Ratchet generate(); + + /** + * @brief Derives a ratchet ID from public key bytes + * + * The ratchet ID is used to identify which ratchet was used to encrypt a packet. + * It's computed as the first 10 bytes of SHA-256(public_key). + * + * @param public_bytes The X25519 public key (32 bytes) + * @return Ratchet ID (10 bytes) + */ + static Bytes get_ratchet_id(const Bytes& public_bytes); + + /** + * @brief Derives a ratchet ID from an X25519PublicKey object + * @param public_key The X25519 public key object + * @return Ratchet ID (10 bytes) + */ + static Bytes get_ratchet_id(X25519PublicKey& public_key); + + public: + /** + * @brief Gets the public key bytes for announcement + * @return X25519 public key (32 bytes) + */ + Bytes public_bytes() const; + + /** + * @brief Gets the private key bytes (use with caution!) + * @return X25519 private key (32 bytes) + */ + Bytes private_bytes() const; + + /** + * @brief Gets the ratchet ID for this ratchet + * @return Ratchet ID (10 bytes) + */ + Bytes get_id() const; + + /** + * @brief Gets the creation timestamp + * @return Unix timestamp (seconds since epoch) + */ + double created_at() const { return _created_at; } + + /** + * @brief Sets the creation timestamp + * @param timestamp Unix timestamp + */ + void set_created_at(double timestamp) { _created_at = timestamp; } + + /** + * @brief Encrypts plaintext using this ratchet and peer's public key + * + * Process: + * 1. Perform X25519 ECDH: shared = ECDH(my_private, peer_public) + * 2. Derive encryption key: key = HKDF(shared, ...) + * 3. Encrypt with Fernet: ciphertext = Fernet.encrypt(key, plaintext) + * + * @param plaintext The data to encrypt + * @param peer_public_key The peer's X25519 public key (32 bytes) + * @return Encrypted ciphertext + */ + Bytes encrypt(const Bytes& plaintext, const Bytes& peer_public_key) const; + + /** + * @brief Decrypts ciphertext using this ratchet and peer's public key + * + * Process: + * 1. Perform X25519 ECDH: shared = ECDH(my_private, peer_public) + * 2. Derive decryption key: key = HKDF(shared, ...) + * 3. Decrypt with Fernet: plaintext = Fernet.decrypt(key, ciphertext) + * + * @param ciphertext The encrypted data + * @param peer_public_key The peer's X25519 public key (32 bytes) + * @return Decrypted plaintext + */ + Bytes decrypt(const Bytes& ciphertext, const Bytes& peer_public_key) const; + + /** + * @brief Derives a shared secret with peer's ratchet + * + * Performs X25519 Elliptic Curve Diffie-Hellman key exchange. + * + * @param peer_public_key The peer's X25519 public key (32 bytes) + * @return Shared secret (32 bytes) + */ + Bytes derive_shared_secret(const Bytes& peer_public_key) const; + + /** + * @brief Derives an encryption/decryption key from a shared secret + * + * Uses HKDF (HMAC-based Key Derivation Function) to derive a key suitable + * for Fernet encryption from the X25519 shared secret. + * + * @param shared_secret The result of X25519 ECDH (32 bytes) + * @return Derived encryption key (32 bytes for Fernet) + */ + Bytes derive_key(const Bytes& shared_secret) const; + + private: + Bytes _private_key; // X25519 private key (32 bytes) + Bytes _public_key; // X25519 public key (32 bytes) + double _created_at; // Unix timestamp + }; + +} } diff --git a/src-shim/Display.cpp b/src-shim/Display.cpp new file mode 100644 index 00000000..c2141eb6 --- /dev/null +++ b/src-shim/Display.cpp @@ -0,0 +1,411 @@ +/* + * Display.cpp - OLED display implementation for microReticulum + */ + +#include "Display.h" + +#ifdef HAS_DISPLAY + +#include "DisplayGraphics.h" +#include "Identity.h" +#include "Interface.h" +#include "Reticulum.h" +#include "Transport.h" +#include "Log.h" +#include "Utilities/OS.h" + +// Display library includes (board-specific) +#ifdef ARDUINO + #include + #ifdef DISPLAY_TYPE_SH1106 + #include + #elif defined(DISPLAY_TYPE_SSD1306) + #include + #endif + #include +#endif + +namespace RNS { + +// Display dimensions +static const int16_t DISPLAY_WIDTH = 128; +static const int16_t DISPLAY_HEIGHT = 64; + +// Layout constants +static const int16_t HEADER_HEIGHT = 17; +static const int16_t CONTENT_Y = 21; +static const int16_t LINE_HEIGHT = 10; +static const int16_t LEFT_MARGIN = 2; + +// Static member initialization +bool Display::_ready = false; +bool Display::_blanked = false; +uint8_t Display::_current_page = 0; +uint32_t Display::_last_page_flip = 0; +uint32_t Display::_last_update = 0; +uint32_t Display::_start_time = 0; +Bytes Display::_identity_hash; +Interface* Display::_interface = nullptr; +Reticulum* Display::_reticulum = nullptr; +float Display::_rssi = -120.0f; + +// Display object (board-specific) +#ifdef ARDUINO + #ifdef DISPLAY_TYPE_SH1106 + static Adafruit_SH1106G* display = nullptr; + #elif defined(DISPLAY_TYPE_SSD1306) + static Adafruit_SSD1306* display = nullptr; + #endif +#endif + +bool Display::init() { +#ifdef ARDUINO + TRACE("Display::init: Initializing display..."); + + // Initialize I2C + #if defined(DISPLAY_SDA) && defined(DISPLAY_SCL) + Wire.begin(DISPLAY_SDA, DISPLAY_SCL); + #else + Wire.begin(); + #endif + + // Create display object + #ifdef DISPLAY_TYPE_SH1106 + display = new Adafruit_SH1106G(DISPLAY_WIDTH, DISPLAY_HEIGHT, &Wire, -1); + #ifndef DISPLAY_ADDR + #define DISPLAY_ADDR 0x3C + #endif + if (!display->begin(DISPLAY_ADDR, true)) { + ERROR("Display::init: SH1106 display not found"); + delete display; + display = nullptr; + return false; + } + #elif defined(DISPLAY_TYPE_SSD1306) + display = new Adafruit_SSD1306(DISPLAY_WIDTH, DISPLAY_HEIGHT, &Wire, -1); + #ifndef DISPLAY_ADDR + #define DISPLAY_ADDR 0x3C + #endif + if (!display->begin(SSD1306_SWITCHCAPVCC, DISPLAY_ADDR)) { + ERROR("Display::init: SSD1306 display not found"); + delete display; + display = nullptr; + return false; + } + #else + ERROR("Display::init: No display type defined"); + return false; + #endif + + // Configure display + display->setRotation(0); // Portrait mode + display->clearDisplay(); + display->setTextSize(1); + display->setTextColor(1); // White + display->cp437(true); // Enable extended characters + display->display(); + + _ready = true; + _start_time = (uint32_t)Utilities::OS::ltime(); + _last_page_flip = _start_time; + _last_update = _start_time; + + INFO("Display::init: Display initialized successfully"); + return true; +#else + // Native build - no display support + return false; +#endif +} + +void Display::update() { + if (!_ready || _blanked) return; + +#ifdef ARDUINO + uint32_t now = (uint32_t)Utilities::OS::ltime(); + + // Check for page rotation + if (now - _last_page_flip >= PAGE_INTERVAL) { + _current_page = (_current_page + 1) % NUM_PAGES; + _last_page_flip = now; + } + + // Throttle display updates + if (now - _last_update < UPDATE_INTERVAL) return; + _last_update = now; + + // Clear and redraw + display->clearDisplay(); + + // Draw header (common to all pages) + draw_header(); + + // Draw page-specific content + switch (_current_page) { + case 0: + draw_page_main(); + break; + case 1: + draw_page_interface(); + break; + case 2: + draw_page_network(); + break; + } + + // Send to display + display->display(); +#endif +} + +void Display::set_identity(const Identity& identity) { + if (identity) { + _identity_hash = identity.hash(); + } +} + +void Display::set_interface(Interface* iface) { + _interface = iface; +} + +void Display::set_reticulum(Reticulum* rns) { + _reticulum = rns; +} + +void Display::blank(bool blank) { + _blanked = blank; +#ifdef ARDUINO + if (_ready && display) { + if (blank) { + display->clearDisplay(); + display->display(); + } + } +#endif +} + +void Display::set_page(uint8_t page) { + if (page < NUM_PAGES) { + _current_page = page; + _last_page_flip = (uint32_t)Utilities::OS::ltime(); + } +} + +void Display::next_page() { + _current_page = (_current_page + 1) % NUM_PAGES; + _last_page_flip = (uint32_t)Utilities::OS::ltime(); +} + +uint8_t Display::current_page() { + return _current_page; +} + +bool Display::ready() { + return _ready; +} + +void Display::set_rssi(float rssi) { + _rssi = rssi; +} + +// Private implementation + +void Display::draw_header() { +#ifdef ARDUINO + // Draw "μRNS" text logo + display->setTextSize(2); // 2x size for visibility + display->setCursor(3, 0); + // Use CP437 code 230 (0xE6) for μ character + display->write(0xE6); // μ + display->print("RNS"); + display->setTextSize(1); // Reset to normal size + + // Draw signal bars on the right + draw_signal_bars(DISPLAY_WIDTH - Graphics::SIGNAL_WIDTH - 2, 4); + + // Draw separator line + display->drawLine(0, HEADER_HEIGHT, DISPLAY_WIDTH - 1, HEADER_HEIGHT, 1); +#endif +} + +void Display::draw_signal_bars(int16_t x, int16_t y) { +#ifdef ARDUINO + // Temporarily disabled to debug stray pixel + // uint8_t level = 0; + // if (_interface && _interface->online()) { + // level = Graphics::rssi_to_level(_rssi); + // } + // const uint8_t* bitmap = Graphics::get_signal_bitmap(level); + // display->drawBitmap(x, y, bitmap, Graphics::SIGNAL_WIDTH, Graphics::SIGNAL_HEIGHT, 1); +#endif +} + +void Display::draw_page_main() { +#ifdef ARDUINO + int16_t y = CONTENT_Y; + + // Identity hash + display->setCursor(LEFT_MARGIN, y); + display->print("ID: "); + if (_identity_hash.size() > 0) { + // Show first 12 hex chars + std::string hex = _identity_hash.toHex(); + if (hex.length() > 12) hex = hex.substr(0, 12); + display->print(hex.c_str()); + } else { + display->print("(none)"); + } + y += LINE_HEIGHT + 4; + + // Interface status + display->setCursor(LEFT_MARGIN, y); + if (_interface) { + display->print(_interface->name().c_str()); + display->print(": "); + display->print(_interface->online() ? "ONLINE" : "OFFLINE"); + } else { + display->print("No interface"); + } + y += LINE_HEIGHT; + + // Link count + display->setCursor(LEFT_MARGIN, y); + display->print("Links: "); + if (_reticulum) { + display->print((int)_reticulum->get_link_count()); + } else { + display->print("0"); + } +#endif +} + +void Display::draw_page_interface() { +#ifdef ARDUINO + int16_t y = CONTENT_Y; + + if (!_interface) { + display->setCursor(LEFT_MARGIN, y); + display->print("No interface"); + return; + } + + // Interface name + display->setCursor(LEFT_MARGIN, y); + display->print("Interface: "); + display->print(_interface->name().c_str()); + y += LINE_HEIGHT; + + // Mode + display->setCursor(LEFT_MARGIN, y); + display->print("Mode: "); + switch (_interface->mode()) { + case Type::Interface::MODE_GATEWAY: + display->print("Gateway"); + break; + case Type::Interface::MODE_ACCESS_POINT: + display->print("Access Point"); + break; + case Type::Interface::MODE_POINT_TO_POINT: + display->print("Point-to-Point"); + break; + case Type::Interface::MODE_ROAMING: + display->print("Roaming"); + break; + case Type::Interface::MODE_BOUNDARY: + display->print("Boundary"); + break; + default: + display->print("Unknown"); + break; + } + y += LINE_HEIGHT; + + // Bitrate + display->setCursor(LEFT_MARGIN, y); + display->print("Bitrate: "); + display->print(format_bitrate(_interface->bitrate()).c_str()); + y += LINE_HEIGHT; + + // Status + display->setCursor(LEFT_MARGIN, y); + display->print("Status: "); + display->print(_interface->online() ? "Online" : "Offline"); +#endif +} + +void Display::draw_page_network() { +#ifdef ARDUINO + int16_t y = CONTENT_Y; + + // Links and paths + display->setCursor(LEFT_MARGIN, y); + display->print("Links: "); + size_t link_count = _reticulum ? _reticulum->get_link_count() : 0; + display->print((int)link_count); + + display->print(" Paths: "); + size_t path_count = _reticulum ? _reticulum->get_path_table().size() : 0; + display->print((int)path_count); + y += LINE_HEIGHT; + + // RTT placeholder (would need link reference to get actual RTT) + display->setCursor(LEFT_MARGIN, y); + display->print("RTT: --"); + y += LINE_HEIGHT; + + // TX/RX bytes (if interface available) + display->setCursor(LEFT_MARGIN, y); + display->print("TX/RX: --"); + y += LINE_HEIGHT; + + // Uptime + display->setCursor(LEFT_MARGIN, y); + display->print("Uptime: "); + uint32_t uptime_sec = ((uint32_t)Utilities::OS::ltime() - _start_time) / 1000; + display->print(format_time(uptime_sec).c_str()); +#endif +} + +std::string Display::format_bytes(size_t bytes) { + char buf[16]; + if (bytes >= 1024 * 1024) { + snprintf(buf, sizeof(buf), "%.1fM", bytes / (1024.0 * 1024.0)); + } else if (bytes >= 1024) { + snprintf(buf, sizeof(buf), "%.1fK", bytes / 1024.0); + } else { + snprintf(buf, sizeof(buf), "%zuB", bytes); + } + return std::string(buf); +} + +std::string Display::format_time(uint32_t seconds) { + char buf[16]; + if (seconds >= 3600) { + uint32_t hours = seconds / 3600; + uint32_t mins = (seconds % 3600) / 60; + snprintf(buf, sizeof(buf), "%luh %lum", (unsigned long)hours, (unsigned long)mins); + } else if (seconds >= 60) { + uint32_t mins = seconds / 60; + uint32_t secs = seconds % 60; + snprintf(buf, sizeof(buf), "%lum %lus", (unsigned long)mins, (unsigned long)secs); + } else { + snprintf(buf, sizeof(buf), "%lus", (unsigned long)seconds); + } + return std::string(buf); +} + +std::string Display::format_bitrate(uint32_t bps) { + char buf[16]; + if (bps >= 1000000) { + snprintf(buf, sizeof(buf), "%.1f Mbps", bps / 1000000.0); + } else if (bps >= 1000) { + snprintf(buf, sizeof(buf), "%.1f kbps", bps / 1000.0); + } else { + snprintf(buf, sizeof(buf), "%lu bps", (unsigned long)bps); + } + return std::string(buf); +} + +} // namespace RNS + +#endif // HAS_DISPLAY diff --git a/src-shim/Display.h b/src-shim/Display.h new file mode 100644 index 00000000..644caf2e --- /dev/null +++ b/src-shim/Display.h @@ -0,0 +1,139 @@ +/* + * Display.h - OLED display support for microReticulum + * + * Provides status display on supported hardware (T-Beam Supreme, etc.) + * with auto-rotating pages showing identity, interface, and network info. + */ + +#pragma once + +#include "Type.h" +#include "Bytes.h" + +#include +#include + +// Only compile display code if enabled +#ifdef HAS_DISPLAY + +namespace RNS { + +// Forward declarations +class Identity; +class Interface; +class Reticulum; + +class Display { +public: + // Number of pages to rotate through + static const uint8_t NUM_PAGES = 3; + + // Page interval in milliseconds + static const uint32_t PAGE_INTERVAL = 4000; + + // Display update interval (~7 FPS) + static const uint32_t UPDATE_INTERVAL = 143; + +public: + /** + * Initialize the display hardware. + * Must be called once during setup. + * @return true if display initialized successfully + */ + static bool init(); + + /** + * Update the display. Call this frequently in the main loop. + * Handles page rotation and display refresh internally. + */ + static void update(); + + /** + * Set the identity to display. + * @param identity The identity whose hash will be shown + */ + static void set_identity(const Identity& identity); + + /** + * Set the primary interface to display status for. + * @param iface Pointer to the interface (can be nullptr) + */ + static void set_interface(Interface* iface); + + /** + * Set the Reticulum instance for network statistics. + * @param rns Pointer to the Reticulum instance + */ + static void set_reticulum(Reticulum* rns); + + /** + * Enable or disable display blanking (power save). + * @param blank true to blank the display + */ + static void blank(bool blank); + + /** + * Set the current page manually. + * @param page Page number (0 to NUM_PAGES-1) + */ + static void set_page(uint8_t page); + + /** + * Advance to the next page. + */ + static void next_page(); + + /** + * Get the current page number. + * @return Current page (0 to NUM_PAGES-1) + */ + static uint8_t current_page(); + + /** + * Check if display is ready. + * @return true if display was initialized successfully + */ + static bool ready(); + + /** + * Set RSSI value for signal bars display. + * @param rssi Signal strength in dBm + */ + static void set_rssi(float rssi); + +private: + // Page rendering functions + static void draw_page_main(); // Page 0: Main status + static void draw_page_interface(); // Page 1: Interface details + static void draw_page_network(); // Page 2: Network info + + // Common elements + static void draw_header(); // Logo + signal bars (all pages) + static void draw_signal_bars(int16_t x, int16_t y); + + // Helper functions + static std::string format_bytes(size_t bytes); + static std::string format_time(uint32_t seconds); + static std::string format_bitrate(uint32_t bps); + +private: + // State + static bool _ready; + static bool _blanked; + static uint8_t _current_page; + static uint32_t _last_page_flip; + static uint32_t _last_update; + static uint32_t _start_time; + + // Data sources + static Bytes _identity_hash; + static Interface* _interface; + static Reticulum* _reticulum; + + // Signal strength + static float _rssi; +}; + +} // namespace RNS + +#endif // HAS_DISPLAY diff --git a/src-shim/DisplayGraphics.h b/src-shim/DisplayGraphics.h new file mode 100644 index 00000000..84b4ea74 --- /dev/null +++ b/src-shim/DisplayGraphics.h @@ -0,0 +1,226 @@ +/* + * DisplayGraphics.h - Bitmap graphics for microReticulum display + * + * Contains the μRNS logo and status icons stored in PROGMEM. + * Bitmaps are in XBM format (1 bit per pixel, LSB first). + */ + +#ifndef DISPLAY_GRAPHICS_H +#define DISPLAY_GRAPHICS_H + +#include + +#ifdef ARDUINO + #ifdef ESP32 + #include + #elif defined(__AVR__) + #include + #else + #define PROGMEM + #endif +#else + #define PROGMEM +#endif + +namespace RNS { +namespace Graphics { + +// μRNS Logo - 40x12 pixels +// Displays "μRNS" text +static const uint8_t LOGO_WIDTH = 40; +static const uint8_t LOGO_HEIGHT = 12; +static const uint8_t logo_urns[] PROGMEM = { + // Row 0 + 0x00, 0x00, 0x00, 0x00, 0x00, + // Row 1 - top of letters + 0x00, 0x00, 0xFC, 0x0F, 0x00, + // Row 2 + 0x66, 0x1E, 0x86, 0x19, 0x3E, + // Row 3 + 0x66, 0x33, 0x83, 0x31, 0x33, + // Row 4 + 0x66, 0x33, 0x83, 0x31, 0x03, + // Row 5 + 0x66, 0x3F, 0x83, 0x31, 0x1E, + // Row 6 + 0x66, 0x33, 0x83, 0x31, 0x30, + // Row 7 + 0x66, 0x33, 0x83, 0x31, 0x30, + // Row 8 + 0x3C, 0x33, 0x86, 0x19, 0x33, + // Row 9 + 0x18, 0x33, 0xFC, 0x0F, 0x1E, + // Row 10 + 0x18, 0x00, 0x00, 0x00, 0x00, + // Row 11 + 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +// Alternate larger logo - 48x16 pixels +// More visible "μRNS" with better spacing +static const uint8_t LOGO_LARGE_WIDTH = 48; +static const uint8_t LOGO_LARGE_HEIGHT = 16; +static const uint8_t logo_urns_large[] PROGMEM = { + // Row 0 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // Row 1 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // Row 2 - μ starts, R N S top + 0x00, 0x00, 0xF8, 0x1F, 0x00, 0x00, + // Row 3 + 0xC6, 0x38, 0x0C, 0x30, 0xF8, 0x00, + // Row 4 + 0xC6, 0x6C, 0x06, 0x60, 0xCC, 0x00, + // Row 5 + 0xC6, 0x6C, 0x06, 0x60, 0x0C, 0x00, + // Row 6 + 0xC6, 0x7C, 0x06, 0x60, 0x0C, 0x00, + // Row 7 + 0xC6, 0x6C, 0x06, 0x60, 0x78, 0x00, + // Row 8 + 0xC6, 0x6C, 0x06, 0x60, 0xC0, 0x00, + // Row 9 + 0xC6, 0x6C, 0x06, 0x60, 0xC0, 0x00, + // Row 10 + 0x7C, 0x6C, 0x0C, 0x30, 0xCC, 0x00, + // Row 11 + 0x38, 0x6C, 0xF8, 0x1F, 0x78, 0x00, + // Row 12 + 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, + // Row 13 + 0x30, 0x00, 0x00, 0x00, 0x00, 0x00, + // Row 14 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + // Row 15 + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, +}; + +// Signal strength bars - 12x8 pixels each +// 5 states: 0 bars (offline), 1-4 bars (signal strength) +static const uint8_t SIGNAL_WIDTH = 12; +static const uint8_t SIGNAL_HEIGHT = 8; + +// Signal bars - 0 (offline/no signal) +static const uint8_t signal_0[] PROGMEM = { + 0x00, 0x00, // Row 0 + 0x00, 0x00, // Row 1 + 0x00, 0x00, // Row 2 + 0x00, 0x00, // Row 3 + 0x00, 0x00, // Row 4 + 0x00, 0x00, // Row 5 + 0x00, 0x00, // Row 6 + 0x00, 0x00, // Row 7 +}; + +// Signal bars - 1 bar (weak) +static const uint8_t signal_1[] PROGMEM = { + 0x00, 0x00, // Row 0 + 0x00, 0x00, // Row 1 + 0x00, 0x00, // Row 2 + 0x00, 0x00, // Row 3 + 0x00, 0x00, // Row 4 + 0x00, 0x00, // Row 5 + 0x03, 0x00, // Row 6 - 1 bar + 0x03, 0x00, // Row 7 +}; + +// Signal bars - 2 bars (fair) +static const uint8_t signal_2[] PROGMEM = { + 0x00, 0x00, // Row 0 + 0x00, 0x00, // Row 1 + 0x00, 0x00, // Row 2 + 0x00, 0x00, // Row 3 + 0x0C, 0x00, // Row 4 - 2nd bar + 0x0C, 0x00, // Row 5 + 0x0F, 0x00, // Row 6 - both bars + 0x0F, 0x00, // Row 7 +}; + +// Signal bars - 3 bars (good) +static const uint8_t signal_3[] PROGMEM = { + 0x00, 0x00, // Row 0 + 0x00, 0x00, // Row 1 + 0x30, 0x00, // Row 2 - 3rd bar + 0x30, 0x00, // Row 3 + 0x3C, 0x00, // Row 4 - bars 2+3 + 0x3C, 0x00, // Row 5 + 0x3F, 0x00, // Row 6 - all 3 bars + 0x3F, 0x00, // Row 7 +}; + +// Signal bars - 4 bars (excellent) +static const uint8_t signal_4[] PROGMEM = { + 0xC0, 0x00, // Row 0 - 4th bar + 0xC0, 0x00, // Row 1 + 0xF0, 0x00, // Row 2 - bars 3+4 + 0xF0, 0x00, // Row 3 + 0xFC, 0x00, // Row 4 - bars 2+3+4 + 0xFC, 0x00, // Row 5 + 0xFF, 0x00, // Row 6 - all 4 bars + 0xFF, 0x00, // Row 7 +}; + +// Online indicator - filled circle 8x8 +static const uint8_t INDICATOR_SIZE = 8; +static const uint8_t indicator_online[] PROGMEM = { + 0x3C, // ..####.. + 0x7E, // .######. + 0xFF, // ######## + 0xFF, // ######## + 0xFF, // ######## + 0xFF, // ######## + 0x7E, // .######. + 0x3C, // ..####.. +}; + +// Offline indicator - empty circle 8x8 +static const uint8_t indicator_offline[] PROGMEM = { + 0x3C, // ..####.. + 0x42, // .#....#. + 0x81, // #......# + 0x81, // #......# + 0x81, // #......# + 0x81, // #......# + 0x42, // .#....#. + 0x3C, // ..####.. +}; + +// Link icon - two connected nodes 12x8 +static const uint8_t LINK_ICON_WIDTH = 12; +static const uint8_t LINK_ICON_HEIGHT = 8; +static const uint8_t icon_link[] PROGMEM = { + 0x1E, 0x07, // Row 0 + 0x21, 0x08, // Row 1 + 0x21, 0x08, // Row 2 + 0xE1, 0x08, // Row 3 - connection line + 0xE1, 0x08, // Row 4 - connection line + 0x21, 0x08, // Row 5 + 0x21, 0x08, // Row 6 + 0x1E, 0x07, // Row 7 +}; + +// Helper to get signal bitmap based on level (0-4) +inline const uint8_t* get_signal_bitmap(uint8_t level) { + switch(level) { + case 1: return signal_1; + case 2: return signal_2; + case 3: return signal_3; + case 4: return signal_4; + default: return signal_0; + } +} + +// Convert RSSI (dBm) to signal level (0-4) +// Typical LoRa RSSI ranges: -120 dBm (weak) to -30 dBm (strong) +inline uint8_t rssi_to_level(float rssi) { + if (rssi >= -60) return 4; // Excellent + if (rssi >= -80) return 3; // Good + if (rssi >= -100) return 2; // Fair + if (rssi >= -120) return 1; // Weak + return 0; // No signal / offline +} + +} // namespace Graphics +} // namespace RNS + +#endif // DISPLAY_GRAPHICS_H diff --git a/src-shim/FileStream.h b/src-shim/FileStream.h new file mode 100644 index 00000000..6b69d655 --- /dev/null +++ b/src-shim/FileStream.h @@ -0,0 +1,144 @@ +#pragma once + +#include "Utilities/Crc.h" +#include "Log.h" +#include "Bytes.h" +#include "Type.h" + +#ifdef ARDUINO +#include +#else +#include "Utilities/Stream.h" +#endif + +#include +#include +#include +#include + +namespace RNS { + + class FileStreamImpl { + + protected: + FileStreamImpl() { MEMF("FileStreamImpl object created, this: 0x%X", this); } + public: + virtual ~FileStreamImpl() { MEMF("FileStreamImpl object destroyed, this: 0x%X", this); } + + protected: + // CBA TODO Can these be pure-virtual??? + virtual const char* name() = 0; + virtual size_t size() = 0; + virtual void close() = 0; + + // Print overrides + virtual size_t write(uint8_t byte) = 0; + virtual size_t write(const uint8_t *buffer, size_t size) = 0; + + // Stream overrides + virtual int available() = 0; + virtual int read() = 0; + virtual int peek() = 0; + virtual void flush() = 0; + + friend class FileStream; + }; + + class FileStream : public Stream { + + public: + enum MODE { + MODE_READ, + MODE_WRITE, + MODE_APPEND, + }; + + public: + FileStream(Type::NoneConstructor none) { + MEMF("FileStream NONE object created, this: 0x%X, impl: 0x%X", this, _impl.get()); + } + FileStream(const FileStream& obj) : _impl(obj._impl), _crc(obj._crc) { + MEMF("FileStream object copy created, this: 0x%X, impl: 0x%X", this, _impl.get()); + } + FileStream(FileStreamImpl* impl) : _impl(impl) { + MEMF("FileStream object impl created, this: 0x%X, impl: 0x%X", this, _impl.get()); + } + virtual ~FileStream() { + MEMF("FileStream object destroyed, this: 0x%X, impl: 0x%X", this, _impl.get()); + } + + inline virtual FileStream& operator = (const FileStream& obj) { + _impl = obj._impl; + _crc = obj._crc; + MEMF("FileStream object copy created by assignment, this: 0x%X, impl: 0x%X", this, _impl.get()); + return *this; + } + inline FileStream& operator = (FileStreamImpl* impl) { + _impl.reset(impl); + MEMF("FileStream object copy created by impl assignment, this: 0x%X, impl: 0x%X", this, _impl.get()); + return *this; + } + inline operator bool() const { + MEMF("FileStream object bool, this: 0x%X, impl: 0x%X", this, _impl.get()); + return _impl.get() != nullptr; + } + inline bool operator < (const FileStream& obj) const { + MEMF("FileStream object <, this: 0x%X, impl: 0x%X", this, _impl.get()); + return _impl.get() < obj._impl.get(); + } + inline bool operator > (const FileStream& obj) const { + MEMF("FileStream object <, this: 0x%X, impl: 0x%X", this, _impl.get()); + return _impl.get() > obj._impl.get(); + } + inline bool operator == (const FileStream& obj) const { + MEMF("FileStream object ==, this: 0x%X, impl: 0x%X", this, _impl.get()); + return _impl.get() == obj._impl.get(); + } + inline bool operator != (const FileStream& obj) const { + MEMF("FileStream object !=, this: 0x%X, impl: 0x%X", this, _impl.get()); + return _impl.get() != obj._impl.get(); + } + inline FileStreamImpl* get() { + return _impl.get(); + } + inline void clear() { + _impl.reset(); + } + + public: + inline uint32_t crc() { return _crc; } + inline size_t write(const char* str) { return write((const uint8_t*)str, strlen(str)); } + + // File overrides + inline const char* name() { assert(_impl); return _impl->name(); } + inline size_t size() { assert(_impl); return _impl->size(); } + inline void close() { assert(_impl); _impl->close(); } + + // Print overrides + inline size_t write(uint8_t byte) { assert(_impl); _crc = Utilities::Crc::crc32(_crc, byte); return _impl->write(byte); } + inline size_t write(const uint8_t *buffer, size_t size) { assert(_impl); _crc = Utilities::Crc::crc32(_crc, buffer, size); return _impl->write(buffer, size); } + + // Stream overrides + inline int available() { assert(_impl); return _impl->available(); } + inline int read() { assert(_impl); if (_impl->available() <= 0) return EOF; int ch = _impl->read(); uint8_t byte = (uint8_t)ch; _crc = Utilities::Crc::crc32(_crc, byte); return ch; } + inline int peek() { assert(_impl); return _impl->peek(); } + inline void flush() { assert(_impl); _impl->flush(); } + + // getters/setters + protected: + public: + +#ifndef NDEBUG + inline std::string debugString() const { + std::string dump; + dump = "FileStream object, this: " + std::to_string((uintptr_t)this) + ", data: " + std::to_string((uintptr_t)_impl.get()); + return dump; + } +#endif + + protected: + std::shared_ptr _impl; + uint32_t _crc = 0; + }; + +} diff --git a/src-shim/FileSystem.h b/src-shim/FileSystem.h new file mode 100644 index 00000000..f224420b --- /dev/null +++ b/src-shim/FileSystem.h @@ -0,0 +1,129 @@ +#pragma once + +#include "FileStream.h" +#include "Log.h" +#include "Bytes.h" +#include "Type.h" + +#include +#include +#include +#include + +namespace RNS { + + class FileSystemImpl { + + protected: + FileSystemImpl() { MEMF("FileSystem::FileSystemImpl object created, this: 0x%X", this); } + public: + virtual ~FileSystemImpl() { MEMF("FileSystem::FileSystemImpl object destroyed, this: 0x%X", this); } + + protected: + virtual bool init() { return true; } + virtual void loop() {} + virtual bool file_exists(const char* file_path) = 0; + virtual size_t read_file(const char* file_path, Bytes& data) = 0; + virtual size_t write_file(const char* file_path, const Bytes& data) = 0; + virtual FileStream open_file(const char* file_path, FileStream::MODE file_mode) = 0; + virtual bool remove_file(const char* file_path) = 0; + virtual bool rename_file(const char* from_file_path, const char* to_file_path) = 0; + virtual bool directory_exists(const char* directory_path) = 0; + virtual bool create_directory(const char* directory_path) = 0; + virtual bool remove_directory(const char* directory_path) = 0; + virtual std::list list_directory(const char* directory_path) = 0; + virtual size_t storage_size() = 0; + virtual size_t storage_available() = 0; + + friend class FileSystem; + }; + + class FileSystem { + + public: + FileSystem(Type::NoneConstructor none) { + MEMF("FileSystem NONE object created, this: 0x%X, impl: 0x%X", this, _impl.get()); + } + FileSystem(const FileSystem& obj) : _impl(obj._impl) { + MEMF("FileSystem object copy created, this: 0x%X, impl: 0x%X", this, _impl.get()); + } + FileSystem(FileSystemImpl* impl) : _impl(impl) { + MEMF("FileSystem object impl created, this: 0x%X, impl: 0x%X", this, _impl.get()); + } + virtual ~FileSystem() { + MEMF("FileSystem object destroyed, this: 0x%X, impl: 0x%X", this, _impl.get()); + } + + inline FileSystem& operator = (const FileSystem& obj) { + _impl = obj._impl; + MEMF("FileSystem object copy created by assignment, this: 0x%X, impl: 0x%X", this, _impl.get()); + return *this; + } + inline FileSystem& operator = (FileSystemImpl* impl) { + _impl.reset(impl); + MEMF("FileSystem object copy created by impl assignment, this: 0x%X, impl: 0x%X", this, _impl.get()); + return *this; + } + inline operator bool() const { + MEMF("FileSystem object bool, this: 0x%X, impl: 0x%X", this, _impl.get()); + return _impl.get() != nullptr; + } + inline bool operator < (const FileSystem& obj) const { + MEMF("FileSystem object <, this: 0x%X, impl: 0x%X", this, _impl.get()); + return _impl.get() < obj._impl.get(); + } + inline bool operator > (const FileSystem& obj) const { + MEMF("FileSystem object <, this: 0x%X, impl: 0x%X", this, _impl.get()); + return _impl.get() > obj._impl.get(); + } + inline bool operator == (const FileSystem& obj) const { + MEMF("FileSystem object ==, this: 0x%X, impl: 0x%X", this, _impl.get()); + return _impl.get() == obj._impl.get(); + } + inline bool operator != (const FileSystem& obj) const { + MEMF("FileSystem object !=, this: 0x%X, impl: 0x%X", this, _impl.get()); + return _impl.get() != obj._impl.get(); + } + inline FileSystemImpl* get() { + return _impl.get(); + } + inline void clear() { + _impl.reset(); + } + + public: + inline bool init() { assert(_impl); return _impl->init(); } + inline void loop() { assert(_impl); return _impl->loop(); } + inline bool file_exists(const char* file_path) { assert(_impl); return _impl->file_exists(file_path); } + inline size_t read_file(const char* file_path, Bytes& data) { assert(_impl); return _impl->read_file(file_path, data); } + inline size_t write_file(const char* file_path, const Bytes& data) { assert(_impl); return _impl->write_file(file_path, data); } + inline FileStream open_file(const char* file_path, FileStream::MODE file_mode) { return _impl->open_file(file_path, file_mode); } + inline bool remove_file(const char* file_path) { assert(_impl); return _impl->remove_file(file_path); } + inline bool rename_file(const char* from_file_path, const char* to_file_path) { assert(_impl); return _impl->rename_file(from_file_path, to_file_path); } + inline bool directory_exists(const char* directory_path) { assert(_impl); return _impl->directory_exists(directory_path); } + inline bool create_directory(const char* directory_path) { assert(_impl); return _impl->create_directory(directory_path); } + inline bool remove_directory(const char* directory_path) { assert(_impl); return _impl->remove_directory(directory_path); } + inline std::list list_directory(const char* directory_path) { assert(_impl); return _impl->list_directory(directory_path); } + inline size_t storage_size() { assert(_impl); return _impl->storage_size(); } + inline size_t storage_available() { assert(_impl); return _impl->storage_available(); } + + private: + std::list _empty; + + // getters/setters + protected: + public: + +#ifndef NDEBUG + inline std::string debugString() const { + std::string dump; + dump = "FileSystem object, this: " + std::to_string((uintptr_t)this) + ", data: " + std::to_string((uintptr_t)_impl.get()); + return dump; + } +#endif + + protected: + std::shared_ptr _impl; + }; + +} diff --git a/src-shim/Instrumentation/BootProfiler.cpp b/src-shim/Instrumentation/BootProfiler.cpp new file mode 100644 index 00000000..8bee385d --- /dev/null +++ b/src-shim/Instrumentation/BootProfiler.cpp @@ -0,0 +1,237 @@ +/* + * BootProfiler - Boot sequence timing instrumentation for ESP32 + * + * Implementation uses millis() for timing (millisecond precision). + * All buffers are statically allocated to avoid dynamic memory during boot. + * + * Key design decisions: + * - First markStart() call establishes boot start time + * - Phase timing tracks both individual duration and cumulative total + * - Wait time is tracked separately to distinguish I/O from CPU work + * - bootComplete() logs final summary for analysis + */ + +#include "BootProfiler.h" + +#ifdef BOOT_PROFILING_ENABLED + +#include + +#include +#include +#include + +namespace RNS { namespace Instrumentation { + +// Static member initialization +uint32_t BootProfiler::_boot_start_ms = 0; +uint32_t BootProfiler::_cumulative_ms = 0; +char BootProfiler::_current_phase[32] = {0}; +uint32_t BootProfiler::_phase_start_ms = 0; +char BootProfiler::_current_wait[32] = {0}; +uint32_t BootProfiler::_wait_start_ms = 0; +uint32_t BootProfiler::_total_wait_ms = 0; +char BootProfiler::_log_buffer[256] = {0}; +bool BootProfiler::_fs_ready = false; + + +void BootProfiler::setFilesystemReady(bool ready) { + _fs_ready = ready; + if (ready) { + NOTICE("[BOOT] Filesystem ready for boot profile persistence"); + } +} + + +void BootProfiler::markStart(const char* phase) { + uint32_t now = millis(); + + // First call establishes boot start time + if (_boot_start_ms == 0) { + _boot_start_ms = now; + NOTICE("[BOOT] Profiling started"); + } + + // Store current phase info + strncpy(_current_phase, phase, sizeof(_current_phase) - 1); + _current_phase[sizeof(_current_phase) - 1] = '\0'; + _phase_start_ms = now; + + snprintf(_log_buffer, sizeof(_log_buffer), + "[BOOT] START: %s (at %ums)", + phase, now - _boot_start_ms); + NOTICE(_log_buffer); +} + + +void BootProfiler::markEnd(const char* phase) { + uint32_t now = millis(); + + // Calculate phase duration + uint32_t duration = 0; + if (_phase_start_ms > 0) { + duration = now - _phase_start_ms; + } + + // Update cumulative time + _cumulative_ms = now - _boot_start_ms; + + snprintf(_log_buffer, sizeof(_log_buffer), + "[BOOT] END: %s (%ums, cumulative: %ums)", + phase, duration, _cumulative_ms); + NOTICE(_log_buffer); + + // Clear current phase + _current_phase[0] = '\0'; + _phase_start_ms = 0; +} + + +void BootProfiler::markWaitStart(const char* phase) { + // Store wait info + strncpy(_current_wait, phase, sizeof(_current_wait) - 1); + _current_wait[sizeof(_current_wait) - 1] = '\0'; + _wait_start_ms = millis(); +} + + +void BootProfiler::markWaitEnd(const char* phase) { + uint32_t now = millis(); + + // Calculate wait duration + uint32_t duration = 0; + if (_wait_start_ms > 0) { + duration = now - _wait_start_ms; + } + + // Accumulate wait time + _total_wait_ms += duration; + + snprintf(_log_buffer, sizeof(_log_buffer), + "[BOOT] WAIT: %s (%ums, total wait: %ums)", + phase, duration, _total_wait_ms); + NOTICE(_log_buffer); + + // Clear current wait + _current_wait[0] = '\0'; + _wait_start_ms = 0; +} + + +void BootProfiler::bootComplete() { + uint32_t now = millis(); + + // Calculate final timings + uint32_t total_ms = 0; + if (_boot_start_ms > 0) { + total_ms = now - _boot_start_ms; + } + + uint32_t init_ms = total_ms - _total_wait_ms; + + snprintf(_log_buffer, sizeof(_log_buffer), + "[BOOT] COMPLETE: total=%ums, init=%ums, wait=%ums", + total_ms, init_ms, _total_wait_ms); + NOTICE(_log_buffer); + + // Additional detail if wait time is significant + if (_total_wait_ms > 0) { + uint8_t wait_pct = (uint8_t)((_total_wait_ms * 100) / total_ms); + snprintf(_log_buffer, sizeof(_log_buffer), + "[BOOT] Wait time: %u%% of boot", + wait_pct); + NOTICE(_log_buffer); + } +} + + +uint32_t BootProfiler::getTotalMs() { + if (_boot_start_ms == 0) { + return 0; + } + return millis() - _boot_start_ms; +} + + +uint32_t BootProfiler::getInitMs() { + uint32_t total = getTotalMs(); + if (total < _total_wait_ms) { + return 0; + } + return total - _total_wait_ms; +} + + +uint32_t BootProfiler::getWaitMs() { + return _total_wait_ms; +} + + +bool BootProfiler::saveToFile() { + if (!_fs_ready) { + WARNING("[BOOT] Cannot save profile - filesystem not ready"); + return false; + } + + // Calculate final timings + uint32_t total_ms = getTotalMs(); + uint32_t init_ms = getInitMs(); + uint32_t wait_ms = getWaitMs(); + + // Rotate existing boot logs (delete oldest, shift others) + // Files: boot_1.log (newest) -> boot_5.log (oldest) + char old_path[32]; + char new_path[32]; + + // Delete oldest log if exists + snprintf(old_path, sizeof(old_path), "/boot_%d.log", MAX_BOOT_LOGS); + if (SPIFFS.exists(old_path)) { + SPIFFS.remove(old_path); + } + + // Shift remaining logs (4->5, 3->4, 2->3, 1->2) + for (int i = MAX_BOOT_LOGS - 1; i >= 1; i--) { + snprintf(old_path, sizeof(old_path), "/boot_%d.log", i); + snprintf(new_path, sizeof(new_path), "/boot_%d.log", i + 1); + if (SPIFFS.exists(old_path)) { + SPIFFS.rename(old_path, new_path); + } + } + + // Write new boot profile to boot_1.log + File file = SPIFFS.open("/boot_1.log", FILE_WRITE); + if (!file) { + ERROR("[BOOT] Failed to create boot profile file"); + return false; + } + + // Write boot timing summary + file.printf("Boot Profile\n"); + file.printf("============\n"); + file.printf("Total: %u ms\n", total_ms); + file.printf("Init: %u ms\n", init_ms); + file.printf("Wait: %u ms\n", wait_ms); + + // Add timestamp if available + time_t now = time(nullptr); + if (now > 1704067200) { // After 2024-01-01 + struct tm timeinfo; + localtime_r(&now, &timeinfo); + char time_str[32]; + strftime(time_str, sizeof(time_str), "%Y-%m-%d %H:%M:%S", &timeinfo); + file.printf("Time: %s\n", time_str); + } + + file.close(); + + snprintf(_log_buffer, sizeof(_log_buffer), + "[BOOT] Profile saved to /boot_1.log (total=%ums, init=%ums, wait=%ums)", + total_ms, init_ms, wait_ms); + NOTICE(_log_buffer); + + return true; +} + +}} // namespace RNS::Instrumentation + +#endif // BOOT_PROFILING_ENABLED diff --git a/src-shim/Instrumentation/BootProfiler.h b/src-shim/Instrumentation/BootProfiler.h new file mode 100644 index 00000000..d17bebf1 --- /dev/null +++ b/src-shim/Instrumentation/BootProfiler.h @@ -0,0 +1,171 @@ +#pragma once + +/* + * BootProfiler - Boot sequence timing instrumentation for ESP32 + * + * Provides phase-level timing for boot sequence analysis, distinguishing + * between initialization time (CPU work) and blocking wait time (I/O waits). + * + * Usage: + * 1. Define BOOT_PROFILING_ENABLED in build flags + * 2. Use BOOT_PROFILE_START("phase") at the beginning of each init phase + * 3. Use BOOT_PROFILE_END("phase") when phase completes + * 4. Use BOOT_PROFILE_WAIT_START/END for blocking waits within phases + * 5. Call BOOT_PROFILE_COMPLETE() when boot is finished + * + * Output includes: + * - Per-phase duration and cumulative time + * - Separate tracking of init vs wait time + * - Final summary with breakdown + * + * When disabled, all API calls compile to no-ops via stub macros. + */ + +#ifdef BOOT_PROFILING_ENABLED + +#include +#include + +namespace RNS { namespace Instrumentation { + +/** + * BootProfiler - Static class for boot sequence timing + * + * Tracks: + * - Boot start time (first markStart call) + * - Per-phase duration with cumulative totals + * - Blocking wait time separate from init time + * - Total boot duration on bootComplete() + * + * All methods are static - no instantiation required. + */ +class BootProfiler { +public: + /** + * Mark the start of a boot phase + * + * First call also records boot start time. + * + * @param phase Human-readable phase name (e.g., "LVGL", "WiFi", "Reticulum") + */ + static void markStart(const char* phase); + + /** + * Mark the end of a boot phase + * + * Logs phase duration and cumulative boot time. + * + * @param phase Phase name (should match corresponding markStart) + */ + static void markEnd(const char* phase); + + /** + * Mark the start of a blocking wait within a phase + * + * Use for I/O waits, network connection, etc. that are not CPU work. + * + * @param phase Phase/operation name for the wait + */ + static void markWaitStart(const char* phase); + + /** + * Mark the end of a blocking wait + * + * Wait time is tracked separately from init time. + * + * @param phase Phase/operation name (should match corresponding markWaitStart) + */ + static void markWaitEnd(const char* phase); + + /** + * Mark boot sequence as complete + * + * Logs summary with total time, init time, and wait time breakdown. + * Call when UI is responsive (LVGL interactive). + */ + static void bootComplete(); + + /** + * Get total milliseconds since boot started + * + * @return Cumulative boot time in milliseconds, 0 if not started + */ + static uint32_t getTotalMs(); + + /** + * Get total init time (excluding waits) + * + * @return Init time in milliseconds + */ + static uint32_t getInitMs(); + + /** + * Get total wait time + * + * @return Accumulated wait time in milliseconds + */ + static uint32_t getWaitMs(); + + /** + * Set filesystem ready state + * + * Must be called after SPIFFS is mounted before saveToFile() can work. + * + * @param ready True if filesystem is ready for writes + */ + static void setFilesystemReady(bool ready); + + /** + * Save boot profile to SPIFFS file + * + * Writes boot timing data to /boot_1.log, rotating existing files + * (max 5 boot logs retained). Requires filesystem to be ready. + * + * @return True if file was saved successfully + */ + static bool saveToFile(); + +private: + // Maximum number of boot log files to retain + static const uint8_t MAX_BOOT_LOGS = 5; + // Boot timing + static uint32_t _boot_start_ms; + static uint32_t _cumulative_ms; + + // Current phase tracking + static char _current_phase[32]; + static uint32_t _phase_start_ms; + + // Wait time tracking + static char _current_wait[32]; + static uint32_t _wait_start_ms; + static uint32_t _total_wait_ms; + + // Static buffer for log formatting (avoid stack allocation) + static char _log_buffer[256]; + + // Filesystem ready flag + static bool _fs_ready; +}; + +}} // namespace RNS::Instrumentation + +// Convenience macros for cleaner integration +#define BOOT_PROFILE_START(phase) RNS::Instrumentation::BootProfiler::markStart(phase) +#define BOOT_PROFILE_END(phase) RNS::Instrumentation::BootProfiler::markEnd(phase) +#define BOOT_PROFILE_WAIT_START(phase) RNS::Instrumentation::BootProfiler::markWaitStart(phase) +#define BOOT_PROFILE_WAIT_END(phase) RNS::Instrumentation::BootProfiler::markWaitEnd(phase) +#define BOOT_PROFILE_COMPLETE() RNS::Instrumentation::BootProfiler::bootComplete() +#define BOOT_PROFILE_SAVE() RNS::Instrumentation::BootProfiler::saveToFile() + +#else // BOOT_PROFILING_ENABLED not defined + +// Stub macros - compile to no-ops when profiling disabled +#define BOOT_PROFILE_START(phase) ((void)0) +#define BOOT_PROFILE_END(phase) ((void)0) +#define BOOT_PROFILE_WAIT_START(phase) ((void)0) +#define BOOT_PROFILE_WAIT_END(phase) ((void)0) +#define BOOT_PROFILE_COMPLETE() ((void)0) +#define BOOT_PROFILE_SAVE() ((void)0) + +#endif // BOOT_PROFILING_ENABLED diff --git a/src-shim/Instrumentation/MemoryMonitor.cpp b/src-shim/Instrumentation/MemoryMonitor.cpp new file mode 100644 index 00000000..c5fbca1b --- /dev/null +++ b/src-shim/Instrumentation/MemoryMonitor.cpp @@ -0,0 +1,263 @@ +/* + * MemoryMonitor - Runtime memory instrumentation for ESP32-S3 + * + * Implementation uses ESP-IDF heap_caps APIs for heap monitoring + * and FreeRTOS APIs for task stack monitoring. + * + * Key design decisions: + * - Static buffers only (no dynamic allocation in instrumentation) + * - FreeRTOS software timer for periodic logging (runs in daemon task) + * - Explicit task registry (uxTaskGetSystemState unavailable in Arduino) + * - Separate internal RAM and PSRAM metrics + */ + +#include "MemoryMonitor.h" + +#ifdef MEMORY_INSTRUMENTATION_ENABLED + +#include +#include + +#include + +namespace RNS { namespace Instrumentation { + +// Maximum number of tasks that can be registered for stack monitoring +static constexpr size_t MAX_MONITORED_TASKS = 16; + +// Task registry entry +struct TaskEntry { + TaskHandle_t handle; + char name[configMAX_TASK_NAME_LEN]; +}; + +// Static task registry (no dynamic allocation) +static TaskEntry _task_registry[MAX_MONITORED_TASKS]; +static size_t _task_count = 0; + +// Static member initialization +TimerHandle_t MemoryMonitor::_timer = nullptr; +bool MemoryMonitor::_verbose = false; + +// Static buffer for log formatting (avoid stack allocation in callbacks) +static char _log_buffer[256]; +static char _stack_buffer[256]; + + +bool MemoryMonitor::init(uint32_t interval_ms) { + // Don't reinitialize if already running + if (_timer != nullptr) { + WARNING("[MEM_MON] Already initialized, stop() first to reinit"); + return false; + } + + // Create FreeRTOS software timer + _timer = xTimerCreate( + "mem_mon", // Timer name (for debugging) + pdMS_TO_TICKS(interval_ms), // Period in ticks + pdTRUE, // Auto-reload (repeat) + nullptr, // Timer ID (unused) + timerCallback // Callback function + ); + + if (_timer == nullptr) { + ERROR("[MEM_MON] Failed to create timer"); + return false; + } + + // Start the timer + if (xTimerStart(_timer, 0) != pdPASS) { + ERROR("[MEM_MON] Failed to start timer"); + xTimerDelete(_timer, 0); + _timer = nullptr; + return false; + } + + NOTICEF("[MEM_MON] Started (interval=%ums)", interval_ms); + + // Log initial state immediately + logHeapStats(); + + return true; +} + + +void MemoryMonitor::stop() { + if (_timer != nullptr) { + xTimerStop(_timer, 0); + xTimerDelete(_timer, 0); + _timer = nullptr; + NOTICE("[MEM_MON] Stopped"); + } +} + + +void MemoryMonitor::registerTask(TaskHandle_t handle, const char* name) { + if (handle == nullptr) { + WARNING("[MEM_MON] Cannot register null task handle"); + return; + } + + if (_task_count >= MAX_MONITORED_TASKS) { + WARNINGF("[MEM_MON] Task registry full, cannot register '%s'", name); + return; + } + + // Check for duplicate + for (size_t i = 0; i < _task_count; i++) { + if (_task_registry[i].handle == handle) { + VERBOSEF("[MEM_MON] Task '%s' already registered", name); + return; + } + } + + // Add to registry + _task_registry[_task_count].handle = handle; + strncpy(_task_registry[_task_count].name, name, configMAX_TASK_NAME_LEN - 1); + _task_registry[_task_count].name[configMAX_TASK_NAME_LEN - 1] = '\0'; + _task_count++; + + VERBOSEF("[MEM_MON] Registered task '%s' (%u/%u)", + name, _task_count, MAX_MONITORED_TASKS); +} + + +void MemoryMonitor::unregisterTask(TaskHandle_t handle) { + for (size_t i = 0; i < _task_count; i++) { + if (_task_registry[i].handle == handle) { + VERBOSEF("[MEM_MON] Unregistered task '%s'", _task_registry[i].name); + + // Shift remaining entries to fill gap + for (size_t j = i; j < _task_count - 1; j++) { + _task_registry[j] = _task_registry[j + 1]; + } + _task_count--; + return; + } + } +} + + +void MemoryMonitor::setVerbose(bool verbose) { + _verbose = verbose; + VERBOSEF("[MEM_MON] Verbose mode %s", verbose ? "enabled" : "disabled"); +} + + +void MemoryMonitor::logNow() { + logHeapStats(); + if (_task_count > 0) { + logTaskStacks(); + } +} + + +void MemoryMonitor::timerCallback(TimerHandle_t timer) { + (void)timer; // Unused parameter + + logHeapStats(); + if (_task_count > 0) { + logTaskStacks(); + } +} + + +void MemoryMonitor::logHeapStats() { + // Internal RAM statistics (critical for stability) + size_t internal_free = heap_caps_get_free_size(MALLOC_CAP_INTERNAL); + size_t internal_largest = heap_caps_get_largest_free_block(MALLOC_CAP_INTERNAL); + size_t internal_min = heap_caps_get_minimum_free_size(MALLOC_CAP_INTERNAL); + + // Calculate internal RAM fragmentation percentage + // Fragmentation = 100 - (largest_block / total_free * 100) + // Higher fragmentation = smaller largest block relative to free space + uint8_t internal_frag = 0; + if (internal_free > 0) { + internal_frag = 100 - (uint8_t)((internal_largest * 100) / internal_free); + } + + // PSRAM statistics + size_t psram_free = heap_caps_get_free_size(MALLOC_CAP_SPIRAM); + size_t psram_largest = heap_caps_get_largest_free_block(MALLOC_CAP_SPIRAM); + size_t psram_min = heap_caps_get_minimum_free_size(MALLOC_CAP_SPIRAM); + + // Calculate PSRAM fragmentation + uint8_t psram_frag = 0; + if (psram_free > 0) { + psram_frag = 100 - (uint8_t)((psram_largest * 100) / psram_free); + } + + if (_verbose) { + // Verbose format: separate lines with labels + NOTICEF("[HEAP] Internal: free=%u largest=%u min=%u frag=%u%%", + internal_free, internal_largest, internal_min, internal_frag); + NOTICEF("[HEAP] PSRAM: free=%u largest=%u min=%u frag=%u%%", + psram_free, psram_largest, psram_min, psram_frag); + } else { + // Compact format: single line for parsing/graphing + snprintf(_log_buffer, sizeof(_log_buffer), + "[HEAP] int_free=%u int_largest=%u int_min=%u int_frag=%u%% " + "psram_free=%u psram_largest=%u", + internal_free, internal_largest, internal_min, internal_frag, + psram_free, psram_largest); + NOTICE(_log_buffer); + } + + // Warn if fragmentation is problematic (>50% is concerning) + if (internal_frag > 50) { + WARNINGF("[HEAP] Internal RAM fragmentation high: %u%%", internal_frag); + } + + // Warn if minimum free dropped significantly (memory pressure) + if (internal_min < 10000) { + WARNINGF("[HEAP] Internal RAM watermark low: %u bytes", internal_min); + } + + // BytesPool stats - shows actual pool usage + auto& pool = BytesPool::instance(); + NOTICEF("[POOL] tiny=%zu/%zu small=%zu/%zu med=%zu/%zu large=%zu/%zu " + "hits=%zu misses=%zu fallbacks=%zu", + pool.tiny_in_use(), BytesPoolConfig::TINY_SLOTS, + pool.small_in_use(), BytesPoolConfig::SMALL_SLOTS, + pool.medium_in_use(), BytesPoolConfig::MEDIUM_SLOTS, + pool.large_in_use(), BytesPoolConfig::LARGE_SLOTS, + pool.pool_hits(), pool.pool_misses(), pool.fallback_count()); +} + + +void MemoryMonitor::logTaskStacks() { + if (_task_count == 0) { + return; + } + + size_t offset = 0; + + // Build compact stack report: "task1=Nbytes task2=Nbytes ..." + offset = snprintf(_stack_buffer, sizeof(_stack_buffer), "[STACK] "); + + for (size_t i = 0; i < _task_count && offset < sizeof(_stack_buffer) - 40; i++) { + // Get stack high water mark (minimum free stack since task start) + // Returns value in words (4 bytes on ESP32) + UBaseType_t hwm_words = uxTaskGetStackHighWaterMark(_task_registry[i].handle); + uint32_t hwm_bytes = hwm_words * 4; + + offset += snprintf(_stack_buffer + offset, sizeof(_stack_buffer) - offset, + "%s=%u ", _task_registry[i].name, hwm_bytes); + } + + NOTICE(_stack_buffer); + + // Warn about low stack (less than 256 bytes remaining is dangerous) + for (size_t i = 0; i < _task_count; i++) { + UBaseType_t hwm_words = uxTaskGetStackHighWaterMark(_task_registry[i].handle); + uint32_t hwm_bytes = hwm_words * 4; + if (hwm_bytes < 256) { + WARNINGF("[STACK] Task '%s' stack low: %u bytes remaining", + _task_registry[i].name, hwm_bytes); + } + } +} + +}} // namespace RNS::Instrumentation + +#endif // MEMORY_INSTRUMENTATION_ENABLED diff --git a/src-shim/Instrumentation/MemoryMonitor.h b/src-shim/Instrumentation/MemoryMonitor.h new file mode 100644 index 00000000..44578ea3 --- /dev/null +++ b/src-shim/Instrumentation/MemoryMonitor.h @@ -0,0 +1,135 @@ +#pragma once + +/* + * MemoryMonitor - Runtime memory instrumentation for ESP32-S3 + * + * Provides heap/stack monitoring capabilities with periodic logging. + * All functionality is guarded by MEMORY_INSTRUMENTATION_ENABLED build flag. + * + * Usage: + * 1. Define MEMORY_INSTRUMENTATION_ENABLED in build flags + * 2. Call MemoryMonitor::init() at startup + * 3. Optionally register tasks for stack monitoring + * 4. Monitor logs for heap fragmentation and stack usage + * + * When disabled, all API calls compile to no-ops via stub macros. + */ + +#ifdef MEMORY_INSTRUMENTATION_ENABLED + +#include +#include +#include +#include + +#include +#include + +namespace RNS { namespace Instrumentation { + +/** + * MemoryMonitor - Static class for runtime memory monitoring + * + * Monitors: + * - Internal RAM: free, largest block, minimum free (watermark), fragmentation % + * - PSRAM: free, largest block + * - Task stack high water marks for registered tasks + * + * All methods are static - no instantiation required. + */ +class MemoryMonitor { +public: + /** + * Initialize and start periodic monitoring + * + * @param interval_ms Logging interval in milliseconds (default 30 seconds) + * @return true if timer created and started successfully + */ + static bool init(uint32_t interval_ms = 30000); + + /** + * Stop the monitoring timer + * + * Safe to call even if not initialized. + */ + static void stop(); + + /** + * Register a task for stack monitoring + * + * @param handle FreeRTOS task handle + * @param name Human-readable name for logging + * + * Note: Maximum 16 tasks can be registered. Additional registrations + * are silently ignored. + */ + static void registerTask(TaskHandle_t handle, const char* name); + + /** + * Unregister a task from stack monitoring + * + * @param handle FreeRTOS task handle to remove + */ + static void unregisterTask(TaskHandle_t handle); + + /** + * Toggle verbose output mode + * + * When verbose: + * - Additional metrics may be logged + * - Format may be more detailed + * + * @param verbose true for verbose output + */ + static void setVerbose(bool verbose); + + /** + * Trigger immediate log output + * + * Useful for debugging or capturing state at specific points. + * Does not affect the periodic timer. + */ + static void logNow(); + +private: + /** + * FreeRTOS timer callback + * + * Called by the timer daemon task at each interval. + */ + static void timerCallback(TimerHandle_t timer); + + /** + * Log heap statistics for internal RAM and PSRAM + */ + static void logHeapStats(); + + /** + * Log stack high water marks for all registered tasks + */ + static void logTaskStacks(); + + // Static members + static TimerHandle_t _timer; + static bool _verbose; +}; + +}} // namespace RNS::Instrumentation + +// Convenience macros for cleaner integration +#define MEMORY_MONITOR_INIT(interval) RNS::Instrumentation::MemoryMonitor::init(interval) +#define MEMORY_MONITOR_REGISTER_TASK(handle, name) RNS::Instrumentation::MemoryMonitor::registerTask(handle, name) +#define MEMORY_MONITOR_UNREGISTER_TASK(handle) RNS::Instrumentation::MemoryMonitor::unregisterTask(handle) +#define MEMORY_MONITOR_LOG_NOW() RNS::Instrumentation::MemoryMonitor::logNow() +#define MEMORY_MONITOR_STOP() RNS::Instrumentation::MemoryMonitor::stop() + +#else // MEMORY_INSTRUMENTATION_ENABLED not defined + +// Stub macros - compile to no-ops when instrumentation disabled +#define MEMORY_MONITOR_INIT(interval) ((void)0) +#define MEMORY_MONITOR_REGISTER_TASK(handle, name) ((void)0) +#define MEMORY_MONITOR_UNREGISTER_TASK(handle) ((void)0) +#define MEMORY_MONITOR_LOG_NOW() ((void)0) +#define MEMORY_MONITOR_STOP() ((void)0) + +#endif // MEMORY_INSTRUMENTATION_ENABLED diff --git a/src-shim/LXMF/LXMRouter.cpp b/src-shim/LXMF/LXMRouter.cpp new file mode 100644 index 00000000..5b605132 --- /dev/null +++ b/src-shim/LXMF/LXMRouter.cpp @@ -0,0 +1,1688 @@ +#include "LXMRouter.h" +#include "PropagationNodeManager.h" +#include +#include +#include +#include +#include + +#include + +using namespace LXMF; +using namespace RNS; + +// ============== Static Fixed Pool Definitions ============== + +// Router registry fixed pool (zero heap fragmentation) +static constexpr size_t ROUTER_REGISTRY_SIZE = 4; +static constexpr size_t REGISTRY_HASH_SIZE = 16; // Truncated hash size +struct RouterRegistrySlot { + bool in_use = false; + uint8_t destination_hash[REGISTRY_HASH_SIZE]; + LXMRouter* router = nullptr; + Bytes destination_hash_bytes() const { return Bytes(destination_hash, REGISTRY_HASH_SIZE); } + void set_destination_hash(const Bytes& b) { + size_t len = std::min(b.size(), REGISTRY_HASH_SIZE); + memcpy(destination_hash, b.data(), len); + if (len < REGISTRY_HASH_SIZE) memset(destination_hash + len, 0, REGISTRY_HASH_SIZE - len); + } + bool destination_hash_equals(const Bytes& b) const { + if (b.size() != REGISTRY_HASH_SIZE) return false; + return memcmp(destination_hash, b.data(), REGISTRY_HASH_SIZE) == 0; + } + void clear() { + in_use = false; + memset(destination_hash, 0, REGISTRY_HASH_SIZE); + router = nullptr; + } +}; +static RouterRegistrySlot _router_registry_pool[ROUTER_REGISTRY_SIZE]; + +static RouterRegistrySlot* find_router_registry_slot(const Bytes& hash) { + for (size_t i = 0; i < ROUTER_REGISTRY_SIZE; i++) { + if (_router_registry_pool[i].in_use && _router_registry_pool[i].destination_hash_equals(hash)) { + return &_router_registry_pool[i]; + } + } + return nullptr; +} + +static RouterRegistrySlot* find_empty_router_registry_slot() { + for (size_t i = 0; i < ROUTER_REGISTRY_SIZE; i++) { + if (!_router_registry_pool[i].in_use) { + return &_router_registry_pool[i]; + } + } + return nullptr; +} + +// Outbound resources fixed pool (zero heap fragmentation) +// Fixed arrays eliminate ~0.8KB Bytes metadata overhead (16 slots × 2 Bytes × 24 bytes) +static constexpr size_t OUTBOUND_RESOURCES_SIZE = 16; +static constexpr size_t OUTBOUND_HASH_SIZE = 32; // SHA256 hash size +struct OutboundResourceSlot { + bool in_use = false; + uint8_t resource_hash[OUTBOUND_HASH_SIZE]; + uint8_t message_hash[OUTBOUND_HASH_SIZE]; + Bytes resource_hash_bytes() const { return Bytes(resource_hash, OUTBOUND_HASH_SIZE); } + Bytes message_hash_bytes() const { return Bytes(message_hash, OUTBOUND_HASH_SIZE); } + void set_resource_hash(const Bytes& b) { + size_t len = std::min(b.size(), OUTBOUND_HASH_SIZE); + memcpy(resource_hash, b.data(), len); + if (len < OUTBOUND_HASH_SIZE) memset(resource_hash + len, 0, OUTBOUND_HASH_SIZE - len); + } + void set_message_hash(const Bytes& b) { + size_t len = std::min(b.size(), OUTBOUND_HASH_SIZE); + memcpy(message_hash, b.data(), len); + if (len < OUTBOUND_HASH_SIZE) memset(message_hash + len, 0, OUTBOUND_HASH_SIZE - len); + } + bool resource_hash_equals(const Bytes& b) const { + if (b.size() != OUTBOUND_HASH_SIZE) return false; + return memcmp(resource_hash, b.data(), OUTBOUND_HASH_SIZE) == 0; + } + void clear() { + in_use = false; + memset(resource_hash, 0, OUTBOUND_HASH_SIZE); + memset(message_hash, 0, OUTBOUND_HASH_SIZE); + } +}; +static OutboundResourceSlot _outbound_resources_pool[OUTBOUND_RESOURCES_SIZE]; + +static OutboundResourceSlot* find_outbound_resource_slot(const Bytes& hash) { + for (size_t i = 0; i < OUTBOUND_RESOURCES_SIZE; i++) { + if (_outbound_resources_pool[i].in_use && _outbound_resources_pool[i].resource_hash_equals(hash)) { + return &_outbound_resources_pool[i]; + } + } + return nullptr; +} + +static OutboundResourceSlot* find_empty_outbound_resource_slot() { + for (size_t i = 0; i < OUTBOUND_RESOURCES_SIZE; i++) { + if (!_outbound_resources_pool[i].in_use) { + return &_outbound_resources_pool[i]; + } + } + return nullptr; +} + +// Static pending proofs pool definition +LXMRouter::PendingProofSlot LXMRouter::_pending_proofs_pool[LXMRouter::PENDING_PROOFS_SIZE]; + +// Static pending propagation resources pool definition +LXMRouter::PropResourceSlot LXMRouter::_pending_prop_resources_pool[LXMRouter::PENDING_PROP_RESOURCES_SIZE]; + +// ============== End Static Fixed Pool Definitions ============== + +// ============== Fixed Pool Helper Functions ============== + +LXMRouter::DirectLinkSlot* LXMRouter::find_direct_link_slot(const Bytes& hash) { + for (size_t i = 0; i < DIRECT_LINKS_SIZE; i++) { + if (_direct_links_pool[i].in_use && _direct_links_pool[i].destination_hash_equals(hash)) { + return &_direct_links_pool[i]; + } + } + return nullptr; +} + +LXMRouter::DirectLinkSlot* LXMRouter::find_empty_direct_link_slot() { + for (size_t i = 0; i < DIRECT_LINKS_SIZE; i++) { + if (!_direct_links_pool[i].in_use) { + return &_direct_links_pool[i]; + } + } + return nullptr; +} + +size_t LXMRouter::direct_links_count() { + size_t count = 0; + for (size_t i = 0; i < DIRECT_LINKS_SIZE; i++) { + if (_direct_links_pool[i].in_use) count++; + } + return count; +} + +LXMRouter::PendingProofSlot* LXMRouter::find_pending_proof_slot(const Bytes& hash) { + for (size_t i = 0; i < PENDING_PROOFS_SIZE; i++) { + if (_pending_proofs_pool[i].in_use && _pending_proofs_pool[i].packet_hash_equals(hash)) { + return &_pending_proofs_pool[i]; + } + } + return nullptr; +} + +LXMRouter::PendingProofSlot* LXMRouter::find_empty_pending_proof_slot() { + for (size_t i = 0; i < PENDING_PROOFS_SIZE; i++) { + if (!_pending_proofs_pool[i].in_use) { + return &_pending_proofs_pool[i]; + } + } + return nullptr; +} + +LXMRouter::PropResourceSlot* LXMRouter::find_prop_resource_slot(const Bytes& hash) { + for (size_t i = 0; i < PENDING_PROP_RESOURCES_SIZE; i++) { + if (_pending_prop_resources_pool[i].in_use && _pending_prop_resources_pool[i].resource_hash_equals(hash)) { + return &_pending_prop_resources_pool[i]; + } + } + return nullptr; +} + +LXMRouter::PropResourceSlot* LXMRouter::find_empty_prop_resource_slot() { + for (size_t i = 0; i < PENDING_PROP_RESOURCES_SIZE; i++) { + if (!_pending_prop_resources_pool[i].in_use) { + return &_pending_prop_resources_pool[i]; + } + } + return nullptr; +} + +bool LXMRouter::transient_ids_contains(const Bytes& id) { + for (size_t i = 0; i < _transient_ids_count; i++) { + size_t idx = (_transient_ids_head + TRANSIENT_IDS_SIZE - _transient_ids_count + i) % TRANSIENT_IDS_SIZE; + if (_transient_ids_buffer[idx] == id) { + return true; + } + } + return false; +} + +void LXMRouter::transient_ids_add(const Bytes& id) { + _transient_ids_buffer[_transient_ids_head] = id; + _transient_ids_head = (_transient_ids_head + 1) % TRANSIENT_IDS_SIZE; + if (_transient_ids_count < TRANSIENT_IDS_SIZE) { + _transient_ids_count++; + } +} + +// ============== End Fixed Pool Helper Functions ============== + +// Static packet callback for destination +static void static_packet_callback(const Bytes& data, const Packet& packet) { + // Look up router by destination hash + RouterRegistrySlot* slot = find_router_registry_slot(packet.destination_hash()); + if (slot) { + slot->router->on_packet(data, packet); + } +} + +// Static link callbacks +static void static_link_established_callback(Link& link) { + // Find router that owns this link destination + RouterRegistrySlot* slot = find_router_registry_slot(link.destination().hash()); + if (slot) { + slot->router->on_link_established(link); + } +} + +static void static_link_closed_callback(Link& link) { + // Find router that owns this link destination + RouterRegistrySlot* slot = find_router_registry_slot(link.destination().hash()); + if (slot) { + slot->router->on_link_closed(link); + } +} + +// Static callback for incoming link established on delivery destination +static void static_delivery_link_established_callback(Link& link) { + // Find router that owns this destination + RouterRegistrySlot* slot = find_router_registry_slot(link.destination().hash()); + if (slot) { + slot->router->on_incoming_link_established(link); + } +} + +// Static callback for resource concluded on delivery links (receiving) +static void static_resource_concluded_callback(const Resource& resource) { + Link link = resource.link(); + if (!link) { + ERROR("static_resource_concluded_callback: Resource has no link"); + return; + } + + // Find router that owns this link's destination + RouterRegistrySlot* slot = find_router_registry_slot(link.destination().hash()); + if (slot) { + slot->router->on_resource_concluded(resource); + } +} + +// Static callback for outbound resource concluded (sending) +// Called when our resource transfer completes (receiver sent RESOURCE_PRF) +static void static_outbound_resource_concluded(const Resource& resource) { + Bytes resource_hash = resource.hash(); + char buf[128]; + snprintf(buf, sizeof(buf), "Outbound resource concluded: %.16s...", resource_hash.toHex().c_str()); + DEBUG(buf); + snprintf(buf, sizeof(buf), " Status: %d", (int)resource.status()); + DEBUG(buf); + + // Check if this resource is one we're tracking + OutboundResourceSlot* slot = find_outbound_resource_slot(resource_hash); + if (!slot) { + DEBUG(" Resource not in pending outbound map"); + return; + } + + Bytes message_hash = slot->message_hash_bytes(); + + // Check if resource completed successfully + if (resource.status() == RNS::Type::Resource::COMPLETE) { + snprintf(buf, sizeof(buf), "DIRECT delivery confirmed for message %.16s...", message_hash.toHex().c_str()); + INFO(buf); + + // Use the public static method to trigger delivered callback + LXMRouter::handle_direct_proof(message_hash); + } else { + snprintf(buf, sizeof(buf), "DIRECT resource transfer failed with status %d", (int)resource.status()); + WARNING(buf); + } + + // Remove from pending map + slot->clear(); +} + +// Static proof callback - called when delivery proof is received +void LXMRouter::static_proof_callback(const PacketReceipt& receipt) { + DEBUG(">>> PROOF CALLBACK ENTRY"); +#ifdef ARDUINO + Serial.flush(); +#endif + + // Get packet hash from receipt + DEBUG(">>> Getting packet hash from receipt"); +#ifdef ARDUINO + Serial.flush(); +#endif + Bytes packet_hash = receipt.hash(); + char buf[128]; + + DEBUG(">>> Looking up pending proof slot"); +#ifdef ARDUINO + Serial.flush(); +#endif + + // Look up message hash for this packet + PendingProofSlot* slot = find_pending_proof_slot(packet_hash); + if (slot) { + DEBUG(">>> Found slot, getting message hash"); +#ifdef ARDUINO + Serial.flush(); +#endif + Bytes message_hash = slot->message_hash_bytes(); + snprintf(buf, sizeof(buf), "Delivery proof received for message %.16s...", message_hash.toHex().c_str()); + INFO(buf); +#ifdef ARDUINO + Serial.flush(); +#endif + + // Track notified routers to avoid duplicates (max ROUTER_REGISTRY_SIZE) + LXMRouter* notified_routers[ROUTER_REGISTRY_SIZE]; + size_t notified_count = 0; + + DEBUG(">>> Iterating router registry"); +#ifdef ARDUINO + Serial.flush(); +#endif + + // Find the router that sent this message and call its delivered callback + for (size_t i = 0; i < ROUTER_REGISTRY_SIZE; i++) { + if (_router_registry_pool[i].in_use) { + LXMRouter* router = _router_registry_pool[i].router; + // Check if already notified + bool already_notified = false; + for (size_t j = 0; j < notified_count; j++) { + if (notified_routers[j] == router) { + already_notified = true; + break; + } + } + if (!already_notified && router && router->_delivered_callback) { + DEBUGF(">>> Calling delivered callback for router %zu", i); +#ifdef ARDUINO + Serial.flush(); +#endif + notified_routers[notified_count++] = router; + // Create a minimal message with just the hash for the callback + // The callback can look up full message from storage if needed + Bytes empty_hash; + LXMessage msg(empty_hash, empty_hash); + msg.hash(message_hash); + msg.state(Type::Message::DELIVERED); + DEBUG(">>> About to invoke callback"); +#ifdef ARDUINO + Serial.flush(); +#endif + router->_delivered_callback(msg); + DEBUG(">>> Callback returned"); +#ifdef ARDUINO + Serial.flush(); +#endif + } + } + } + + // Remove from pending proofs + DEBUG(">>> Clearing slot"); +#ifdef ARDUINO + Serial.flush(); +#endif + slot->clear(); + DEBUG(">>> Slot cleared"); +#ifdef ARDUINO + Serial.flush(); +#endif + } else { + snprintf(buf, sizeof(buf), "Received proof for unknown packet: %.16s...", packet_hash.toHex().c_str()); + DEBUG(buf); + } + DEBUG(">>> PROOF CALLBACK EXIT"); +#ifdef ARDUINO + Serial.flush(); +#endif +} + +// Constructor +LXMRouter::LXMRouter( + const Identity& identity, + const std::string& storage_path, + bool announce_at_start +) : + _identity(identity), + _delivery_destination(RNS::Type::NONE), + _storage_path(storage_path), + _announce_at_start(announce_at_start) +{ + INFO("Initializing LXMF Router"); + + // Create delivery destination: /lxmf/delivery + _delivery_destination = Destination( + _identity, + RNS::Type::Destination::IN, + RNS::Type::Destination::SINGLE, + "lxmf", + "delivery" + ); + + // Register this router in global registry for callback dispatch + RouterRegistrySlot* slot = find_empty_router_registry_slot(); + if (slot) { + slot->in_use = true; + slot->set_destination_hash(_delivery_destination.hash()); + slot->router = this; + } else { + ERROR("Router registry full - cannot register router!"); + } + + // Register packet callback for receiving LXMF messages (OPPORTUNISTIC) + _delivery_destination.set_packet_callback(static_packet_callback); + + // Register link established callback for incoming links (DIRECT delivery) + _delivery_destination.set_link_established_callback(static_delivery_link_established_callback); + + char buf[128]; + snprintf(buf, sizeof(buf), " Delivery destination: %s", _delivery_destination.hash().toHex().c_str()); + INFO(buf); + snprintf(buf, sizeof(buf), " Destination type: %u", (uint8_t)_delivery_destination.type()); + INFO(buf); + snprintf(buf, sizeof(buf), " Destination direction: %u", (uint8_t)_delivery_destination.direction()); + INFO(buf); + + // Announce at start if enabled + if (_announce_at_start) { + INFO(" Auto-announce enabled"); + announce(); + } + + _initialized = true; + INFO("LXMF Router initialized"); +} + +LXMRouter::~LXMRouter() { + // Unregister from global registry + RouterRegistrySlot* slot = find_router_registry_slot(_delivery_destination.hash()); + if (slot) { + slot->clear(); + } + TRACE("LXMRouter destroyed"); +} + +// ============== Circular Buffer Helpers ============== + +bool LXMRouter::pending_outbound_push(const LXMessage& msg) { + if (_pending_outbound_count >= PENDING_OUTBOUND_SIZE) { + WARNING("Pending outbound queue full, dropping oldest message"); + // Advance tail to drop oldest + _pending_outbound_pool[_pending_outbound_tail] = LXMessage(); // Clear slot + _pending_outbound_tail = (_pending_outbound_tail + 1) % PENDING_OUTBOUND_SIZE; + _pending_outbound_count--; + } + _pending_outbound_pool[_pending_outbound_head] = msg; + _pending_outbound_head = (_pending_outbound_head + 1) % PENDING_OUTBOUND_SIZE; + _pending_outbound_count++; + return true; +} + +LXMessage* LXMRouter::pending_outbound_front() { + if (_pending_outbound_count == 0) return nullptr; + return &_pending_outbound_pool[_pending_outbound_tail]; +} + +bool LXMRouter::pending_outbound_pop(LXMessage& msg) { + if (_pending_outbound_count == 0) return false; + msg = _pending_outbound_pool[_pending_outbound_tail]; + _pending_outbound_pool[_pending_outbound_tail] = LXMessage(); // Clear slot + _pending_outbound_tail = (_pending_outbound_tail + 1) % PENDING_OUTBOUND_SIZE; + _pending_outbound_count--; + return true; +} + +bool LXMRouter::pending_inbound_push(const LXMessage& msg) { + if (_pending_inbound_count >= PENDING_INBOUND_SIZE) { + WARNING("Pending inbound queue full, dropping oldest message"); + // Advance tail to drop oldest + _pending_inbound_pool[_pending_inbound_tail] = LXMessage(); // Clear slot + _pending_inbound_tail = (_pending_inbound_tail + 1) % PENDING_INBOUND_SIZE; + _pending_inbound_count--; + } + _pending_inbound_pool[_pending_inbound_head] = msg; + _pending_inbound_head = (_pending_inbound_head + 1) % PENDING_INBOUND_SIZE; + _pending_inbound_count++; + return true; +} + +LXMessage* LXMRouter::pending_inbound_front() { + if (_pending_inbound_count == 0) return nullptr; + return &_pending_inbound_pool[_pending_inbound_tail]; +} + +bool LXMRouter::pending_inbound_pop(LXMessage& msg) { + if (_pending_inbound_count == 0) return false; + msg = _pending_inbound_pool[_pending_inbound_tail]; + _pending_inbound_pool[_pending_inbound_tail] = LXMessage(); // Clear slot + _pending_inbound_tail = (_pending_inbound_tail + 1) % PENDING_INBOUND_SIZE; + _pending_inbound_count--; + return true; +} + +bool LXMRouter::failed_outbound_push(const LXMessage& msg) { + if (_failed_outbound_count >= FAILED_OUTBOUND_SIZE) { + WARNING("Failed outbound queue full, dropping oldest message"); + // Advance tail to drop oldest + _failed_outbound_pool[_failed_outbound_tail] = LXMessage(); // Clear slot + _failed_outbound_tail = (_failed_outbound_tail + 1) % FAILED_OUTBOUND_SIZE; + _failed_outbound_count--; + } + _failed_outbound_pool[_failed_outbound_head] = msg; + _failed_outbound_head = (_failed_outbound_head + 1) % FAILED_OUTBOUND_SIZE; + _failed_outbound_count++; + return true; +} + +bool LXMRouter::failed_outbound_pop(LXMessage& msg) { + if (_failed_outbound_count == 0) return false; + msg = _failed_outbound_pool[_failed_outbound_tail]; + _failed_outbound_pool[_failed_outbound_tail] = LXMessage(); // Clear slot + _failed_outbound_tail = (_failed_outbound_tail + 1) % FAILED_OUTBOUND_SIZE; + _failed_outbound_count--; + return true; +} + +// ============== End Circular Buffer Helpers ============== + +// Register callbacks +void LXMRouter::register_delivery_callback(DeliveryCallback callback) { + _delivery_callback = callback; + DEBUG("Delivery callback registered"); +} + +void LXMRouter::register_sent_callback(SentCallback callback) { + _sent_callback = callback; + DEBUG("Sent callback registered"); +} + +void LXMRouter::register_delivered_callback(DeliveredCallback callback) { + _delivered_callback = callback; + DEBUG("Delivered callback registered"); +} + +void LXMRouter::register_failed_callback(FailedCallback callback) { + _failed_callback = callback; + DEBUG("Failed callback registered"); +} + +// Queue outbound message +void LXMRouter::handle_outbound(LXMessage& message) { + INFO("Handling outbound LXMF message"); + char buf[128]; + snprintf(buf, sizeof(buf), " Destination: %s", message.destination_hash().toHex().c_str()); + DEBUG(buf); + snprintf(buf, sizeof(buf), " Content size: %zu bytes", message.content().size()); + DEBUG(buf); + + // Pack the message + message.pack(); + + // Check if message fits in a single packet - use OPPORTUNISTIC if so + // OPPORTUNISTIC is simpler (no link needed) and works when identity is known + if (message.packed_size() <= Type::Constants::ENCRYPTED_PACKET_MDU) { + INFO(" Message fits in single packet, will use OPPORTUNISTIC delivery"); + } else { + INFO(" Message too large for single packet, will use DIRECT (link) delivery"); + } + + // Set state to outbound + message.state(Type::Message::OUTBOUND); + + // Add to pending queue + pending_outbound_push(message); + + snprintf(buf, sizeof(buf), "Message queued for delivery (%zu pending)", _pending_outbound_count); + INFO(buf); +} + +// Process outbound queue +void LXMRouter::process_outbound() { + if (_pending_outbound_count == 0) { + return; + } + + // Check backoff timer - don't process if we're in retry delay + double now = Utilities::OS::time(); + if (now < _next_outbound_process_time) { + return; // Wait until retry delay expires + } + + // Process one message per call to avoid blocking + LXMessage* message_ptr = pending_outbound_front(); + if (!message_ptr) return; + LXMessage& message = *message_ptr; + char buf[128]; + + snprintf(buf, sizeof(buf), "Processing outbound message to %s", message.destination_hash().toHex().c_str()); + DEBUG(buf); + + try { + // If propagation-only mode is enabled, send via propagation node + if (_propagation_only) { + DEBUG(" Using PROPAGATED delivery (propagation-only mode)"); + message.set_method(Type::Message::PROPAGATED); + if (send_propagated(message)) { + INFO("Message sent via PROPAGATED delivery"); + if (_sent_callback) { + _sent_callback(message); + } + LXMessage dummy; + pending_outbound_pop(dummy); + } else { + // Propagation not ready yet - wait and retry + DEBUG(" Propagation delivery not ready, will retry..."); + _next_outbound_process_time = now + OUTBOUND_RETRY_DELAY; + } + return; + } + + // Determine delivery method based on message size + bool use_opportunistic = (message.packed_size() <= Type::Constants::ENCRYPTED_PACKET_MDU); + + if (use_opportunistic) { + // OPPORTUNISTIC delivery - send as single encrypted packet + DEBUG(" Using OPPORTUNISTIC delivery (single packet)"); + + // Check if we have a path to the destination + if (!Transport::has_path(message.destination_hash())) { + // Request path from network + INFO(" No path to destination, requesting..."); + Transport::request_path(message.destination_hash()); + _next_outbound_process_time = now + PATH_REQUEST_WAIT; + return; + } + + // Try to recall the destination identity + Identity dest_identity = Identity::recall(message.destination_hash()); + if (!dest_identity) { + // Path exists but identity not cached yet - wait for announce + INFO(" Path exists but identity not known, waiting for announce..."); + _next_outbound_process_time = now + OUTBOUND_RETRY_DELAY; + return; + } + + // Create destination and send packet + if (send_opportunistic(message, dest_identity)) { + INFO("Message sent via OPPORTUNISTIC delivery"); + + // Call sent callback if registered + if (_sent_callback) { + _sent_callback(message); + } + + // Remove from pending queue + LXMessage dummy; + pending_outbound_pop(dummy); + } else { + ERROR("Failed to send OPPORTUNISTIC message"); + message.state(Type::Message::FAILED); + + if (_failed_callback) { + _failed_callback(message); + } + + failed_outbound_push(message); + LXMessage dummy; + pending_outbound_pop(dummy); + } + } else { + // DIRECT delivery - need a link for large messages + DEBUG(" Using DIRECT delivery (via link)"); + + // Check if we have a path to the destination + if (!Transport::has_path(message.destination_hash())) { + // Request path from network + INFO(" No path to destination, requesting..."); + Transport::request_path(message.destination_hash()); + _next_outbound_process_time = now + PATH_REQUEST_WAIT; + return; + } + + // Get or establish link + Link link = get_link_for_destination(message.destination_hash()); + + if (!link) { + WARNING("Failed to establish link for message delivery"); + // Set backoff timer to avoid tight loop + _next_outbound_process_time = now + OUTBOUND_RETRY_DELAY; + snprintf(buf, sizeof(buf), " Will retry in %d seconds", (int)OUTBOUND_RETRY_DELAY); + INFO(buf); + return; + } + + // Check link status + if (link.status() != RNS::Type::Link::ACTIVE) { + DEBUG("Link not yet active, waiting..."); + // Set shorter backoff for pending links + _next_outbound_process_time = now + 1.0; // Check again in 1 second + return; + } + + // Send via link + if (send_via_link(message, link)) { + INFO("Message sent successfully via link"); + + // Call sent callback if registered + if (_sent_callback) { + _sent_callback(message); + } + + // Remove from pending queue + LXMessage dummy; + pending_outbound_pop(dummy); + } else { + ERROR("Failed to send message via link"); + message.state(Type::Message::FAILED); + + // Call failed callback + if (_failed_callback) { + _failed_callback(message); + } + + // Move to failed queue + failed_outbound_push(message); + LXMessage dummy; + pending_outbound_pop(dummy); + } + } + + } catch (const std::exception& e) { + snprintf(buf, sizeof(buf), "Exception processing outbound message: %s", e.what()); + ERROR(buf); + message.state(Type::Message::FAILED); + + // Call failed callback + if (_failed_callback) { + _failed_callback(message); + } + + // Move to failed queue + failed_outbound_push(message); + LXMessage dummy; + pending_outbound_pop(dummy); + } +} + +// Process inbound queue +void LXMRouter::process_inbound() { + if (_pending_inbound_count == 0) { + return; + } + + // Process one message per call + LXMessage* message_ptr = pending_inbound_front(); + if (!message_ptr) return; + LXMessage& message = *message_ptr; + char buf[128]; + + snprintf(buf, sizeof(buf), "Processing inbound message from %s", message.source_hash().toHex().c_str()); + DEBUG(buf); + + try { + // Message is already unpacked and validated in on_packet() + // Just invoke the delivery callback + if (_delivery_callback) { + _delivery_callback(message); + } + + // Remove from pending queue + LXMessage dummy; + pending_inbound_pop(dummy); + + snprintf(buf, sizeof(buf), "Inbound message processed (%zu remaining)", _pending_inbound_count); + INFO(buf); + + } catch (const std::exception& e) { + snprintf(buf, sizeof(buf), "Exception processing inbound message: %s", e.what()); + ERROR(buf); + // Discard message on error + LXMessage dummy; + pending_inbound_pop(dummy); + } +} + +// Announce delivery destination +void LXMRouter::announce(const Bytes& app_data, bool path_response) { + char buf[128]; + snprintf(buf, sizeof(buf), "Announcing LXMF delivery destination: %s", _delivery_destination.hash().toHex().c_str()); + INFO(buf); + + try { + Bytes announce_data; + + // If app_data provided, use it directly + if (app_data && app_data.size() > 0) { + announce_data = app_data; + } + // Otherwise build LXMF-format app_data: [display_name, stamp_cost] + else if (!_display_name.empty()) { + MsgPack::Packer packer; + // LXMF 0.5.0+ format: [display_name_bytes, stamp_cost] + packer.pack(MsgPack::arr_size_t(2)); + // Pack display name as raw bytes + std::vector name_bytes(_display_name.begin(), _display_name.end()); + MsgPack::bin_t name_bin; + name_bin = name_bytes; + packer.pack(name_bin); + // Pack stamp_cost as nil (not used) + packer.packNil(); + + announce_data = Bytes(packer.data(), packer.size()); + snprintf(buf, sizeof(buf), " Built LXMF app_data for display_name: %s", _display_name.c_str()); + DEBUG(buf); + } + + snprintf(buf, sizeof(buf), " Name hash: %s", RNS::Destination::name_hash("lxmf", "delivery").toHex().c_str()); + DEBUG(buf); + snprintf(buf, sizeof(buf), " App_data (%zu bytes): %s", announce_data.size(), + (announce_data.size() > 0 ? announce_data.toHex().c_str() : "(empty)")); + DEBUG(buf); + DEBUG(" Calling _delivery_destination.announce()..."); + _delivery_destination.announce(announce_data, path_response); + _last_announce_time = Utilities::OS::time(); + INFO("Announce sent successfully"); + + } catch (const std::exception& e) { + snprintf(buf, sizeof(buf), "Failed to announce: %s", e.what()); + ERROR(buf); + } +} + +// Set announce interval +void LXMRouter::set_announce_interval(uint32_t interval) { + _announce_interval = interval; + if (interval > 0) { + char buf[64]; + snprintf(buf, sizeof(buf), "Auto-announce interval set to %u seconds", interval); + INFO(buf); + } else { + INFO("Auto-announce disabled"); + } +} + +// Set announce at start +void LXMRouter::set_announce_at_start(bool enabled) { + _announce_at_start = enabled; + DEBUG(enabled ? "Announce at start: enabled" : "Announce at start: disabled"); +} + +// Set display name for announces +void LXMRouter::set_display_name(const std::string& name) { + _display_name = name; + if (!name.empty()) { + char buf[128]; + snprintf(buf, sizeof(buf), "Display name set to: %s", name.c_str()); + INFO(buf); + } +} + +// Clear failed outbound +void LXMRouter::clear_failed_outbound() { + size_t count = _failed_outbound_count; + // Reset circular buffer indices + for (size_t i = 0; i < FAILED_OUTBOUND_SIZE; i++) { + _failed_outbound_pool[i] = LXMessage(); + } + _failed_outbound_head = 0; + _failed_outbound_tail = 0; + _failed_outbound_count = 0; + char buf[64]; + snprintf(buf, sizeof(buf), "Cleared %zu failed outbound messages", count); + INFO(buf); +} + +// Retry failed outbound +void LXMRouter::retry_failed_outbound() { + if (_failed_outbound_count == 0) { + return; + } + + char buf[64]; + snprintf(buf, sizeof(buf), "Retrying %zu failed messages", _failed_outbound_count); + INFO(buf); + + // Move all failed messages back to pending + LXMessage message; + while (failed_outbound_pop(message)) { + message.state(Type::Message::OUTBOUND); + pending_outbound_push(message); + } +} + +// Packet callback - receive LXMF messages +void LXMRouter::on_packet(const Bytes& data, const Packet& packet) { + char buf[128]; + snprintf(buf, sizeof(buf), "Received LXMF message packet (%zu bytes)", data.size()); + INFO(buf); + snprintf(buf, sizeof(buf), " From: %s", packet.destination_hash().toHex().c_str()); + DEBUG(buf); + snprintf(buf, sizeof(buf), " Destination type: %u", (uint8_t)packet.destination_type()); + DEBUG(buf); + + try { + // Build LXMF data based on delivery method (matches Python LXMF exactly) + Bytes lxmf_data; + Type::Message::Method method; + + if (packet.destination_type() != RNS::Type::Destination::LINK) { + // OPPORTUNISTIC delivery: destination hash is NOT in the encrypted data + // We need to prepend it from the packet destination + method = Type::Message::OPPORTUNISTIC; + INFO(" Delivery method: OPPORTUNISTIC (prepending destination hash)"); + lxmf_data = _delivery_destination.hash() + data; + } else { + // DIRECT delivery via Link: data already contains everything + method = Type::Message::DIRECT; + INFO(" Delivery method: DIRECT (data complete)"); + lxmf_data = data; + } + + snprintf(buf, sizeof(buf), " LXMF data size after processing: %zu bytes", lxmf_data.size()); + DEBUG(buf); + + // Unpack LXMF message from packet data + LXMessage message = LXMessage::unpack_from_bytes(lxmf_data, method); + + snprintf(buf, sizeof(buf), " Message hash: %s", message.hash().toHex().c_str()); + DEBUG(buf); + snprintf(buf, sizeof(buf), " Source: %s", message.source_hash().toHex().c_str()); + DEBUG(buf); + snprintf(buf, sizeof(buf), " Content size: %zu bytes", message.content().size()); + DEBUG(buf); + + // Verify destination matches our delivery destination + if (message.destination_hash() != _delivery_destination.hash()) { + WARNING("Message destination mismatch - ignoring"); + return; + } + + // Signature validation + if (!message.signature_validated()) { + WARNING("Message signature not validated"); + snprintf(buf, sizeof(buf), " Unverified reason: %u", (uint8_t)message.unverified_reason()); + DEBUG(buf); + + // For Phase 1 MVP, we'll still accept messages with unknown source + // (signature will be validated later if source identity is learned) + if (message.unverified_reason() != Type::Message::SOURCE_UNKNOWN) { + WARNING(" Rejecting message with invalid signature"); + return; + } + } + + // Stamp enforcement (if enabled) + if (_stamp_cost > 0 && _enforce_stamps) { + if (!message.validate_stamp(_stamp_cost)) { + snprintf(buf, sizeof(buf), " Rejecting message with invalid or missing stamp (required cost=%u)", _stamp_cost); + WARNING(buf); + return; + } + INFO(" Stamp validated"); + } + + // Send delivery proof back to sender (matches Python LXMF packet.prove()) + // Make a non-const copy to call prove() since Packet uses shared_ptr internally + Packet proof_packet = packet; + INFO(" Sending delivery proof"); + proof_packet.prove(); + + // Add to inbound queue + pending_inbound_push(message); + + snprintf(buf, sizeof(buf), "Message queued for processing (%zu pending)", _pending_inbound_count); + INFO(buf); + + } catch (const std::exception& e) { + snprintf(buf, sizeof(buf), "Failed to unpack LXMF message: %s", e.what()); + ERROR(buf); + } +} + +// Get or establish link to destination +Link LXMRouter::get_link_for_destination(const Bytes& destination_hash) { + char buf[128]; + snprintf(buf, sizeof(buf), "Getting link for destination %s", destination_hash.toHex().c_str()); + DEBUG(buf); + + // Check if we already have an active link + DirectLinkSlot* slot = find_direct_link_slot(destination_hash); + if (slot) { + Link& existing_link = slot->link; + + // Check if link is still valid + if (existing_link && existing_link.status() == RNS::Type::Link::ACTIVE) { + DEBUG(" Using existing active link"); + return existing_link; + } else if (existing_link && existing_link.status() == RNS::Type::Link::PENDING) { + // Check if pending link has timed out + double age = Utilities::OS::time() - slot->creation_time; + if (age > LINK_ESTABLISHMENT_TIMEOUT) { + snprintf(buf, sizeof(buf), " Pending link timed out after %ds, removing", (int)age); + WARNING(buf); + slot->clear(); + // Fall through to create new link + } else { + snprintf(buf, sizeof(buf), " Using existing pending link (age: %ds)", (int)age); + DEBUG(buf); + return existing_link; + } + } else { + DEBUG(" Existing link is not active, removing"); + slot->clear(); + } + } + + // Need to establish new link + snprintf(buf, sizeof(buf), " Establishing new link to %s", destination_hash.toHex().c_str()); + INFO(buf); + + try { + // Create destination object for link + // We need to recall the identity if we have it + Identity dest_identity = Identity::recall(destination_hash); + + if (!dest_identity) { + // We need the identity to establish link + WARNING(" Don't have identity for destination - cannot establish link"); + WARNING(" Destination must announce first"); + return Link(RNS::Type::NONE); + } + + // Create destination for link + Destination link_destination( + dest_identity, + RNS::Type::Destination::OUT, + RNS::Type::Destination::SINGLE, + "lxmf", + "delivery" + ); + + // Create link + Link link(link_destination); + + // Register in global registry for link callbacks + RouterRegistrySlot* reg_slot = find_router_registry_slot(link_destination.hash()); + if (!reg_slot) { + reg_slot = find_empty_router_registry_slot(); + if (reg_slot) { + reg_slot->in_use = true; + reg_slot->set_destination_hash(link_destination.hash()); + reg_slot->router = this; + } + } + + // Register link callbacks (using static functions) + link.set_link_established_callback(static_link_established_callback); + link.set_link_closed_callback(static_link_closed_callback); + + // Store link and creation time in fixed pool + DirectLinkSlot* new_slot = find_empty_direct_link_slot(); + if (new_slot) { + new_slot->in_use = true; + new_slot->set_destination_hash(destination_hash); + new_slot->link = link; + new_slot->creation_time = Utilities::OS::time(); + } else { + WARNING(" Direct links pool full - cannot store new link"); + } + + INFO(" Link establishment initiated"); + return link; + + } catch (const std::exception& e) { + snprintf(buf, sizeof(buf), " Failed to establish link: %s", e.what()); + ERROR(buf); + return Link(RNS::Type::NONE); + } +} + +// Send message via link +bool LXMRouter::send_via_link(LXMessage& message, Link& link) { + INFO("Sending LXMF message via link"); + char buf[128]; + snprintf(buf, sizeof(buf), " Message size: %zu bytes", message.packed_size()); + DEBUG(buf); + snprintf(buf, sizeof(buf), " Representation: %u", (uint8_t)message.representation()); + DEBUG(buf); + + try { + // Ensure message is packed + if (!message.packed_size()) { + message.pack(); + } + + // Check that link is active + if (!link || link.status() != RNS::Type::Link::ACTIVE) { + ERROR("Cannot send message - link is not active"); + return false; + } + + message.state(Type::Message::SENDING); + + if (message.representation() == Type::Message::PACKET) { + // Send as single packet over link + snprintf(buf, sizeof(buf), " Sending as single packet (%zu bytes)", message.packed_size()); + INFO(buf); + + Packet packet(link, message.packed()); + packet.send(); + + message.state(Type::Message::SENT); + INFO("Message sent successfully as packet"); + return true; + + } else if (message.representation() == Type::Message::RESOURCE) { + // Send as resource over link with concluded callback + snprintf(buf, sizeof(buf), " Sending as resource (%zu bytes)", message.packed_size()); + INFO(buf); + + // Create resource with our callback to track completion + Resource resource(message.packed(), link, true, true, static_outbound_resource_concluded); + + // Track this resource so we can match the callback to the message + if (resource.hash()) { + OutboundResourceSlot* slot = find_empty_outbound_resource_slot(); + if (slot) { + slot->in_use = true; + slot->set_resource_hash(resource.hash()); + slot->set_message_hash(message.hash()); + snprintf(buf, sizeof(buf), " Tracking resource %.16s for message %.16s", + resource.hash().toHex().c_str(), message.hash().toHex().c_str()); + DEBUG(buf); + } else { + WARNING(" Outbound resources pool full - cannot track resource"); + } + } + + message.state(Type::Message::SENT); + INFO("Message resource transfer initiated"); + return true; + + } else { + ERROR("Unknown message representation"); + message.state(Type::Message::FAILED); + return false; + } + + } catch (const std::exception& e) { + snprintf(buf, sizeof(buf), "Failed to send message: %s", e.what()); + ERROR(buf); + return false; + } +} + +// Send message via OPPORTUNISTIC delivery (single encrypted packet) +bool LXMRouter::send_opportunistic(LXMessage& message, const Identity& dest_identity) { + INFO("Sending LXMF message via OPPORTUNISTIC delivery"); + char buf[128]; + snprintf(buf, sizeof(buf), " Message size: %zu bytes", message.packed_size()); + DEBUG(buf); + + try { + // Create destination object for the remote peer's LXMF delivery + Destination destination( + dest_identity, + RNS::Type::Destination::OUT, + RNS::Type::Destination::SINGLE, + "lxmf", + "delivery" + ); + + // Verify destination hash matches + if (destination.hash() != message.destination_hash()) { + ERROR("Destination hash mismatch!"); + snprintf(buf, sizeof(buf), " Expected: %s", message.destination_hash().toHex().c_str()); + DEBUG(buf); + snprintf(buf, sizeof(buf), " Got: %s", destination.hash().toHex().c_str()); + DEBUG(buf); + return false; + } + + // For OPPORTUNISTIC, we strip the destination hash from the packed data + // since it's already in the packet header + Bytes packet_data = message.packed().mid(Type::Constants::DESTINATION_LENGTH); + snprintf(buf, sizeof(buf), " Packet data size: %zu bytes", packet_data.size()); + DEBUG(buf); + + // Create and send packet + Packet packet(destination, packet_data, RNS::Type::Packet::DATA); + PacketReceipt receipt = packet.send(); + + // Register proof callback to track delivery confirmation + if (receipt) { + receipt.set_delivery_callback(static_proof_callback); + PendingProofSlot* slot = find_empty_pending_proof_slot(); + if (slot) { + slot->in_use = true; + slot->set_packet_hash(receipt.hash()); + slot->set_message_hash(message.hash()); + snprintf(buf, sizeof(buf), " Registered proof callback for packet %.16s...", receipt.hash().toHex().c_str()); + DEBUG(buf); + } else { + WARNING(" Pending proofs pool full - cannot track delivery proof"); + } + } + + message.state(Type::Message::SENT); + INFO(" OPPORTUNISTIC packet sent"); + + return true; + + } catch (const std::exception& e) { + snprintf(buf, sizeof(buf), "Failed to send OPPORTUNISTIC message: %s", e.what()); + ERROR(buf); + return false; + } +} + +// Handle delivery proof for DIRECT messages +void LXMRouter::handle_direct_proof(const Bytes& message_hash) { + char buf[128]; + snprintf(buf, sizeof(buf), "Processing DIRECT delivery proof for message %.16s...", message_hash.toHex().c_str()); + INFO(buf); + + // Track notified routers to avoid duplicates (max ROUTER_REGISTRY_SIZE) + LXMRouter* notified_routers[ROUTER_REGISTRY_SIZE]; + size_t notified_count = 0; + + // Call delivered callback for all unique routers + for (size_t i = 0; i < ROUTER_REGISTRY_SIZE; i++) { + if (_router_registry_pool[i].in_use) { + LXMRouter* router = _router_registry_pool[i].router; + // Check if already notified + bool already_notified = false; + for (size_t j = 0; j < notified_count; j++) { + if (notified_routers[j] == router) { + already_notified = true; + break; + } + } + if (!already_notified && router && router->_delivered_callback) { + notified_routers[notified_count++] = router; + Bytes empty_hash; + LXMessage msg(empty_hash, empty_hash); + msg.hash(message_hash); + msg.state(Type::Message::DELIVERED); + router->_delivered_callback(msg); + } + } + } +} + +// Link established callback +void LXMRouter::on_link_established(const Link& link) { + char buf[128]; + snprintf(buf, sizeof(buf), "Link established to %s", link.destination().hash().toHex().c_str()); + INFO(buf); + // Link is now active, process_outbound() will pick it up + // Note: Delivery confirmation comes from Resource completed callback +} + +// Link closed callback +void LXMRouter::on_link_closed(const Link& link) { + char buf[128]; + snprintf(buf, sizeof(buf), "Link closed to %s", link.destination().hash().toHex().c_str()); + INFO(buf); + + // Remove from active links + DirectLinkSlot* slot = find_direct_link_slot(link.destination().hash()); + if (slot) { + slot->clear(); + DEBUG(" Removed link from cache"); + } +} + +// Incoming link established callback (for DIRECT delivery to our destination) +void LXMRouter::on_incoming_link_established(Link& link) { + INFO("Incoming link established from remote peer"); + char buf[128]; + snprintf(buf, sizeof(buf), " Link ID: %s", link.link_id().toHex().c_str()); + DEBUG(buf); + + // Set up resource concluded callback to receive LXMF messages over this link + link.set_resource_concluded_callback(static_resource_concluded_callback); + DEBUG(" Resource callback registered for incoming LXMF messages"); +} + +// Resource concluded callback (LXMF message received via DIRECT delivery) +void LXMRouter::on_resource_concluded(const RNS::Resource& resource) { + char buf[128]; + snprintf(buf, sizeof(buf), "Resource concluded, status=%d", (int)resource.status()); + DEBUG(buf); + + if (resource.status() != RNS::Type::Resource::COMPLETE) { + snprintf(buf, sizeof(buf), "Resource transfer failed with status %d", (int)resource.status()); + WARNING(buf); + return; + } + + // Get the resource data - this is the LXMF message + Bytes data = resource.data(); + snprintf(buf, sizeof(buf), "Received LXMF message via DIRECT delivery (%zu bytes)", data.size()); + INFO(buf); + + try { + // DIRECT delivery via resource: data is the full packed LXMF message + // Format: destination_hash + source_hash + signature + payload + // (same as OPPORTUNISTIC, just delivered via link resource instead of single packet) + + // Unpack the LXMF message + LXMessage message = LXMessage::unpack_from_bytes(data, Type::Message::DIRECT); + + snprintf(buf, sizeof(buf), " Message hash: %s", message.hash().toHex().c_str()); + DEBUG(buf); + snprintf(buf, sizeof(buf), " Source: %s", message.source_hash().toHex().c_str()); + DEBUG(buf); + snprintf(buf, sizeof(buf), " Content size: %zu bytes", message.content().size()); + DEBUG(buf); + + // Verify destination matches + if (message.destination_hash() != _delivery_destination.hash()) { + WARNING("Message destination mismatch - ignoring"); + return; + } + + // Signature validation (same as on_packet) + if (!message.signature_validated()) { + WARNING("Message signature not validated"); + snprintf(buf, sizeof(buf), " Unverified reason: %u", (uint8_t)message.unverified_reason()); + DEBUG(buf); + + // Accept messages with unknown source (signature validated later) + if (message.unverified_reason() != Type::Message::SOURCE_UNKNOWN) { + WARNING(" Rejecting message with invalid signature"); + return; + } + } + + // Stamp enforcement (if enabled) + if (_stamp_cost > 0 && _enforce_stamps) { + if (!message.validate_stamp(_stamp_cost)) { + snprintf(buf, sizeof(buf), " Rejecting message with invalid or missing stamp (required cost=%u)", _stamp_cost); + WARNING(buf); + return; + } + INFO(" Stamp validated"); + } + + // Note: We don't need to send a custom delivery proof here. + // The sender gets delivery confirmation when the Resource completes + // (via RESOURCE_PRF from RNS layer), which triggers their callback. + + // Add to inbound queue for processing + pending_inbound_push(message); + INFO(" Message queued for delivery"); + + } catch (const std::exception& e) { + snprintf(buf, sizeof(buf), "Failed to process DIRECT message: %s", e.what()); + ERROR(buf); + } +} + +// ============== Propagation Node Support ============== + +void LXMRouter::set_propagation_node_manager(PropagationNodeManager* manager) { + _propagation_manager = manager; + INFO("Propagation node manager set"); +} + +void LXMRouter::set_outbound_propagation_node(const Bytes& node_hash) { + if (node_hash.size() == 0) { + _outbound_propagation_node = {}; + _outbound_propagation_link = Link(RNS::Type::NONE); + INFO("Cleared outbound propagation node"); + return; + } + + // Check if changing to a different node + if (_outbound_propagation_node != node_hash) { + // Tear down existing link if any + if (_outbound_propagation_link && _outbound_propagation_link.status() != RNS::Type::Link::CLOSED) { + _outbound_propagation_link.teardown(); + } + _outbound_propagation_link = Link(RNS::Type::NONE); + } + + _outbound_propagation_node = node_hash; + char buf[64]; + snprintf(buf, sizeof(buf), "Set outbound propagation node to %.16s...", node_hash.toHex().c_str()); + INFO(buf); +} + +void LXMRouter::register_sync_complete_callback(SyncCompleteCallback callback) { + _sync_complete_callback = callback; + DEBUG("Sync complete callback registered"); +} + +// Static callback for outbound propagation resource +void LXMRouter::static_propagation_resource_concluded(const Resource& resource) { + Bytes resource_hash = resource.hash(); + char buf[128]; + snprintf(buf, sizeof(buf), "Propagation resource concluded: %.16s...", resource_hash.toHex().c_str()); + DEBUG(buf); + snprintf(buf, sizeof(buf), " Status: %d", (int)resource.status()); + DEBUG(buf); + + PropResourceSlot* slot = find_prop_resource_slot(resource_hash); + if (!slot) { + DEBUG(" Resource not in pending propagation map"); + return; + } + + Bytes message_hash = slot->message_hash_bytes(); + + if (resource.status() == RNS::Type::Resource::COMPLETE) { + snprintf(buf, sizeof(buf), "PROPAGATED delivery to node confirmed for message %.16s...", message_hash.toHex().c_str()); + INFO(buf); + + // For PROPAGATED, "delivered" means delivered to propagation node, not final recipient + // We mark it as SENT (not DELIVERED) to indicate it's on the propagation network + LXMRouter* notified_routers[ROUTER_REGISTRY_SIZE]; + size_t notified_count = 0; + + for (size_t i = 0; i < ROUTER_REGISTRY_SIZE; i++) { + if (_router_registry_pool[i].in_use) { + LXMRouter* router = _router_registry_pool[i].router; + bool already_notified = false; + for (size_t j = 0; j < notified_count; j++) { + if (notified_routers[j] == router) { + already_notified = true; + break; + } + } + if (!already_notified && router && router->_sent_callback) { + notified_routers[notified_count++] = router; + Bytes empty_hash; + LXMessage msg(empty_hash, empty_hash); + msg.hash(message_hash); + msg.state(Type::Message::SENT); + router->_sent_callback(msg); + } + } + } + } else { + snprintf(buf, sizeof(buf), "PROPAGATED resource transfer failed with status %d", (int)resource.status()); + WARNING(buf); + } + + slot->clear(); +} + +bool LXMRouter::send_propagated(LXMessage& message) { + INFO("Sending LXMF message via PROPAGATED delivery"); + char buf[128]; + + // Get propagation node + Bytes prop_node = _outbound_propagation_node; + if (prop_node.size() == 0 && _propagation_manager) { + DEBUG(" Looking for propagation node via manager..."); + auto nodes = _propagation_manager->get_nodes(); + snprintf(buf, sizeof(buf), " Manager has %zu nodes", nodes.size()); + DEBUG(buf); + prop_node = _propagation_manager->get_effective_node(); + } + + if (prop_node.size() == 0) { + WARNING("No propagation node available for PROPAGATED delivery"); + return false; + } + + snprintf(buf, sizeof(buf), " Using propagation node: %.16s...", prop_node.toHex().c_str()); + DEBUG(buf); + + // Check/establish link to propagation node + if (!_outbound_propagation_link || + _outbound_propagation_link.status() == RNS::Type::Link::CLOSED) { + + // Check if we have a path + if (!Transport::has_path(prop_node)) { + INFO(" No path to propagation node, requesting..."); + Transport::request_path(prop_node); + return false; // Will retry next cycle + } + + // Recall identity for propagation node + Identity node_identity = Identity::recall(prop_node); + if (!node_identity) { + INFO(" Propagation node identity not known, waiting for announce..."); + return false; + } + + // Create destination for propagation node + Destination prop_dest( + node_identity, + RNS::Type::Destination::OUT, + RNS::Type::Destination::SINGLE, + "lxmf", + "propagation" + ); + + // Create link with established callback + _outbound_propagation_link = Link(prop_dest); + INFO(" Establishing link to propagation node..."); + return false; // Will retry when link established + } + + // Check if link is active + if (_outbound_propagation_link.status() != RNS::Type::Link::ACTIVE) { + DEBUG(" Propagation link not yet active, waiting..."); + return false; // Will retry + } + + // Generate propagation stamp if required by node + if (_propagation_manager) { + auto node_info = _propagation_manager->get_node(prop_node); + if (node_info && node_info.stamp_cost > 0) { + snprintf(buf, sizeof(buf), " Generating propagation stamp (cost=%u)...", node_info.stamp_cost); + DEBUG(buf); + Bytes stamp = message.generate_propagation_stamp(node_info.stamp_cost); + if (stamp.size() == 0) { + WARNING(" Failed to generate propagation stamp, sending anyway"); + } + } + } + + // Pack message for propagation + Bytes prop_packed = message.pack_propagated(); + if (!prop_packed || prop_packed.size() == 0) { + ERROR(" Failed to pack message for propagation"); + return false; + } + + snprintf(buf, sizeof(buf), " Propagated message size: %zu bytes", prop_packed.size()); + DEBUG(buf); + + // Send via resource with callback + Resource resource(prop_packed, _outbound_propagation_link, true, true, static_propagation_resource_concluded); + + // Track this resource + if (resource.hash()) { + PropResourceSlot* slot = find_empty_prop_resource_slot(); + if (slot) { + slot->in_use = true; + slot->set_resource_hash(resource.hash()); + slot->set_message_hash(message.hash()); + snprintf(buf, sizeof(buf), " Tracking propagation resource %.16s", resource.hash().toHex().c_str()); + DEBUG(buf); + } else { + WARNING(" Propagation resources pool full - cannot track resource"); + } + } + + message.state(Type::Message::SENDING); + INFO(" PROPAGATED resource transfer initiated"); + return true; +} + +void LXMRouter::request_messages_from_propagation_node() { + if (_sync_state != PR_IDLE && _sync_state != PR_COMPLETE && _sync_state != PR_FAILED) { + char buf[64]; + snprintf(buf, sizeof(buf), "Sync already in progress (state=%d)", _sync_state); + WARNING(buf); + return; + } + + // Get propagation node + Bytes prop_node = _outbound_propagation_node; + if (!prop_node && _propagation_manager) { + prop_node = _propagation_manager->get_effective_node(); + } + + if (!prop_node) { + WARNING("No propagation node available for sync"); + _sync_state = PR_FAILED; + return; + } + + char buf[64]; + snprintf(buf, sizeof(buf), "Requesting messages from propagation node %.16s...", prop_node.toHex().c_str()); + INFO(buf); + _sync_progress = 0.0f; + + // Check if link exists and is active + if (_outbound_propagation_link && _outbound_propagation_link.status() == RNS::Type::Link::ACTIVE) { + _sync_state = PR_LINK_ESTABLISHED; + + // TODO: Implement link.identify() and link.request() for full sync protocol + // For now, we log that sync would happen here + INFO(" Link active - sync protocol not yet implemented"); + INFO(" (Requires Link.identify() and Link.request() support)"); + + _sync_state = PR_COMPLETE; + _sync_progress = 1.0f; + if (_sync_complete_callback) { + _sync_complete_callback(0); + } + } else { + // Need to establish link first + if (!Transport::has_path(prop_node)) { + INFO(" No path to propagation node, requesting..."); + Transport::request_path(prop_node); + _sync_state = PR_PATH_REQUESTED; + } else { + Identity node_identity = Identity::recall(prop_node); + if (!node_identity) { + INFO(" Propagation node identity not known"); + _sync_state = PR_FAILED; + return; + } + + Destination prop_dest( + node_identity, + RNS::Type::Destination::OUT, + RNS::Type::Destination::SINGLE, + "lxmf", + "propagation" + ); + + _outbound_propagation_link = Link(prop_dest); + _sync_state = PR_LINK_ESTABLISHING; + INFO(" Establishing link for sync..."); + } + } +} + +void LXMRouter::on_message_list_response(const Bytes& response) { + // TODO: Implement when Link.request() is available + DEBUG("on_message_list_response: Not yet implemented"); +} + +void LXMRouter::on_message_get_response(const Bytes& response) { + // TODO: Implement when Link.request() is available + DEBUG("on_message_get_response: Not yet implemented"); +} + +void LXMRouter::process_propagated_lxmf(const Bytes& lxmf_data) { + // lxmf_data format: dest_hash (16 bytes) + encrypted_content + if (lxmf_data.size() < Type::Constants::DESTINATION_LENGTH) { + WARNING("Propagated LXMF data too short"); + return; + } + + Bytes dest_hash = lxmf_data.left(Type::Constants::DESTINATION_LENGTH); + + // Verify this is for us + if (dest_hash != _delivery_destination.hash()) { + DEBUG("Received propagated message not addressed to us"); + return; + } + + // Decrypt the content + Bytes encrypted = lxmf_data.mid(Type::Constants::DESTINATION_LENGTH); + Bytes decrypted = _identity.decrypt(encrypted); + + if (!decrypted || decrypted.size() == 0) { + WARNING("Failed to decrypt propagated message"); + return; + } + + // Reconstruct full LXMF data: dest_hash + decrypted + Bytes full_data; + full_data << dest_hash << decrypted; + + try { + LXMessage message = LXMessage::unpack_from_bytes(full_data, Type::Message::PROPAGATED); + + // Check if message was unpacked successfully (has valid hash) + if (message.hash().size() > 0) { + // Track transient ID to avoid re-downloading + Bytes transient_id = Identity::full_hash(lxmf_data); + transient_ids_add(transient_id); + + // Queue for delivery + pending_inbound_push(message); + INFO("Propagated message queued for delivery"); + } + } catch (const std::exception& e) { + char buf[128]; + snprintf(buf, sizeof(buf), "Failed to unpack propagated message: %s", e.what()); + ERROR(buf); + } +} diff --git a/src-shim/LXMF/LXMRouter.h b/src-shim/LXMF/LXMRouter.h new file mode 100644 index 00000000..a3b883a6 --- /dev/null +++ b/src-shim/LXMF/LXMRouter.h @@ -0,0 +1,679 @@ +#pragma once + +#include "Type.h" +#include "LXMessage.h" +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +namespace LXMF { + + // Forward declarations + class PropagationNodeManager; + + /** + * @brief LXMF Router - Message delivery orchestration + * + * Manages message queues, link establishment, and delivery for LXMF messages. + * Supports DIRECT delivery method (via established links) for Phase 1 MVP. + * + * Usage: + * LXMRouter router(identity, "/path/to/storage"); + * router.register_delivery_callback([](LXMessage& msg) { + * // Handle received message + * }); + * router.announce(); // Announce delivery destination + * + * // Send message + * LXMessage msg(dest, source, content); + * router.handle_outbound(msg); + * + * // Process queues periodically + * loop() { + * router.process_outbound(); + * router.process_inbound(); + * } + */ + class LXMRouter { + + public: + using Ptr = std::shared_ptr; + + /** + * @brief Callback for message delivery (incoming messages) + * @param message The delivered message + */ + using DeliveryCallback = std::function; + + /** + * @brief Callback for message sent confirmation + * @param message The sent message + */ + using SentCallback = std::function; + + /** + * @brief Callback for message delivery confirmation + * @param message The delivered message + */ + using DeliveredCallback = std::function; + + /** + * @brief Callback for message failure + * @param message The failed message + */ + using FailedCallback = std::function; + + /** + * @brief Callback for sync completion + * @param messages_received Number of messages received from propagation node + */ + using SyncCompleteCallback = std::function; + + /** + * @brief Propagation sync state + */ + enum PropagationSyncState : uint8_t { + PR_IDLE = 0x00, + PR_PATH_REQUESTED = 0x01, + PR_LINK_ESTABLISHING = 0x02, + PR_LINK_ESTABLISHED = 0x03, + PR_REQUEST_SENT = 0x04, + PR_RECEIVING = 0x05, + PR_COMPLETE = 0x06, + PR_FAILED = 0x07 + }; + + public: + /** + * @brief Construct LXMF Router + * + * @param identity Local identity for sending/receiving messages + * @param storage_path Path for message persistence (optional) + * @param announce_at_start Announce delivery destination on startup (default: false) + */ + LXMRouter( + const RNS::Identity& identity, + const std::string& storage_path = "", + bool announce_at_start = false + ); + + ~LXMRouter(); + + public: + /** + * @brief Register callback for incoming message delivery + * + * Called when a message is received and validated. + * + * @param callback Function to call with delivered messages + */ + void register_delivery_callback(DeliveryCallback callback); + + /** + * @brief Register callback for message sent confirmation + * + * Called when a message has been sent (packet transmitted or resource started). + * + * @param callback Function to call when message is sent + */ + void register_sent_callback(SentCallback callback); + + /** + * @brief Register callback for message delivery confirmation + * + * Called when remote confirms message delivery. + * + * @param callback Function to call when message is delivered + */ + void register_delivered_callback(DeliveredCallback callback); + + /** + * @brief Register callback for message failure + * + * Called when message sending fails (link timeout, etc). + * + * @param callback Function to call when message fails + */ + void register_failed_callback(FailedCallback callback); + + /** + * @brief Queue an outbound message for delivery + * + * Message will be sent via DIRECT method (link-based delivery). + * Link will be established if needed. + * + * @param message Message to send + */ + void handle_outbound(LXMessage& message); + + /** + * @brief Process outbound message queue + * + * Call periodically (e.g., in main loop) to process pending outbound messages. + * Establishes links, sends messages, handles retries. + */ + void process_outbound(); + + /** + * @brief Process inbound message queue + * + * Call periodically to process received messages and invoke delivery callbacks. + */ + void process_inbound(); + + /** + * @brief Announce the delivery destination + * + * Sends an announce for the LXMF delivery destination so others can + * discover this node and send messages to it. + * + * @param app_data Optional application data to include in announce + * @param path_response Optional path response + */ + void announce(const RNS::Bytes& app_data = {}, bool path_response = false); + + /** + * @brief Set announce interval + * + * @param interval Seconds between announces (0 = disable auto-announce) + */ + void set_announce_interval(uint32_t interval); + + /** + * @brief Enable/disable auto-announce on startup + * + * @param enabled True to announce on startup + */ + void set_announce_at_start(bool enabled); + + /** + * @brief Set display name for announces + * + * @param name Display name to include in announces + */ + void set_display_name(const std::string& name); + + /** + * @brief Get current display name + * + * @return Display name string + */ + inline const std::string& display_name() const { return _display_name; } + + /** + * @brief Get the delivery destination + * + * @return Destination for receiving LXMF messages + */ + inline const RNS::Destination& delivery_destination() const { return _delivery_destination; } + + /** + * @brief Get the local identity + * + * @return Identity used by this router + */ + inline const RNS::Identity& identity() const { return _identity; } + + /** + * @brief Get pending outbound message count + * + * @return Number of messages waiting to be sent + */ + inline size_t pending_outbound_count() const { return _pending_outbound_count; } + + /** + * @brief Get pending inbound message count + * + * @return Number of messages waiting to be processed + */ + inline size_t pending_inbound_count() const { return _pending_inbound_count; } + + /** + * @brief Get failed outbound message count + * + * @return Number of failed messages + */ + inline size_t failed_outbound_count() const { return _failed_outbound_count; } + + /** + * @brief Clear failed outbound messages + */ + void clear_failed_outbound(); + + /** + * @brief Retry failed outbound messages + */ + void retry_failed_outbound(); + + // ============== Propagation Node Support ============== + + /** + * @brief Set the propagation node manager + * + * @param manager Pointer to PropagationNodeManager (not owned) + */ + void set_propagation_node_manager(PropagationNodeManager* manager); + + /** + * @brief Set the outbound propagation node + * + * @param node_hash Destination hash of the propagation node + */ + void set_outbound_propagation_node(const RNS::Bytes& node_hash); + + /** + * @brief Get the current outbound propagation node + * + * @return Destination hash of the propagation node (or empty) + */ + RNS::Bytes get_outbound_propagation_node() const { return _outbound_propagation_node; } + + /** + * @brief Enable/disable fallback to PROPAGATED delivery + * + * When enabled, messages that fail DIRECT/OPPORTUNISTIC delivery will + * automatically be retried via a propagation node. + * + * @param enabled True to enable fallback + */ + void set_fallback_to_propagation(bool enabled) { _fallback_to_propagation = enabled; } + + /** + * @brief Check if fallback to propagation is enabled + * + * @return True if fallback is enabled + */ + bool fallback_to_propagation() const { return _fallback_to_propagation; } + + /** + * @brief Enable/disable propagation-only mode + * + * When enabled, all messages are sent via propagation node only. + * DIRECT and OPPORTUNISTIC delivery are skipped. + * + * @param enabled True to enable propagation-only mode + */ + void set_propagation_only(bool enabled) { _propagation_only = enabled; } + + /** + * @brief Check if propagation-only mode is enabled + * + * @return True if propagation-only mode is enabled + */ + bool propagation_only() const { return _propagation_only; } + + /** + * @brief Request messages from the propagation node + * + * Initiates a sync with the configured/selected propagation node to + * retrieve any pending messages. + */ + void request_messages_from_propagation_node(); + + /** + * @brief Get the current sync state + * + * @return Current propagation sync state + */ + PropagationSyncState get_sync_state() const { return _sync_state; } + + /** + * @brief Get the current sync progress + * + * @return Progress from 0.0 to 1.0 + */ + float get_sync_progress() const { return _sync_progress; } + + /** + * @brief Register callback for sync completion + * + * @param callback Function to call when sync completes + */ + void register_sync_complete_callback(SyncCompleteCallback callback); + + // ============== End Propagation Node Support ============== + + // ============== Stamp Enforcement ============== + + /** + * @brief Set the required stamp cost for incoming messages + * + * Messages without a valid stamp meeting this cost will be rejected + * when enforcement is enabled. + * + * @param cost Required number of leading zero bits (0 = no stamp required) + */ + void set_stamp_cost(uint8_t cost) { _stamp_cost = cost; } + + /** + * @brief Get the current stamp cost requirement + * + * @return Required stamp cost + */ + uint8_t stamp_cost() const { return _stamp_cost; } + + /** + * @brief Enable stamp enforcement + * + * When enabled, incoming messages must have a valid stamp meeting + * the configured cost or they will be dropped. + */ + void enforce_stamps() { _enforce_stamps = true; } + + /** + * @brief Disable stamp enforcement + * + * Incoming messages will be accepted regardless of stamp validity. + */ + void ignore_stamps() { _enforce_stamps = false; } + + /** + * @brief Check if stamp enforcement is enabled + * + * @return True if stamps are enforced + */ + bool stamps_enforced() const { return _enforce_stamps; } + + // ============== End Stamp Enforcement ============== + + /** + * @brief Packet callback for receiving LXMF messages + * + * Called by RNS when a packet is received for the delivery destination. + * NOTE: Public for static callback access, not intended for direct use. + * + * @param data Packet data + * @param packet The received packet + */ + void on_packet(const RNS::Bytes& data, const RNS::Packet& packet); + + /** + * @brief Handle link established callback + * + * NOTE: Public for static callback access, not intended for direct use. + * + * @param link The established link + */ + void on_link_established(const RNS::Link& link); + + /** + * @brief Handle link closed callback + * + * NOTE: Public for static callback access, not intended for direct use. + * + * @param link The closed link + */ + void on_link_closed(const RNS::Link& link); + + /** + * @brief Handle incoming link established to our delivery destination + * + * Called when a remote peer establishes a link to send us DIRECT messages. + * Sets up resource callbacks to receive LXMF messages. + * + * @param link The incoming link (non-const to set callbacks) + */ + void on_incoming_link_established(RNS::Link& link); + + /** + * @brief Handle resource concluded on incoming link + * + * Called when an LXMF message resource transfer completes. + * Unpacks and queues the message for delivery. + * + * @param resource The completed resource containing LXMF message + */ + void on_resource_concluded(const RNS::Resource& resource); + + private: + /** + * @brief Establish a link to a destination + * + * Creates or retrieves an existing link for DIRECT delivery. + * + * @param destination_hash Hash of destination to link to + * @return Link object (or empty if failed) + */ + RNS::Link get_link_for_destination(const RNS::Bytes& destination_hash); + + /** + * @brief Send message via existing link + * + * @param message Message to send + * @param link Link to send over + * @return True if send initiated successfully + */ + bool send_via_link(LXMessage& message, RNS::Link& link); + + /** + * @brief Send message via OPPORTUNISTIC delivery (single packet) + * + * @param message Message to send + * @param dest_identity Destination identity (for encryption) + * @return True if send initiated successfully + */ + bool send_opportunistic(LXMessage& message, const RNS::Identity& dest_identity); + + /** + * @brief Send message via PROPAGATED delivery + * + * @param message Message to send + * @return True if send initiated successfully + */ + bool send_propagated(LXMessage& message); + + /** + * @brief Handle message list response from propagation node + */ + void on_message_list_response(const RNS::Bytes& response); + + /** + * @brief Handle message get response from propagation node + */ + void on_message_get_response(const RNS::Bytes& response); + + /** + * @brief Process received propagated LXMF data + */ + void process_propagated_lxmf(const RNS::Bytes& lxmf_data); + + /** + * @brief Static callback for outbound propagation resource + */ + static void static_propagation_resource_concluded(const RNS::Resource& resource); + + // Circular buffer helpers for message queues + bool pending_outbound_push(const LXMessage& msg); + bool pending_outbound_pop(LXMessage& msg); + LXMessage* pending_outbound_front(); + bool pending_inbound_push(const LXMessage& msg); + bool pending_inbound_pop(LXMessage& msg); + LXMessage* pending_inbound_front(); + bool failed_outbound_push(const LXMessage& msg); + bool failed_outbound_pop(LXMessage& msg); + + private: + // Core components + RNS::Identity _identity; // Local identity + RNS::Destination _delivery_destination; // For receiving messages + std::string _storage_path; // Storage path for persistence + + // Message queues as fixed circular buffers (zero heap fragmentation) + static constexpr size_t PENDING_OUTBOUND_SIZE = 16; + LXMessage _pending_outbound_pool[PENDING_OUTBOUND_SIZE]; + size_t _pending_outbound_head = 0; + size_t _pending_outbound_tail = 0; + size_t _pending_outbound_count = 0; + + static constexpr size_t PENDING_INBOUND_SIZE = 16; + LXMessage _pending_inbound_pool[PENDING_INBOUND_SIZE]; + size_t _pending_inbound_head = 0; + size_t _pending_inbound_tail = 0; + size_t _pending_inbound_count = 0; + + static constexpr size_t FAILED_OUTBOUND_SIZE = 8; + LXMessage _failed_outbound_pool[FAILED_OUTBOUND_SIZE]; + size_t _failed_outbound_head = 0; + size_t _failed_outbound_tail = 0; + size_t _failed_outbound_count = 0; + + // Link management for DIRECT delivery - fixed pool (zero heap fragmentation) + static constexpr size_t DIRECT_LINKS_SIZE = 8; + static constexpr size_t DEST_HASH_SIZE = 16; // Truncated hash size + struct DirectLinkSlot { + bool in_use = false; + uint8_t destination_hash[DEST_HASH_SIZE]; + RNS::Link link; + double creation_time = 0; + RNS::Bytes destination_hash_bytes() const { return RNS::Bytes(destination_hash, DEST_HASH_SIZE); } + void set_destination_hash(const RNS::Bytes& b) { + size_t len = std::min(b.size(), DEST_HASH_SIZE); + memcpy(destination_hash, b.data(), len); + if (len < DEST_HASH_SIZE) memset(destination_hash + len, 0, DEST_HASH_SIZE - len); + } + bool destination_hash_equals(const RNS::Bytes& b) const { + if (b.size() != DEST_HASH_SIZE) return false; + return memcmp(destination_hash, b.data(), DEST_HASH_SIZE) == 0; + } + void clear() { + in_use = false; + memset(destination_hash, 0, DEST_HASH_SIZE); + link = RNS::Link(RNS::Type::NONE); + creation_time = 0; + } + }; + DirectLinkSlot _direct_links_pool[DIRECT_LINKS_SIZE]; + DirectLinkSlot* find_direct_link_slot(const RNS::Bytes& hash); + DirectLinkSlot* find_empty_direct_link_slot(); + size_t direct_links_count(); + static constexpr double LINK_ESTABLISHMENT_TIMEOUT = 30.0; // Seconds to wait for pending links + + // Proof tracking for delivery confirmation - fixed pool (zero heap fragmentation) + // Maps packet hash -> message hash so we can update message state when proof arrives + // Fixed arrays eliminate ~0.8KB Bytes metadata overhead (16 slots × 2 Bytes × 24 bytes) + static constexpr size_t PENDING_PROOFS_SIZE = 16; + static constexpr size_t HASH_SIZE = 32; // SHA256 hash size + struct PendingProofSlot { + bool in_use = false; + uint8_t packet_hash[HASH_SIZE]; + uint8_t message_hash[HASH_SIZE]; + RNS::Bytes packet_hash_bytes() const { return RNS::Bytes(packet_hash, HASH_SIZE); } + RNS::Bytes message_hash_bytes() const { return RNS::Bytes(message_hash, HASH_SIZE); } + void set_packet_hash(const RNS::Bytes& b) { + size_t len = std::min(b.size(), HASH_SIZE); + memcpy(packet_hash, b.data(), len); + if (len < HASH_SIZE) memset(packet_hash + len, 0, HASH_SIZE - len); + } + void set_message_hash(const RNS::Bytes& b) { + size_t len = std::min(b.size(), HASH_SIZE); + memcpy(message_hash, b.data(), len); + if (len < HASH_SIZE) memset(message_hash + len, 0, HASH_SIZE - len); + } + bool packet_hash_equals(const RNS::Bytes& b) const { + if (b.size() != HASH_SIZE) return false; + return memcmp(packet_hash, b.data(), HASH_SIZE) == 0; + } + void clear() { + in_use = false; + memset(packet_hash, 0, HASH_SIZE); + memset(message_hash, 0, HASH_SIZE); + } + }; + static PendingProofSlot _pending_proofs_pool[PENDING_PROOFS_SIZE]; + static PendingProofSlot* find_pending_proof_slot(const RNS::Bytes& hash); + static PendingProofSlot* find_empty_pending_proof_slot(); + static void static_proof_callback(const RNS::PacketReceipt& receipt); + + public: + // Handle delivery proof for DIRECT messages (called from link packet callback) + static void handle_direct_proof(const RNS::Bytes& message_hash); + + private: + + // Callbacks + DeliveryCallback _delivery_callback; + SentCallback _sent_callback; + DeliveredCallback _delivered_callback; + FailedCallback _failed_callback; + + // Announce settings + uint32_t _announce_interval = 0; // Seconds (0 = disabled) + bool _announce_at_start = true; // Announce on startup + double _last_announce_time = 0.0; // Last announce timestamp + std::string _display_name; // Display name for announces + + // Internal state + bool _initialized = false; + + // Retry backoff + double _next_outbound_process_time = 0.0; // Next time to process outbound queue + static constexpr double OUTBOUND_RETRY_DELAY = 5.0; // Seconds between retries + static constexpr double PATH_REQUEST_WAIT = 3.0; // Seconds to wait after path request + + // Propagation node support + PropagationNodeManager* _propagation_manager = nullptr; + RNS::Bytes _outbound_propagation_node; + RNS::Link _outbound_propagation_link{RNS::Type::NONE}; + bool _fallback_to_propagation = true; + bool _propagation_only = false; + + // Propagation sync state + PropagationSyncState _sync_state = PR_IDLE; + float _sync_progress = 0.0f; + SyncCompleteCallback _sync_complete_callback; + + // Locally delivered transient IDs circular buffer (zero heap fragmentation) + // Note: Keep as Bytes for now - converting to static array causes issues + static constexpr size_t TRANSIENT_IDS_SIZE = 64; + RNS::Bytes _transient_ids_buffer[TRANSIENT_IDS_SIZE]; + size_t _transient_ids_head = 0; + size_t _transient_ids_count = 0; + bool transient_ids_contains(const RNS::Bytes& id); + void transient_ids_add(const RNS::Bytes& id); + + // Track outbound propagation resources - fixed pool (zero heap fragmentation) + // Fixed arrays eliminate ~0.8KB Bytes metadata overhead (16 slots × 2 Bytes × 24 bytes) + static constexpr size_t PENDING_PROP_RESOURCES_SIZE = 16; + struct PropResourceSlot { + bool in_use = false; + uint8_t resource_hash[HASH_SIZE]; + uint8_t message_hash[HASH_SIZE]; + RNS::Bytes resource_hash_bytes() const { return RNS::Bytes(resource_hash, HASH_SIZE); } + RNS::Bytes message_hash_bytes() const { return RNS::Bytes(message_hash, HASH_SIZE); } + void set_resource_hash(const RNS::Bytes& b) { + size_t len = std::min(b.size(), HASH_SIZE); + memcpy(resource_hash, b.data(), len); + if (len < HASH_SIZE) memset(resource_hash + len, 0, HASH_SIZE - len); + } + void set_message_hash(const RNS::Bytes& b) { + size_t len = std::min(b.size(), HASH_SIZE); + memcpy(message_hash, b.data(), len); + if (len < HASH_SIZE) memset(message_hash + len, 0, HASH_SIZE - len); + } + bool resource_hash_equals(const RNS::Bytes& b) const { + if (b.size() != HASH_SIZE) return false; + return memcmp(resource_hash, b.data(), HASH_SIZE) == 0; + } + void clear() { + in_use = false; + memset(resource_hash, 0, HASH_SIZE); + memset(message_hash, 0, HASH_SIZE); + } + }; + static PropResourceSlot _pending_prop_resources_pool[PENDING_PROP_RESOURCES_SIZE]; + static PropResourceSlot* find_prop_resource_slot(const RNS::Bytes& hash); + static PropResourceSlot* find_empty_prop_resource_slot(); + + // Stamp enforcement + uint8_t _stamp_cost = 0; // Required stamp cost (0 = no stamp required) + bool _enforce_stamps = false; // Whether to enforce stamp requirements + }; + +} // namespace LXMF diff --git a/src-shim/LXMF/LXMessage.cpp b/src-shim/LXMF/LXMessage.cpp new file mode 100644 index 00000000..72280183 --- /dev/null +++ b/src-shim/LXMF/LXMessage.cpp @@ -0,0 +1,746 @@ +#include "LXMessage.h" +#include "LXStamper.h" +#include +#include +#include +#include + +#include + +using namespace LXMF; +using namespace RNS; + +// Constructor with destination and source objects +LXMessage::LXMessage( + const Destination& destination, + const Destination& source, + const Bytes& content, + const Bytes& title, + Type::Message::Method desired_method +) : + _destination(destination), + _source(source), + _content(content), + _title(title), + _desired_method(desired_method), + _method(desired_method) +{ + // Extract hashes from destination objects + if (destination) { + _destination_hash = destination.hash(); + } + if (source) { + _source_hash = source.hash(); + } + + INFO("Created new LXMF message"); + DEBUG(" Destination: " + _destination_hash.toHex()); + DEBUG(" Source: " + _source_hash.toHex()); + DEBUG(" Content size: " + std::to_string(_content.size()) + " bytes"); +} + +// Constructor with hashes only (for unpacking) +LXMessage::LXMessage( + const Bytes& destination_hash, + const Bytes& source_hash, + const Bytes& content, + const Bytes& title, + Type::Message::Method desired_method +) : + _destination(RNS::Type::NONE), + _source(RNS::Type::NONE), + _destination_hash(destination_hash), + _source_hash(source_hash), + _content(content), + _title(title), + _desired_method(desired_method), + _method(desired_method) +{ + DEBUG("Created LXMF message from hashes"); +} + +LXMessage::~LXMessage() { + TRACE("LXMessage destroyed"); +} + +// Field helper methods for fixed-size array +bool LXMessage::fields_set(const Bytes& key, const Bytes& value) { + // Check if key already exists, update if so + for (size_t i = 0; i < MAX_FIELDS; ++i) { + if (_fields_pool[i].in_use && _fields_pool[i].key == key) { + _fields_pool[i].value = value; + _packed_valid = false; + return true; + } + } + + // Find an empty slot + for (size_t i = 0; i < MAX_FIELDS; ++i) { + if (!_fields_pool[i].in_use) { + _fields_pool[i].in_use = true; + _fields_pool[i].key = key; + _fields_pool[i].value = value; + ++_fields_count; + _packed_valid = false; + return true; + } + } + + // Pool is full + WARNING("LXMessage field pool full, cannot add more fields"); + return false; +} + +const Bytes* LXMessage::fields_get(const Bytes& key) const { + for (size_t i = 0; i < MAX_FIELDS; ++i) { + if (_fields_pool[i].in_use && _fields_pool[i].key == key) { + return &_fields_pool[i].value; + } + } + return nullptr; +} + +bool LXMessage::fields_has(const Bytes& key) const { + return fields_get(key) != nullptr; +} + +void LXMessage::fields_clear() { + for (size_t i = 0; i < MAX_FIELDS; ++i) { + _fields_pool[i].clear(); + } + _fields_count = 0; + _packed_valid = false; +} + +// Pack the message into binary format +const Bytes& LXMessage::pack() { + if (_packed_valid) { + return _packed; + } + + INFO("Packing LXMF message"); + + // 1. Set timestamp if not already set + if (_timestamp == 0.0) { + _timestamp = Utilities::OS::time(); + } + + // 2. Create payload array: [timestamp, title, content, fields, stamp?] - matches Python LXMF exactly + // Python: msgpack.packb([self.timestamp, self.title, self.content, self.fields]) + // If stamp is present, it's appended as 5th element + MsgPack::Packer packer; + + // Pack as array with 4 or 5 elements (5 if stamp present) + bool has_stamp = (_stamp.size() == LXStamper::STAMP_SIZE); + packer.packArraySize(has_stamp ? 5 : 4); + + // Element 0: timestamp (float64) + packer.pack(_timestamp); + + // Element 1: title (binary) + packer.packBinary(_title.data(), _title.size()); + + // Element 2: content (binary) + packer.packBinary(_content.data(), _content.size()); + + // Element 3: fields (map) - iterate over fixed array + packer.packMapSize(_fields_count); + for (size_t i = 0; i < MAX_FIELDS; ++i) { + if (_fields_pool[i].in_use) { + packer.packBinary(_fields_pool[i].key.data(), _fields_pool[i].key.size()); + packer.packBinary(_fields_pool[i].value.data(), _fields_pool[i].value.size()); + } + } + + // Element 4 (optional): stamp - 32 bytes + if (has_stamp) { + packer.packBinary(_stamp.data(), _stamp.size()); + DEBUG(" Stamp included in payload (" + std::to_string(_stamp.size()) + " bytes)"); + } + + Bytes packed_payload(packer.data(), packer.size()); + + // 3. Calculate hash: SHA256(dest_hash + source_hash + packed_payload) + Bytes hashed_part; + hashed_part << _destination_hash; + hashed_part << _source_hash; + hashed_part << packed_payload; + + _hash = Identity::full_hash(hashed_part); + + DEBUG(" Message hash: " + _hash.toHex()); + + // 4. Create signed part: hashed_part + hash + Bytes signed_part; + signed_part << hashed_part; + signed_part << _hash; + + // 5. Sign with source identity + if (_source) { + _signature = _source.sign(signed_part); + _signature_validated = true; + DEBUG(" Message signed (" + std::to_string(_signature.size()) + " bytes)"); + } else { + ERROR("Cannot sign message - source destination not available"); + throw std::runtime_error("Cannot sign message without source destination"); + } + + // 6. Pack final message: dest_hash + source_hash + signature + packed_payload + _packed.clear(); + _packed << _destination_hash; + _packed << _source_hash; + _packed << _signature; + _packed << packed_payload; + + _packed_valid = true; + + // 7. Determine delivery method and representation + size_t content_size = packed_payload.size() - Type::Constants::TIMESTAMP_SIZE - Type::Constants::STRUCT_OVERHEAD; + + // For Phase 1 MVP, we only support DIRECT delivery + if (_desired_method == Type::Message::DIRECT) { + if (content_size <= Type::Constants::LINK_PACKET_MAX_CONTENT) { + _method = Type::Message::DIRECT; + _representation = Type::Message::PACKET; + INFO(" Message will be sent as single packet (" + std::to_string(_packed.size()) + " bytes)"); + } else { + _method = Type::Message::DIRECT; + _representation = Type::Message::RESOURCE; + INFO(" Message will be sent as resource (" + std::to_string(_packed.size()) + " bytes)"); + } + } else { + WARNING("Only DIRECT delivery method is supported in Phase 1 MVP"); + _method = Type::Message::DIRECT; + _representation = Type::Message::PACKET; + } + + _state = Type::Message::OUTBOUND; + + INFO("Message packed successfully (" + std::to_string(_packed.size()) + " bytes total)"); + DEBUG(" Overhead: " + std::to_string(Type::Constants::LXMF_OVERHEAD) + " bytes"); + DEBUG(" Payload: " + std::to_string(packed_payload.size()) + " bytes"); + + return _packed; +} + +// Unpack an LXMF message from bytes +LXMessage LXMessage::unpack_from_bytes(const Bytes& lxmf_bytes, Type::Message::Method original_method, bool skip_signature_validation) { + INFO("Unpacking LXMF message from " + std::to_string(lxmf_bytes.size()) + " bytes"); + + // 1. Extract fixed-size fields + if (lxmf_bytes.size() < 2 * Type::Constants::DESTINATION_LENGTH + Type::Constants::SIGNATURE_LENGTH) { + throw std::runtime_error("LXMF message too short"); + } + + size_t offset = 0; + + Bytes destination_hash = lxmf_bytes.mid(offset, Type::Constants::DESTINATION_LENGTH); + offset += Type::Constants::DESTINATION_LENGTH; + + Bytes source_hash = lxmf_bytes.mid(offset, Type::Constants::DESTINATION_LENGTH); + offset += Type::Constants::DESTINATION_LENGTH; + + Bytes signature = lxmf_bytes.mid(offset, Type::Constants::SIGNATURE_LENGTH); + offset += Type::Constants::SIGNATURE_LENGTH; + + Bytes packed_payload = lxmf_bytes.mid(offset); + + DEBUG(" Destination hash: " + destination_hash.toHex()); + DEBUG(" Source hash: " + source_hash.toHex()); + DEBUG(" Signature: " + std::to_string(signature.size()) + " bytes"); + DEBUG(" Payload: " + std::to_string(packed_payload.size()) + " bytes"); + + // Check for empty payload - this would cause msgpack to crash + if (packed_payload.size() == 0) { + ERROR("LXMF message has empty payload - cannot unpack"); + ERROR(" Raw bytes (" + std::to_string(lxmf_bytes.size()) + "): " + lxmf_bytes.toHex()); + throw std::runtime_error("LXMF message has empty payload"); + } + + // Log first few bytes of payload for debugging + if (packed_payload.size() > 0) { + DEBUG(" Payload first bytes: " + packed_payload.left(std::min((size_t)16, packed_payload.size())).toHex()); + } + + // 2. Unpack payload: [timestamp, title, content, fields] - matches Python LXMF exactly + // The payload is msgpack array: [float64_timestamp, bin_title, bin_content, map_fields] + MsgPack::Unpacker unpacker; + unpacker.feed(packed_payload.data(), packed_payload.size()); + + double timestamp = 0.0; + Bytes title; + Bytes content; + + // Temporary storage for fields before copying to message + FieldEntry temp_fields[MAX_FIELDS]; + size_t temp_fields_count = 0; + + Bytes stamp; // Optional 5th element + + try { + MsgPack::bin_t title_bin; + MsgPack::bin_t content_bin; + + // First, read the array header (should be 4 or 5 elements) + MsgPack::arr_size_t arr_size; + unpacker.deserialize(arr_size); + DEBUG(" Msgpack array size: " + std::to_string(arr_size.size())); + + if (arr_size.size() < 4) { + throw std::runtime_error("LXMF payload array too short: " + std::to_string(arr_size.size())); + } + + // Unpack timestamp (element 0) + unpacker.deserialize(timestamp); + DEBUG(" Parsed timestamp: " + std::to_string(timestamp)); + + // Unpack title (element 1) - as binary + unpacker.deserialize(title_bin); + title = Bytes(title_bin); + DEBUG(" Parsed title: " + std::to_string(title.size()) + " bytes"); + + // Unpack content (element 2) - as binary + unpacker.deserialize(content_bin); + content = Bytes(content_bin); + DEBUG(" Parsed content: " + std::to_string(content.size()) + " bytes"); + + // Unpack fields map (element 3) + MsgPack::map_size_t map_size; + unpacker.deserialize(map_size); + DEBUG(" Msgpack map size: " + std::to_string(map_size.size())); + + // Unpack each field (key-value pairs) into temporary storage + for (size_t i = 0; i < map_size.size() && temp_fields_count < MAX_FIELDS; ++i) { + MsgPack::bin_t key_bin; + MsgPack::bin_t value_bin; + + unpacker.deserialize(key_bin); + unpacker.deserialize(value_bin); + + temp_fields[temp_fields_count].in_use = true; + temp_fields[temp_fields_count].key = Bytes(key_bin); + temp_fields[temp_fields_count].value = Bytes(value_bin); + ++temp_fields_count; + } + + if (map_size.size() > MAX_FIELDS) { + WARNING("LXMF message has " + std::to_string(map_size.size()) + + " fields, but max is " + std::to_string(MAX_FIELDS) + " - some fields truncated"); + } + + // Unpack stamp (element 4) - optional, 32 bytes + if (arr_size.size() > 4) { + MsgPack::bin_t stamp_bin; + unpacker.deserialize(stamp_bin); + stamp = Bytes(stamp_bin); + DEBUG(" Parsed stamp: " + std::to_string(stamp.size()) + " bytes"); + } + + } catch (const std::exception& e) { + ERROR("Failed to unpack LXMF message payload: " + std::string(e.what())); + throw; + } + + DEBUG(" Timestamp: " + std::to_string(timestamp)); + DEBUG(" Title size: " + std::to_string(title.size()) + " bytes"); + DEBUG(" Content size: " + std::to_string(content.size()) + " bytes"); + DEBUG(" Fields: " + std::to_string(temp_fields_count)); + + // 3. Create message object + LXMessage message(destination_hash, source_hash, content, title, original_method); + + // Copy fields to message + for (size_t i = 0; i < temp_fields_count; ++i) { + message.fields_set(temp_fields[i].key, temp_fields[i].value); + } + message._timestamp = timestamp; + message._signature = signature; + message._packed = lxmf_bytes; + message._packed_valid = true; + message._incoming = true; + message._state = Type::Message::DELIVERED; + + // Assign stamp if present + if (stamp.size() == LXStamper::STAMP_SIZE) { + message._stamp = stamp; + DEBUG(" Stamp attached to message"); + } + + // 4. Calculate hash for verification + Bytes hashed_part; + hashed_part << destination_hash; + hashed_part << source_hash; + hashed_part << packed_payload; + + message._hash = Identity::full_hash(hashed_part); + + DEBUG(" Calculated hash: " + message._hash.toHex()); + + // 5. Try to validate signature (skip if loading from trusted storage) + if (skip_signature_validation) { + // Trust that signature was validated when originally received + message._signature_validated = true; + DEBUG(" Skipping signature validation (trusted storage)"); + } else { + // Check if we have the source identity cached + Identity source_identity = Identity::recall(source_hash); + if (source_identity) { + INFO(" Source identity found in cache, validating signature"); + message._source = Destination(source_identity, RNS::Type::Destination::OUT, RNS::Type::Destination::SINGLE, "lxmf", "delivery"); + + // Validate signature + Bytes signed_part; + signed_part << hashed_part; + signed_part << message._hash; + + if (source_identity.validate(signature, signed_part)) { + message._signature_validated = true; + INFO(" Signature validated successfully"); + } else { + message._signature_validated = false; + message._unverified_reason = Type::Message::SIGNATURE_INVALID; + WARNING(" Signature validation failed!"); + } + } else { + message._signature_validated = false; + message._unverified_reason = Type::Message::SOURCE_UNKNOWN; + DEBUG(" Source identity unknown, signature not validated"); + } + } + + // Similarly try to get destination identity + Identity dest_identity = Identity::recall(destination_hash); + if (dest_identity) { + message._destination = Destination(dest_identity, RNS::Type::Destination::OUT, RNS::Type::Destination::SINGLE, "lxmf", "delivery"); + } + + INFO("Message unpacked successfully"); + return message; +} + +// Validate the message signature +bool LXMessage::validate_signature() { + if (_signature_validated) { + return true; + } + + INFO("Validating message signature"); + + // Try to get source identity if not already available + if (!_source) { + Identity source_identity = Identity::recall(_source_hash); + if (source_identity) { + _source = Destination(source_identity, RNS::Type::Destination::OUT, RNS::Type::Destination::SINGLE, "lxmf", "delivery"); + } else { + _unverified_reason = Type::Message::SOURCE_UNKNOWN; + WARNING("Cannot validate signature - source identity unknown"); + return false; + } + } + + // Reconstruct signed part + Bytes hashed_part; + hashed_part << _destination_hash; + hashed_part << _source_hash; + + // Need to repack payload for hashed_part + MsgPack::Packer packer; + packer.serialize(_timestamp); + packer.serialize(_title); + packer.serialize(_content); + packer.serialize((uint32_t)_fields_count); + for (size_t i = 0; i < MAX_FIELDS; ++i) { + if (_fields_pool[i].in_use) { + packer.serialize(_fields_pool[i].key); + packer.serialize(_fields_pool[i].value); + } + } + Bytes packed_payload(packer.data(), packer.size()); + hashed_part << packed_payload; + + Bytes signed_part; + signed_part << hashed_part; + signed_part << _hash; + + // Validate signature + if (_source.identity().validate(_signature, signed_part)) { + _signature_validated = true; + INFO("Signature validated successfully"); + return true; + } else { + _signature_validated = false; + _unverified_reason = Type::Message::SIGNATURE_INVALID; + WARNING("Signature validation failed"); + return false; + } +} + +// Send the message via a link +bool LXMessage::send_via_link(const Link& link) { + INFO("Sending LXMF message via link"); + + // Ensure message is packed + if (!_packed_valid) { + pack(); + } + + // Check that link is active + if (!link || link.status() != RNS::Type::Link::ACTIVE) { + ERROR("Cannot send message - link is not active"); + return false; + } + + _state = Type::Message::SENDING; + + try { + if (_representation == Type::Message::PACKET) { + // Send as single packet over link + INFO(" Sending as single packet (" + std::to_string(_packed.size()) + " bytes)"); + + Packet packet(link, _packed); + packet.send(); + + _state = Type::Message::SENT; + INFO("Message sent successfully as packet"); + return true; + + } else if (_representation == Type::Message::RESOURCE) { + // Send as resource over link + INFO(" Sending as resource (" + std::to_string(_packed.size()) + " bytes)"); + + // TODO: Implement resource transfer with callbacks + // For now, we'll create the resource but won't set callbacks + Resource resource(_packed, link); + + _state = Type::Message::SENT; + INFO("Message resource transfer initiated"); + return true; + + } else { + ERROR("Unknown message representation"); + _state = Type::Message::FAILED; + return false; + } + } catch (const std::exception& e) { + ERROR("Failed to send message: " + std::string(e.what())); + _state = Type::Message::FAILED; + return false; + } +} + +// String representation +std::string LXMessage::toString() const { + if (_hash) { + return ""; + } else { + return ""; + } +} + +// Pack the message for PROPAGATED delivery +Bytes LXMessage::pack_propagated() { + INFO("Packing LXMF message for PROPAGATED delivery"); + + // Ensure message is packed + if (!_packed_valid) { + pack(); + } + + // Get the destination identity for encryption + Identity dest_identity; + if (_destination) { + dest_identity = _destination.identity(); + } else { + dest_identity = Identity::recall(_destination_hash); + } + + if (!dest_identity) { + ERROR("Cannot pack for propagation - destination identity unknown"); + return {}; + } + + // Encrypt everything after the destination hash + // Format: packed = dest_hash (16) + source_hash (16) + signature (64) + payload + // We encrypt: source_hash + signature + payload + Bytes encrypted; + + // Use cached encrypted data if available (from generate_propagation_stamp) + // This ensures the stamp matches the encrypted content + if (_propagation_encrypted.size() > 0) { + encrypted = _propagation_encrypted; + DEBUG(" Using cached encrypted data: " + std::to_string(encrypted.size()) + " bytes"); + } else { + Bytes to_encrypt = _packed.mid(Type::Constants::DESTINATION_LENGTH); + DEBUG(" To encrypt: " + std::to_string(to_encrypt.size()) + " bytes"); + + encrypted = dest_identity.encrypt(to_encrypt); + if (!encrypted || encrypted.size() == 0) { + ERROR("Failed to encrypt message for propagation"); + return {}; + } + DEBUG(" Encrypted: " + std::to_string(encrypted.size()) + " bytes"); + } + + // Build lxmf_data: dest_hash + encrypted + Bytes lxmf_data; + lxmf_data << _destination_hash << encrypted; + + // Append propagation stamp if present + // The stamp is appended to lxmf_data before packing + if (_propagation_stamp.size() == LXStamper::STAMP_SIZE) { + lxmf_data << _propagation_stamp; + DEBUG(" Propagation stamp appended (" + std::to_string(_propagation_stamp.size()) + " bytes)"); + } + + // Pack as msgpack: [timestamp, [lxmf_data]] + // Format: array(2) [ float64(time), array(1) [ bin(lxmf_data) ] ] + MsgPack::Packer packer; + + packer.packArraySize(2); + + // Current timestamp + double current_time = Utilities::OS::time(); + packer.pack(current_time); + + // Inner array with lxmf_data (includes stamp if present) + packer.packArraySize(1); + packer.packBinary(lxmf_data.data(), lxmf_data.size()); + + Bytes result(packer.data(), packer.size()); + + INFO(" Propagation packed size: " + std::to_string(result.size()) + " bytes"); + + return result; +} + +// Validate the attached stamp against a required cost +bool LXMessage::validate_stamp(uint8_t required_cost) { + INFO("Validating stamp with required cost " + std::to_string(required_cost)); + + if (_stamp.size() != LXStamper::STAMP_SIZE) { + DEBUG(" No valid stamp attached (size=" + std::to_string(_stamp.size()) + ")"); + _stamp_valid = false; + return false; + } + + if (!_hash) { + // Need message hash - try packing if not done + if (!_packed_valid) { + ERROR(" Cannot validate stamp - message not packed and hash not available"); + _stamp_valid = false; + return false; + } + } + + // Generate workblock from message hash + Bytes workblock = LXStamper::stamp_workblock(_hash, LXStamper::WORKBLOCK_EXPAND_ROUNDS); + + // Validate stamp + _stamp_valid = LXStamper::stamp_valid(_stamp, required_cost, workblock); + + if (_stamp_valid) { + uint8_t value = LXStamper::stamp_value(workblock, _stamp); + INFO(" Stamp valid with value " + std::to_string(value)); + } else { + DEBUG(" Stamp invalid (does not meet cost " + std::to_string(required_cost) + ")"); + } + + return _stamp_valid; +} + +// Generate a stamp for this message +Bytes LXMessage::generate_stamp() { + if (_stamp_cost == 0) { + DEBUG("No stamp cost set, skipping stamp generation"); + return {}; + } + + // Ensure message is packed so we have the hash + if (!_hash) { + if (!_packed_valid) { + pack(); + } + } + + if (!_hash) { + ERROR("Cannot generate stamp - no message hash available"); + return {}; + } + + INFO("Generating stamp for message " + _hash.toHex() + " with cost " + std::to_string(_stamp_cost)); + + auto [stamp, value] = LXStamper::generate_stamp(_hash, _stamp_cost, LXStamper::WORKBLOCK_EXPAND_ROUNDS); + + if (stamp.size() == LXStamper::STAMP_SIZE) { + _stamp = stamp; + _stamp_valid = true; + INFO("Stamp generated with value " + std::to_string(value)); + + // Invalidate packed so it gets repacked with stamp + _packed_valid = false; + } else { + ERROR("Stamp generation failed"); + } + + return _stamp; +} + +// Generate propagation stamp for this message +Bytes LXMessage::generate_propagation_stamp(uint8_t target_cost) { + if (target_cost == 0) { + DEBUG("No propagation stamp cost specified, skipping"); + return {}; + } + + // Ensure message is packed + if (!_packed_valid) { + pack(); + } + + // Get the destination identity for encryption + Identity dest_identity; + if (_destination) { + dest_identity = _destination.identity(); + } else { + dest_identity = Identity::recall(_destination_hash); + } + + if (!dest_identity) { + ERROR("Cannot generate propagation stamp - destination identity unknown"); + return {}; + } + + // Build lxmf_data (same as pack_propagated but without stamp) + Bytes to_encrypt = _packed.mid(Type::Constants::DESTINATION_LENGTH); + Bytes encrypted = dest_identity.encrypt(to_encrypt); + if (!encrypted || encrypted.size() == 0) { + ERROR("Failed to encrypt message for propagation stamp calculation"); + return {}; + } + + // Cache the encrypted data so pack_propagated uses the same ciphertext + // (encryption produces different output each time due to ephemeral keys) + _propagation_encrypted = encrypted; + + Bytes lxmf_data; + lxmf_data << _destination_hash << encrypted; + + // Calculate transient_id = SHA256(lxmf_data) + Bytes transient_id = Identity::full_hash(lxmf_data); + + INFO("Generating propagation stamp for transient_id " + transient_id.toHex() + + " with cost " + std::to_string(target_cost)); + + // Generate stamp using PN-specific workblock rounds + auto [stamp, value] = LXStamper::generate_stamp( + transient_id, target_cost, LXStamper::WORKBLOCK_EXPAND_ROUNDS_PN); + + if (stamp.size() == LXStamper::STAMP_SIZE) { + _propagation_stamp = stamp; + INFO("Propagation stamp generated with value " + std::to_string(value)); + } else { + ERROR("Propagation stamp generation failed"); + } + + return _propagation_stamp; +} diff --git a/src-shim/LXMF/LXMessage.h b/src-shim/LXMF/LXMessage.h new file mode 100644 index 00000000..f81641ce --- /dev/null +++ b/src-shim/LXMF/LXMessage.h @@ -0,0 +1,418 @@ +#pragma once + +#include "Type.h" +#include +#include +#include +#include + +#include +#include + +namespace LXMF { + + /** + * @brief Fixed-size field entry to avoid heap fragmentation from std::map + */ + struct FieldEntry { + bool in_use = false; + RNS::Bytes key; + RNS::Bytes value; + void clear() { in_use = false; key.clear(); value.clear(); } + }; + + /// Maximum number of fields in an LXMF message + static constexpr size_t MAX_FIELDS = 16; + + /** + * @brief LXMF Message class + * + * Represents a Lightweight Extensible Messaging Format message that can be + * sent over Reticulum network. Supports message packing, unpacking, signing, + * and verification. + * + * Message structure (packed): + * - 16 bytes: destination hash + * - 16 bytes: source hash + * - 64 bytes: Ed25519 signature + * - Variable: msgpack([timestamp, title, content, fields]) + */ + class LXMessage { + + public: + using Ptr = std::shared_ptr; + + public: + /** + * @brief Default constructor for use in fixed pools + */ + LXMessage() = default; + + /** + * @brief Construct a new LXMF message + * + * @param destination Destination object (or {Type::NONE} if creating from hash) + * @param source Source destination object (our identity) + * @param content Message content as bytes or string + * @param title Message title (optional) + * @param desired_method Delivery method (default: DIRECT) + */ + LXMessage( + const RNS::Destination& destination, + const RNS::Destination& source, + const RNS::Bytes& content = {}, + const RNS::Bytes& title = {}, + Type::Message::Method desired_method = Type::Message::DIRECT + ); + + /** + * @brief Construct message from hashes (for unpacking) + */ + LXMessage( + const RNS::Bytes& destination_hash, + const RNS::Bytes& source_hash, + const RNS::Bytes& content = {}, + const RNS::Bytes& title = {}, + Type::Message::Method desired_method = Type::Message::DIRECT + ); + + ~LXMessage(); + + public: + /** + * @brief Pack the message into binary format + * + * Creates the LXMF message structure: + * 1. Payload = [timestamp, title, content, fields] + * 2. Hashed part = dest_hash + source_hash + msgpack(payload) + * 3. Hash (message ID) = SHA256(hashed_part) + * 4. Signed part = hashed_part + hash + * 5. Signature = Ed25519Sign(signed_part) + * 6. Packed = dest_hash + source_hash + signature + msgpack(payload) + * + * @return Packed message bytes + */ + const RNS::Bytes& pack(); + + /** + * @brief Unpack an LXMF message from bytes + * + * @param lxmf_bytes Packed LXMF message + * @param original_method Original delivery method (optional) + * @param skip_signature_validation Skip Ed25519 validation (for trusted local storage) + * @return LXMessage object (or throws on error) + */ + static LXMessage unpack_from_bytes(const RNS::Bytes& lxmf_bytes, Type::Message::Method original_method = Type::Message::DIRECT, bool skip_signature_validation = false); + + /** + * @brief Validate the message signature + * + * Verifies the Ed25519 signature using the source identity. + * Requires source identity to be known (cached in Identity::_known_destinations). + * + * @return true if signature is valid, false otherwise + */ + bool validate_signature(); + + /** + * @brief Send the message via a link + * + * For DIRECT delivery method: + * - If packed size <= 319 bytes: send as single packet + * - If packed size > 319 bytes: send as resource + * + * @param link Active link to send over + * @return true if send initiated successfully + */ + bool send_via_link(const RNS::Link& link); + + /** + * @brief Pack the message for PROPAGATED delivery + * + * Creates the LXMF propagation format: + * 1. Encrypt packed message (minus dest_hash) to destination + * 2. Format: [timestamp, [dest_hash + encrypted_data]] + * 3. Pack with msgpack + * + * @return Propagation-formatted message bytes + */ + RNS::Bytes pack_propagated(); + + // ========== Stamp Proof-of-Work Methods ========== + + /** + * @brief Set the required stamp cost for this message + * + * @param cost Number of leading zero bits required (0 = no stamp) + */ + inline void set_stamp_cost(uint8_t cost) { _stamp_cost = cost; } + + /** + * @brief Get the required stamp cost + */ + inline uint8_t stamp_cost() const { return _stamp_cost; } + + /** + * @brief Check if message has a valid stamp + */ + inline bool has_valid_stamp() const { return _stamp_valid; } + + /** + * @brief Get the attached stamp + */ + inline const RNS::Bytes& stamp() const { return _stamp; } + + /** + * @brief Set the stamp (for received messages) + */ + inline void stamp(const RNS::Bytes& stamp) { _stamp = stamp; } + + /** + * @brief Validate the attached stamp against a required cost + * + * Generates workblock from message hash and verifies stamp. + * + * @param required_cost Minimum number of leading zero bits + * @return true if stamp is valid + */ + bool validate_stamp(uint8_t required_cost); + + /** + * @brief Generate a stamp for this message (blocking, CPU-intensive) + * + * Generates workblock and mines for valid stamp. + * Sets _stamp and _stamp_valid on success. + * + * @return The generated stamp, or empty on failure + */ + RNS::Bytes generate_stamp(); + + /** + * @brief Set the propagation stamp for PROPAGATED delivery + * + * @param stamp The stamp to append to lxmf_data + */ + inline void set_propagation_stamp(const RNS::Bytes& stamp) { _propagation_stamp = stamp; } + + /** + * @brief Get the propagation stamp + */ + inline const RNS::Bytes& propagation_stamp() const { return _propagation_stamp; } + + /** + * @brief Generate propagation stamp for this message + * + * Uses transient_id and PN-specific workblock rounds. + * + * @param target_cost Required stamp cost from propagation node + * @return The generated stamp, or empty on failure + */ + RNS::Bytes generate_propagation_stamp(uint8_t target_cost); + + /** + * @brief Get message state + */ + inline Type::Message::State state() const { return _state; } + + /** + * @brief Set message state + */ + inline void state(Type::Message::State state) { _state = state; } + + /** + * @brief Get message hash (ID) + */ + inline const RNS::Bytes& hash() const { return _hash; } + + /** + * @brief Set message hash (for delivery confirmation callbacks) + */ + inline void hash(const RNS::Bytes& hash) { _hash = hash; } + + /** + * @brief Get message content + */ + inline const RNS::Bytes& content() const { return _content; } + + /** + * @brief Set message content + */ + inline void content(const RNS::Bytes& content) { _content = content; _packed_valid = false; } + + /** + * @brief Get message title + */ + inline const RNS::Bytes& title() const { return _title; } + + /** + * @brief Set message title + */ + inline void title(const RNS::Bytes& title) { _title = title; _packed_valid = false; } + + /** + * @brief Get field count + */ + inline size_t fields_count() const { return _fields_count; } + + /** + * @brief Get field at index (for iteration) + * @return Pointer to field entry, or nullptr if index out of range + */ + inline const FieldEntry* field_at(size_t index) const { + if (index >= MAX_FIELDS) return nullptr; + return _fields_pool[index].in_use ? &_fields_pool[index] : nullptr; + } + + /** + * @brief Set a field value + * @return true if successful, false if pool is full + */ + bool fields_set(const RNS::Bytes& key, const RNS::Bytes& value); + + /** + * @brief Get a field value by key + * @return Pointer to value, or nullptr if not found + */ + const RNS::Bytes* fields_get(const RNS::Bytes& key) const; + + /** + * @brief Check if a field exists + */ + bool fields_has(const RNS::Bytes& key) const; + + /** + * @brief Clear all fields + */ + void fields_clear(); + + /** + * @brief Get timestamp + */ + inline double timestamp() const { return _timestamp; } + + /** + * @brief Get destination hash + */ + inline const RNS::Bytes& destination_hash() const { return _destination_hash; } + + /** + * @brief Set destination hash (for when identity is not known) + */ + inline void destination_hash(const RNS::Bytes& hash) { _destination_hash = hash; _packed_valid = false; } + + /** + * @brief Get source hash + */ + inline const RNS::Bytes& source_hash() const { return _source_hash; } + + /** + * @brief Check if signature was validated + */ + inline bool signature_validated() const { return _signature_validated; } + + /** + * @brief Get unverified reason (if signature not validated) + */ + inline Type::Message::UnverifiedReason unverified_reason() const { return _unverified_reason; } + + /** + * @brief Get packed message bytes + */ + inline const RNS::Bytes& packed() const { return _packed; } + + /** + * @brief Get packed message size + */ + inline size_t packed_size() const { return _packed.size(); } + + /** + * @brief Get delivery method + */ + inline Type::Message::Method method() const { return _method; } + + /** + * @brief Set delivery method + */ + inline void set_method(Type::Message::Method method) { _method = method; _desired_method = method; } + + /** + * @brief Get message representation (PACKET or RESOURCE) + */ + inline Type::Message::Representation representation() const { return _representation; } + + /** + * @brief Check if this is an incoming message + */ + inline bool incoming() const { return _incoming; } + + /** + * @brief Set incoming flag (for loading from storage) + */ + inline void incoming(bool incoming) { _incoming = incoming; } + + /** + * @brief String representation of message + */ + std::string toString() const; + + private: + // Core message data + RNS::Bytes _destination_hash; + RNS::Bytes _source_hash; + RNS::Bytes _content; + RNS::Bytes _title; + + // Fixed-size field pool to avoid heap fragmentation + FieldEntry _fields_pool[MAX_FIELDS]; + size_t _fields_count = 0; + + // Destination/Source objects (may be {Type::NONE} if creating from hashes) + RNS::Destination _destination; + RNS::Destination _source; + + // Message metadata + RNS::Bytes _hash; // Message ID (SHA256 of hashed_part) + RNS::Bytes _signature; // Ed25519 signature (64 bytes) + double _timestamp = 0.0; // Unix timestamp + + // Packing state + RNS::Bytes _packed; // Packed message bytes + bool _packed_valid = false; // Whether _packed is up-to-date + + // Delivery parameters + Type::Message::Method _desired_method = Type::Message::DIRECT; + Type::Message::Method _method = Type::Message::DIRECT; + Type::Message::Representation _representation = Type::Message::UNKNOWN; + + // Message state + Type::Message::State _state = Type::Message::GENERATING; + + // Signature validation + bool _signature_validated = false; + Type::Message::UnverifiedReason _unverified_reason = Type::Message::SOURCE_UNKNOWN; + + // Direction + bool _incoming = false; // true if received, false if created locally + + // Stamp proof-of-work + RNS::Bytes _stamp; // 32-byte stamp for direct messages + RNS::Bytes _propagation_stamp; // 32-byte stamp for propagation delivery + bool _stamp_valid = false; // Whether stamp has been validated + uint8_t _stamp_cost = 0; // Required stamp cost (0 = no stamp needed) + + // Cached propagation data (to ensure stamp matches encrypted content) + RNS::Bytes _propagation_encrypted; // Cached encrypted content for propagation + + private: + /** + * @brief Calculate message hash from components + */ + void calculate_hash(); + + /** + * @brief Sign the message with source identity + */ + void sign_message(); + }; + +} // namespace LXMF diff --git a/src-shim/LXMF/LXStamper.cpp b/src-shim/LXMF/LXStamper.cpp new file mode 100644 index 00000000..c3ba0890 --- /dev/null +++ b/src-shim/LXMF/LXStamper.cpp @@ -0,0 +1,241 @@ +#include "LXStamper.h" +#include "Type.h" +#include +#include +#include +#include +#include + +#include + +#ifdef ESP_PLATFORM +#include +#include +#include "esp_task_wdt.h" +#endif + +using namespace LXMF; +using namespace RNS; + +// Count leading zero bits in a hash buffer +static uint8_t count_leading_zeros(const uint8_t* hash, size_t size) { + uint8_t value = 0; + for (size_t i = 0; i < size && value < 256; i++) { + uint8_t byte = hash[i]; + if (byte == 0) { + value += 8; + } else { + while ((byte & 0x80) == 0 && value < 256) { + value++; + byte <<= 1; + } + break; + } + } + return value; +} + +// Pack uint16 as msgpack (fixint or uint16 format) +Bytes LXStamper::msgpack_pack_uint16(uint16_t n) { + Bytes result; + if (n <= 127) { + // Positive fixint: single byte 0x00-0x7f + result.append((uint8_t)n); + } else if (n <= 255) { + // uint8: 0xcc followed by uint8 + result.append((uint8_t)0xcc); + result.append((uint8_t)n); + } else { + // uint16: 0xcd followed by big-endian uint16 + result.append((uint8_t)0xcd); + result.append((uint8_t)(n >> 8)); + result.append((uint8_t)(n & 0xff)); + } + return result; +} + +// Generate workblock from message ID using HKDF expansion +Bytes LXStamper::stamp_workblock(const Bytes& material, uint16_t expand_rounds) { + DEBUG("Generating stamp workblock with " + std::to_string(expand_rounds) + " rounds"); + + // Pre-allocate for efficiency + // Each round produces 256 bytes + size_t total_size = 256 * expand_rounds; + Bytes workblock; + workblock.reserve(total_size); + + for (uint16_t n = 0; n < expand_rounds; n++) { + // Pack n with msgpack (matches Python: msgpack.packb(n)) + Bytes packed_n = msgpack_pack_uint16(n); + + // salt = full_hash(material + msgpack.packb(n)) + Bytes salt_input; + salt_input << material << packed_n; + Bytes salt = Identity::full_hash(salt_input); + + // chunk = hkdf(length=256, derive_from=material, salt=salt, context=None) + Bytes chunk = Cryptography::hkdf(256, material, salt, {}); + + workblock << chunk; + } + + DEBUG("Workblock generated: " + std::to_string(workblock.size()) + " bytes"); + return workblock; +} + +// Count leading zero bits in SHA256(workblock + stamp) +uint8_t LXStamper::stamp_value(const Bytes& workblock, const Bytes& stamp) { + // Hash the concatenation + Bytes material; + material << workblock << stamp; + Bytes hash = Identity::full_hash(material); + + // Count leading zero bits + uint8_t value = 0; + + for (size_t i = 0; i < hash.size() && value < 256; i++) { + uint8_t byte = hash.data()[i]; + if (byte == 0) { + // Entire byte is zeros + value += 8; + } else { + // Count leading zeros in this byte + while ((byte & 0x80) == 0 && value < 256) { + value++; + byte <<= 1; + } + break; // Non-zero bit found + } + } + + return value; +} + +// Check if stamp meets target cost +bool LXStamper::stamp_valid(const Bytes& stamp, uint8_t target_cost, const Bytes& workblock) { + if (stamp.size() != STAMP_SIZE) { + return false; + } + return stamp_value(workblock, stamp) >= target_cost; +} + +// Generate a valid stamp (blocking, CPU-intensive) +// OPTIMIZED: Pre-hash workblock once, only hash 16-byte stamp per iteration +// This is ~16,000x faster than hashing the full 256KB workblock each time +std::pair LXStamper::generate_stamp( + const Bytes& message_id, + uint8_t stamp_cost, + uint16_t expand_rounds, + std::atomic* cancel, + ProgressCallback progress) +{ + INFO("Generating stamp with cost " + std::to_string(stamp_cost) + " for " + message_id.toHex()); + + // Generate workblock + Bytes workblock = stamp_workblock(message_id, expand_rounds); + + // OPTIMIZATION: Pre-hash the workblock once and save the SHA256 state + // This avoids re-hashing 256KB for every stamp attempt + SHA256 base_hash; + base_hash.reset(); + base_hash.update(workblock.data(), workblock.size()); + + uint32_t rounds = 0; + double start_time = Utilities::OS::time(); + uint8_t stamp_buffer[STAMP_SIZE]; + uint8_t hash_result[32]; + + while (true) { + // Check for cancellation + if (cancel && cancel->load()) { + INFO("Stamp generation cancelled after " + std::to_string(rounds) + " rounds"); + return {{}, 0}; + } + + // Generate random stamp candidate directly into buffer + Bytes stamp = Cryptography::random(STAMP_SIZE); + memcpy(stamp_buffer, stamp.data(), STAMP_SIZE); + rounds++; + + // OPTIMIZATION: Copy the pre-computed hash state and only hash the stamp + SHA256 test_hash = base_hash; // Copy constructor copies state + test_hash.update(stamp_buffer, STAMP_SIZE); + test_hash.finalize(hash_result, 32); + + // Count leading zeros in the hash + uint8_t value = count_leading_zeros(hash_result, 32); + + // Check if it meets the target cost + if (value >= stamp_cost) { + double duration = Utilities::OS::time() - start_time; + double speed = (duration > 0) ? (rounds / duration) : 0; + + INFO("Stamp with value " + std::to_string(value) + " generated in " + + std::to_string((int)duration) + "s, " + std::to_string(rounds) + + " rounds, " + std::to_string((int)speed) + " rounds/sec"); + + return {stamp, value}; + } + + // Progress callback every 1000 rounds + if (progress && (rounds % 1000 == 0)) { + progress(rounds); + } + + // Log progress every 5000 rounds + if (rounds % 5000 == 0) { + double elapsed = Utilities::OS::time() - start_time; + double speed = (elapsed > 0) ? (rounds / elapsed) : 0; + DEBUG("Stamp generation: " + std::to_string(rounds) + " rounds, " + + std::to_string((int)speed) + " rounds/sec"); + } + + // Yield to allow other tasks (LVGL, network) to run + // This prevents UI freeze during stamp generation + // Yield every 10 rounds (was 100) for better UI responsiveness +#ifdef ESP_PLATFORM + if (rounds % 10 == 0) { + vTaskDelay(1); // Yield for 1 tick + esp_task_wdt_reset(); // Feed watchdog during long operations + } +#endif + } + + // Should never reach here + return {{}, 0}; +} + +// Validate propagation node stamp +std::tuple LXStamper::validate_pn_stamp( + const Bytes& transient_data, + uint8_t target_cost) +{ + // Check minimum size: need at least LXMF_OVERHEAD + STAMP_SIZE + if (transient_data.size() <= Type::Constants::LXMF_OVERHEAD + STAMP_SIZE) { + WARNING("Transient data too short for stamp validation"); + return {{}, {}, 0, {}}; + } + + // Extract lxm_data and stamp + size_t lxm_data_len = transient_data.size() - STAMP_SIZE; + Bytes lxm_data = transient_data.left(lxm_data_len); + Bytes stamp = transient_data.mid(lxm_data_len, STAMP_SIZE); + + // Calculate transient_id = full_hash(lxm_data) + Bytes transient_id = Identity::full_hash(lxm_data); + + // Generate workblock with PN-specific rounds + Bytes workblock = stamp_workblock(transient_id, WORKBLOCK_EXPAND_ROUNDS_PN); + + // Validate stamp + if (!stamp_valid(stamp, target_cost, workblock)) { + DEBUG("PN stamp validation failed for transient_id " + transient_id.toHex()); + return {{}, {}, 0, {}}; + } + + uint8_t value = stamp_value(workblock, stamp); + DEBUG("PN stamp validated: transient_id=" + transient_id.toHex() + + ", value=" + std::to_string(value)); + + return {transient_id, lxm_data, value, stamp}; +} diff --git a/src-shim/LXMF/LXStamper.h b/src-shim/LXMF/LXStamper.h new file mode 100644 index 00000000..b6555dea --- /dev/null +++ b/src-shim/LXMF/LXStamper.h @@ -0,0 +1,121 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace LXMF { + + /** + * @brief LXMF Stamp Proof-of-Work generator and validator + * + * Implements hashcash-style proof-of-work stamps for anti-spam protection. + * Stamps require finding a 32-byte nonce where SHA256(workblock + stamp) + * has a target number of leading zero bits. + * + * The workblock is generated by expanding the message ID using HKDF, + * creating a large memory-hard challenge that makes stamp generation + * computationally expensive. + */ + class LXStamper { + + public: + // Stamp size in bytes (256-bit hash) + static const uint8_t STAMP_SIZE = 32; + + // Workblock expansion rounds + static const uint16_t WORKBLOCK_EXPAND_ROUNDS = 3000; // Direct messages (768KB) + static const uint16_t WORKBLOCK_EXPAND_ROUNDS_PN = 1000; // Propagation nodes (256KB) + + // Callback for progress updates during stamp generation + using ProgressCallback = std::function; + + public: + /** + * @brief Generate a workblock from message ID + * + * Creates a large memory-hard block by expanding the material + * through repeated HKDF operations. + * + * @param material Message ID or transient ID (32 bytes) + * @param expand_rounds Number of HKDF expansion rounds (default 3000) + * @return Workblock bytes (256 * expand_rounds bytes) + */ + static RNS::Bytes stamp_workblock( + const RNS::Bytes& material, + uint16_t expand_rounds = WORKBLOCK_EXPAND_ROUNDS); + + /** + * @brief Calculate stamp value (number of leading zero bits) + * + * @param workblock The workblock generated from message ID + * @param stamp The stamp to evaluate + * @return Number of leading zero bits in SHA256(workblock + stamp) + */ + static uint8_t stamp_value( + const RNS::Bytes& workblock, + const RNS::Bytes& stamp); + + /** + * @brief Check if a stamp meets the target cost + * + * @param stamp The stamp to validate + * @param target_cost Required number of leading zero bits + * @param workblock The workblock generated from message ID + * @return true if stamp is valid (has >= target_cost leading zeros) + */ + static bool stamp_valid( + const RNS::Bytes& stamp, + uint8_t target_cost, + const RNS::Bytes& workblock); + + /** + * @brief Generate a valid stamp (blocking, CPU-intensive) + * + * Mines for a stamp by repeatedly trying random values until + * one is found that meets the target cost requirement. + * + * @param message_id The message ID to generate stamp for + * @param stamp_cost Required number of leading zero bits + * @param expand_rounds HKDF expansion rounds (default 3000) + * @param cancel Atomic flag to cancel generation (optional) + * @param progress Callback for progress updates (optional) + * @return Pair of [stamp, value] or empty stamp if cancelled + */ + static std::pair generate_stamp( + const RNS::Bytes& message_id, + uint8_t stamp_cost, + uint16_t expand_rounds = WORKBLOCK_EXPAND_ROUNDS, + std::atomic* cancel = nullptr, + ProgressCallback progress = nullptr); + + /** + * @brief Validate a propagation node stamp + * + * Extracts and validates a stamp from propagation node transient data. + * The stamp is the last STAMP_SIZE bytes of transient_data. + * + * @param transient_data Full data including stamp at end + * @param target_cost Required stamp cost + * @return Tuple of [transient_id, lxm_data, value, stamp] or all empty on failure + */ + static std::tuple + validate_pn_stamp( + const RNS::Bytes& transient_data, + uint8_t target_cost); + + /** + * @brief Pack uint16 as msgpack for salt generation + * + * Public for testing purposes. + * + * @param n Number to pack + * @return Msgpack-encoded bytes + */ + static RNS::Bytes msgpack_pack_uint16(uint16_t n); + }; + +} // namespace LXMF diff --git a/src-shim/LXMF/MessageStore.cpp b/src-shim/LXMF/MessageStore.cpp new file mode 100644 index 00000000..14660cbb --- /dev/null +++ b/src-shim/LXMF/MessageStore.cpp @@ -0,0 +1,735 @@ +#include "MessageStore.h" +#include +#include + +#include +#include +#include + +using namespace LXMF; +using namespace RNS; + +// ConversationInfo helper methods +bool MessageStore::ConversationInfo::add_message_hash(const Bytes& hash) { + // Check if already exists + if (has_message(hash)) { + return false; + } + // Check if pool is full + if (message_count >= MAX_MESSAGES_PER_CONVERSATION) { + return false; + } + // Copy hash to fixed array + size_t len = std::min(hash.size(), MESSAGE_HASH_SIZE); + memcpy(message_hashes[message_count], hash.data(), len); + if (len < MESSAGE_HASH_SIZE) { + memset(message_hashes[message_count] + len, 0, MESSAGE_HASH_SIZE - len); + } + ++message_count; + return true; +} + +bool MessageStore::ConversationInfo::has_message(const Bytes& hash) const { + if (hash.size() == 0 || hash.size() > MESSAGE_HASH_SIZE) return false; + for (size_t i = 0; i < message_count; ++i) { + if (memcmp(message_hashes[i], hash.data(), hash.size()) == 0) { + return true; + } + } + return false; +} + +bool MessageStore::ConversationInfo::remove_message_hash(const Bytes& hash) { + if (hash.size() == 0 || hash.size() > MESSAGE_HASH_SIZE) return false; + for (size_t i = 0; i < message_count; ++i) { + if (memcmp(message_hashes[i], hash.data(), hash.size()) == 0) { + // Shift remaining elements down + for (size_t j = i; j < message_count - 1; ++j) { + memcpy(message_hashes[j], message_hashes[j + 1], MESSAGE_HASH_SIZE); + } + memset(message_hashes[message_count - 1], 0, MESSAGE_HASH_SIZE); + --message_count; + return true; + } + } + return false; +} + +void MessageStore::ConversationInfo::clear() { + memset(peer_hash, 0, PEER_HASH_SIZE); + memset(message_hashes, 0, sizeof(message_hashes)); + message_count = 0; + last_activity = 0.0; + unread_count = 0; + memset(last_message_hash, 0, MESSAGE_HASH_SIZE); +} + +// ConversationSlot helper method +void MessageStore::ConversationSlot::clear() { + in_use = false; + memset(peer_hash, 0, PEER_HASH_SIZE); + info.clear(); +} + +// Constructor +MessageStore::MessageStore(const std::string& base_path) : + _base_path(base_path), + _initialized(false) +{ + INFO("Initializing MessageStore at: " + _base_path); + + // Initialize pool + for (size_t i = 0; i < MAX_CONVERSATIONS; ++i) { + _conversations_pool[i].clear(); + } + + if (initialize_storage()) { + load_index(); + _initialized = true; + INFO("MessageStore initialized with " + std::to_string(count_conversations()) + " conversations"); + } else { + ERROR("Failed to initialize MessageStore"); + } +} + +MessageStore::~MessageStore() { + if (_initialized) { + save_index(); + } + TRACE("MessageStore destroyed"); +} + +// Initialize storage directories +bool MessageStore::initialize_storage() { + // Create short directories for SPIFFS compatibility + // SPIFFS is flat so these are mostly no-ops, but we try anyway + Utilities::OS::create_directory("/m"); // messages + Utilities::OS::create_directory("/c"); // conversations + + DEBUG("Storage directories initialized"); + return true; +} + +// Load conversation index from disk +void MessageStore::load_index() { + std::string index_path = "/conv.json"; // Short path for SPIFFS + + if (!Utilities::OS::file_exists(index_path.c_str())) { + DEBUG("No existing conversation index found"); + return; + } + + try { + // Read JSON file via OS abstraction (SPIFFS compatible) + Bytes data; + if (Utilities::OS::read_file(index_path.c_str(), data) == 0) { + WARNING("Failed to read index file or empty: " + index_path); + return; + } + + // Parse JSON from bytes using reusable document to reduce heap fragmentation + _json_doc.clear(); + DeserializationError error = deserializeJson(_json_doc, data.data(), data.size()); + + if (error) { + ERROR("Failed to parse conversation index: " + std::string(error.c_str())); + return; + } + + // Load conversations into pool + JsonArray conversations = _json_doc["conversations"].as(); + size_t slot_index = 0; + for (JsonObject conv : conversations) { + if (slot_index >= MAX_CONVERSATIONS) { + WARNING("Too many conversations in index, some will be skipped"); + break; + } + + ConversationSlot& slot = _conversations_pool[slot_index]; + slot.in_use = true; + + // Parse peer hash + const char* peer_hex = conv["peer_hash"]; + Bytes peer_bytes; + peer_bytes.assignHex(peer_hex); + slot.set_peer_hash(peer_bytes); + slot.info.set_peer_hash(peer_bytes); + + // Parse message hashes + JsonArray messages = conv["messages"].as(); + for (const char* msg_hex : messages) { + if (slot.info.message_count >= MAX_MESSAGES_PER_CONVERSATION) { + WARNING("Too many messages in conversation, some will be skipped"); + break; + } + Bytes msg_hash; + msg_hash.assignHex(msg_hex); + slot.info.add_message_hash(msg_hash); + } + + // Parse metadata + slot.info.last_activity = conv["last_activity"] | 0.0; + slot.info.unread_count = conv["unread_count"] | 0; + + if (!conv["last_message_hash"].isNull()) { + const char* last_msg_hex = conv["last_message_hash"]; + Bytes last_msg_bytes; + last_msg_bytes.assignHex(last_msg_hex); + slot.info.set_last_message_hash(last_msg_bytes); + } + + ++slot_index; + } + + DEBUG("Loaded " + std::to_string(count_conversations()) + " conversations from index"); + + } catch (const std::exception& e) { + ERROR("Exception loading conversation index: " + std::string(e.what())); + } +} + +// Save conversation index to disk +bool MessageStore::save_index() { + std::string index_path = "/conv.json"; // Short path for SPIFFS + + try { + // Use reusable document to reduce heap fragmentation + _json_doc.clear(); + JsonArray conversations = _json_doc["conversations"].to(); + + // Serialize each active conversation from pool + for (size_t i = 0; i < MAX_CONVERSATIONS; ++i) { + const ConversationSlot& slot = _conversations_pool[i]; + if (!slot.in_use) { + continue; + } + + const ConversationInfo& info = slot.info; + + JsonObject conv = conversations.add(); + conv["peer_hash"] = slot.peer_hash_bytes().toHex(); + conv["last_activity"] = info.last_activity; + conv["unread_count"] = info.unread_count; + + Bytes last_msg = info.last_message_hash_bytes(); + if (last_msg) { + conv["last_message_hash"] = last_msg.toHex(); + } + + // Serialize message hashes + JsonArray messages = conv["messages"].to(); + for (size_t j = 0; j < info.message_count; ++j) { + messages.add(info.message_hash_bytes(j).toHex()); + } + } + + // Serialize to string then write via OS abstraction (SPIFFS compatible) + std::string json_str; + serializeJsonPretty(_json_doc, json_str); + Bytes data((const uint8_t*)json_str.data(), json_str.size()); + + if (Utilities::OS::write_file(index_path.c_str(), data) != data.size()) { + ERROR("Failed to write index file: " + index_path); + return false; + } + + DEBUG("Saved conversation index"); + return true; + + } catch (const std::exception& e) { + ERROR("Exception saving conversation index: " + std::string(e.what())); + return false; + } +} + +// Save message to storage +bool MessageStore::save_message(const LXMessage& message) { + if (!_initialized) { + ERROR("MessageStore not initialized"); + return false; + } + + INFO("Saving message: " + message.hash().toHex()); + + try { + // Use reusable document to reduce heap fragmentation + _json_doc.clear(); + + _json_doc["hash"] = message.hash().toHex(); + _json_doc["destination_hash"] = message.destination_hash().toHex(); + _json_doc["source_hash"] = message.source_hash().toHex(); + _json_doc["incoming"] = message.incoming(); + _json_doc["timestamp"] = message.timestamp(); + _json_doc["state"] = static_cast(message.state()); + + // Store content as UTF-8 for fast loading (no msgpack unpacking needed) + std::string content_str((const char*)message.content().data(), message.content().size()); + _json_doc["content"] = content_str; + + // Store the entire packed message to preserve hash/signature + // This ensures exact reconstruction on load + _json_doc["packed"] = message.packed().toHex(); + + // Write message file via OS abstraction (SPIFFS compatible) + std::string message_path = get_message_path(message.hash()); + std::string json_str; + serializeJsonPretty(_json_doc, json_str); + Bytes data((const uint8_t*)json_str.data(), json_str.size()); + + if (Utilities::OS::write_file(message_path.c_str(), data) != data.size()) { + ERROR("Failed to write message file: " + message_path); + return false; + } + + DEBUG(" Message file saved: " + message_path); + + // Update conversation index + // Determine peer hash (the other party in the conversation) + // For incoming: peer = source, for outgoing: peer = destination + Bytes peer_hash = message.incoming() ? message.source_hash() : message.destination_hash(); + + // Get or create conversation slot + ConversationSlot* slot = get_or_create_conversation(peer_hash); + if (!slot) { + ERROR("Conversation pool is full, cannot add message"); + return false; + } + + ConversationInfo& conv = slot->info; + + // Add message to conversation (if not already present) + bool already_exists = conv.has_message(message.hash()); + + if (!already_exists) { + if (!conv.add_message_hash(message.hash())) { + WARNING("Message pool full for conversation: " + peer_hash.toHex()); + } else { + conv.last_activity = message.timestamp(); + conv.set_last_message_hash(message.hash()); + + // Increment unread count for incoming messages + if (message.incoming()) { + conv.unread_count++; + } + + DEBUG(" Added to conversation (now " + std::to_string(conv.message_count) + " messages)"); + } + } + + // Save updated index + save_index(); + + INFO("Message saved successfully"); + return true; + + } catch (const std::exception& e) { + ERROR("Exception saving message: " + std::string(e.what())); + return false; + } +} + +// Load message from storage +LXMessage MessageStore::load_message(const Bytes& message_hash) { + if (!_initialized) { + ERROR("MessageStore not initialized"); + return LXMessage(Bytes(), Bytes(), Bytes(), Bytes()); + } + + std::string message_path = get_message_path(message_hash); + + if (!Utilities::OS::file_exists(message_path.c_str())) { + WARNING("Message file not found: " + message_path); + return LXMessage(Bytes(), Bytes(), Bytes(), Bytes()); + } + + try { + // Read JSON file via OS abstraction (SPIFFS compatible) + Bytes data; + if (Utilities::OS::read_file(message_path.c_str(), data) == 0) { + ERROR("Failed to read message file: " + message_path); + return LXMessage(Bytes(), Bytes(), Bytes(), Bytes()); + } + + // Use reusable document to reduce heap fragmentation + _json_doc.clear(); + DeserializationError error = deserializeJson(_json_doc, data.data(), data.size()); + + if (error) { + ERROR("Failed to parse message file: " + std::string(error.c_str())); + return LXMessage(Bytes(), Bytes(), Bytes(), Bytes()); + } + + // Unpack the message from stored packed bytes + // This preserves the exact hash and signature + Bytes packed; + packed.assignHex(_json_doc["packed"].as()); + + // Skip signature validation - messages from storage were already validated when received + LXMessage message = LXMessage::unpack_from_bytes(packed, LXMF::Type::Message::DIRECT, true); + + // Restore incoming flag from storage (unpack_from_bytes defaults to true) + if (!_json_doc["incoming"].isNull()) { + message.incoming(_json_doc["incoming"].as()); + } + + DEBUG("Loaded message: " + message_hash.toHex()); + return message; + + } catch (const std::exception& e) { + ERROR("Exception loading message: " + std::string(e.what())); + return LXMessage(Bytes(), Bytes(), Bytes(), Bytes()); + } +} + +// Load only message metadata (fast path - no msgpack unpacking) +MessageStore::MessageMetadata MessageStore::load_message_metadata(const Bytes& message_hash) { + MessageMetadata meta; + meta.valid = false; + + if (!_initialized) { + return meta; + } + + std::string message_path = get_message_path(message_hash); + + if (!Utilities::OS::file_exists(message_path.c_str())) { + return meta; + } + + try { + Bytes data; + if (Utilities::OS::read_file(message_path.c_str(), data) == 0) { + return meta; + } + + // Use reusable document to reduce heap fragmentation + _json_doc.clear(); + DeserializationError error = deserializeJson(_json_doc, data.data(), data.size()); + + if (error) { + return meta; + } + + meta.hash = message_hash; + + // Read pre-extracted fields (no msgpack unpacking needed) + if (_json_doc["content"].is()) { + meta.content = _json_doc["content"].as(); + } + meta.timestamp = _json_doc["timestamp"] | 0.0; + meta.incoming = _json_doc["incoming"] | true; + meta.state = _json_doc["state"] | 0; + meta.valid = true; + + return meta; + + } catch (...) { + return meta; + } +} + +// Update message state in storage +bool MessageStore::update_message_state(const Bytes& message_hash, Type::Message::State state) { + if (!_initialized) { + ERROR("MessageStore not initialized"); + return false; + } + + std::string message_path = get_message_path(message_hash); + + if (!Utilities::OS::file_exists(message_path.c_str())) { + WARNING("Message file not found: " + message_path); + return false; + } + + try { + // Read existing JSON + Bytes data; + if (Utilities::OS::read_file(message_path.c_str(), data) == 0) { + ERROR("Failed to read message file: " + message_path); + return false; + } + + // Use reusable document to reduce heap fragmentation + _json_doc.clear(); + DeserializationError error = deserializeJson(_json_doc, data.data(), data.size()); + if (error) { + ERROR("Failed to parse message file: " + std::string(error.c_str())); + return false; + } + + // Update state + _json_doc["state"] = static_cast(state); + + // Write back + std::string json_str; + serializeJson(_json_doc, json_str); + if (!Utilities::OS::write_file(message_path.c_str(), Bytes((uint8_t*)json_str.c_str(), json_str.length()))) { + ERROR("Failed to write message file: " + message_path); + return false; + } + + INFO("Message state updated to " + std::to_string(static_cast(state))); + return true; + + } catch (const std::exception& e) { + ERROR("Exception updating message state: " + std::string(e.what())); + return false; + } +} + +// Delete message from storage +bool MessageStore::delete_message(const Bytes& message_hash) { + if (!_initialized) { + ERROR("MessageStore not initialized"); + return false; + } + + INFO("Deleting message: " + message_hash.toHex()); + + // Remove message file + std::string message_path = get_message_path(message_hash); + if (Utilities::OS::file_exists(message_path.c_str())) { + if (!Utilities::OS::remove_file(message_path.c_str())) { + ERROR("Failed to delete message file: " + message_path); + return false; + } + } + + // Update conversation index - remove from all conversations + for (size_t i = 0; i < MAX_CONVERSATIONS; ++i) { + ConversationSlot& slot = _conversations_pool[i]; + if (!slot.in_use) { + continue; + } + + ConversationInfo& conv = slot.info; + if (conv.remove_message_hash(message_hash)) { + // Update last message if this was it + if (conv.last_message_hash_bytes() == message_hash) { + if (conv.message_count > 0) { + conv.set_last_message_hash(conv.message_hash_bytes(conv.message_count - 1)); + } else { + memset(conv.last_message_hash, 0, MESSAGE_HASH_SIZE); + } + } + + DEBUG(" Removed from conversation"); + break; + } + } + + save_index(); + INFO("Message deleted"); + return true; +} + +// Get list of conversations (sorted by last activity) +std::vector MessageStore::get_conversations() { + std::vector> sorted; + + for (size_t i = 0; i < MAX_CONVERSATIONS; ++i) { + const ConversationSlot& slot = _conversations_pool[i]; + if (slot.in_use) { + sorted.push_back({slot.info.last_activity, slot.peer_hash_bytes()}); + } + } + + // Sort by last activity (most recent first) + std::sort(sorted.begin(), sorted.end(), + [](const std::pair& a, const std::pair& b) { return a.first > b.first; }); + + std::vector result; + for (const auto& pair : sorted) { + result.push_back(pair.second); + } + + return result; +} + +// Get conversation info +MessageStore::ConversationInfo MessageStore::get_conversation_info(const Bytes& peer_hash) { + const ConversationSlot* slot = find_conversation(peer_hash); + if (slot) { + return slot->info; + } + return ConversationInfo(); +} + +// Get messages for conversation +std::vector MessageStore::get_messages_for_conversation(const Bytes& peer_hash) { + const ConversationSlot* slot = find_conversation(peer_hash); + if (slot) { + std::vector result; + result.reserve(slot->info.message_count); + for (size_t i = 0; i < slot->info.message_count; ++i) { + result.push_back(slot->info.message_hash_bytes(i)); + } + return result; + } + return std::vector(); +} + +// Mark conversation as read +void MessageStore::mark_conversation_read(const Bytes& peer_hash) { + ConversationSlot* slot = find_conversation(peer_hash); + if (slot) { + slot->info.unread_count = 0; + save_index(); + DEBUG("Marked conversation as read: " + peer_hash.toHex()); + } +} + +// Delete entire conversation +bool MessageStore::delete_conversation(const Bytes& peer_hash) { + ConversationSlot* slot = find_conversation(peer_hash); + if (!slot) { + WARNING("Conversation not found: " + peer_hash.toHex()); + return false; + } + + INFO("Deleting conversation: " + peer_hash.toHex()); + + // Delete all message files + for (size_t i = 0; i < slot->info.message_count; ++i) { + std::string message_path = get_message_path(slot->info.message_hash_bytes(i)); + if (Utilities::OS::file_exists(message_path.c_str())) { + Utilities::OS::remove_file(message_path.c_str()); + } + } + + // Clear slot and mark as not in use + slot->clear(); + save_index(); + + INFO("Conversation deleted"); + return true; +} + +// Get total message count +size_t MessageStore::get_message_count() const { + size_t count = 0; + for (size_t i = 0; i < MAX_CONVERSATIONS; ++i) { + if (_conversations_pool[i].in_use) { + count += _conversations_pool[i].info.message_count; + } + } + return count; +} + +// Get conversation count +size_t MessageStore::get_conversation_count() const { + return count_conversations(); +} + +// Get total unread count +size_t MessageStore::get_unread_count() const { + size_t count = 0; + for (size_t i = 0; i < MAX_CONVERSATIONS; ++i) { + if (_conversations_pool[i].in_use) { + count += _conversations_pool[i].info.unread_count; + } + } + return count; +} + +// Clear all data +bool MessageStore::clear_all() { + INFO("Clearing all message store data"); + + // Delete all message files + for (size_t i = 0; i < MAX_CONVERSATIONS; ++i) { + ConversationSlot& slot = _conversations_pool[i]; + if (!slot.in_use) { + continue; + } + for (size_t j = 0; j < slot.info.message_count; ++j) { + std::string message_path = get_message_path(slot.info.message_hash_bytes(j)); + if (Utilities::OS::file_exists(message_path.c_str())) { + Utilities::OS::remove_file(message_path.c_str()); + } + } + slot.clear(); + } + + // Save empty index + save_index(); + + INFO("Message store cleared"); + return true; +} + +// Get message file path +// Use short path for SPIFFS compatibility (32 char filename limit) +// Format: /m/.j (12 chars of hash = 6 bytes = plenty unique for local store) +std::string MessageStore::get_message_path(const Bytes& message_hash) const { + return "/m/" + message_hash.toHex().substr(0, 12) + ".j"; +} + +// Get conversation directory path +std::string MessageStore::get_conversation_path(const Bytes& peer_hash) const { + return "/c/" + peer_hash.toHex().substr(0, 12); +} + +// Determine peer hash from message +Bytes MessageStore::get_peer_hash(const LXMessage& message, const Bytes& our_hash) const { + // For incoming messages: peer = source + // For outgoing messages: peer = destination + if (message.incoming()) { + return message.source_hash(); + } else { + return message.destination_hash(); + } +} + +// Find a conversation slot by peer hash +MessageStore::ConversationSlot* MessageStore::find_conversation(const Bytes& peer_hash) { + for (size_t i = 0; i < MAX_CONVERSATIONS; ++i) { + if (_conversations_pool[i].in_use && _conversations_pool[i].peer_hash_equals(peer_hash)) { + return &_conversations_pool[i]; + } + } + return nullptr; +} + +const MessageStore::ConversationSlot* MessageStore::find_conversation(const Bytes& peer_hash) const { + for (size_t i = 0; i < MAX_CONVERSATIONS; ++i) { + if (_conversations_pool[i].in_use && _conversations_pool[i].peer_hash_equals(peer_hash)) { + return &_conversations_pool[i]; + } + } + return nullptr; +} + +// Get or create a conversation slot for a peer +MessageStore::ConversationSlot* MessageStore::get_or_create_conversation(const Bytes& peer_hash) { + // First try to find existing + ConversationSlot* slot = find_conversation(peer_hash); + if (slot) { + return slot; + } + + // Find a free slot + for (size_t i = 0; i < MAX_CONVERSATIONS; ++i) { + if (!_conversations_pool[i].in_use) { + _conversations_pool[i].in_use = true; + _conversations_pool[i].set_peer_hash(peer_hash); + _conversations_pool[i].info.set_peer_hash(peer_hash); + DEBUG(" Created new conversation with: " + peer_hash.toHex()); + return &_conversations_pool[i]; + } + } + + return nullptr; // Pool is full +} + +// Count number of active conversations in pool +size_t MessageStore::count_conversations() const { + size_t count = 0; + for (size_t i = 0; i < MAX_CONVERSATIONS; ++i) { + if (_conversations_pool[i].in_use) { + ++count; + } + } + return count; +} diff --git a/src-shim/LXMF/MessageStore.h b/src-shim/LXMF/MessageStore.h new file mode 100644 index 00000000..135f3291 --- /dev/null +++ b/src-shim/LXMF/MessageStore.h @@ -0,0 +1,370 @@ +#pragma once + +#include "LXMessage.h" +#include + +#include +#include +#include + +namespace LXMF { + + // Fixed pool sizes to eliminate heap fragmentation + static constexpr size_t MAX_CONVERSATIONS = 32; + static constexpr size_t MAX_MESSAGES_PER_CONVERSATION = 256; + static constexpr size_t MESSAGE_HASH_SIZE = 32; // SHA256 hash + static constexpr size_t PEER_HASH_SIZE = 16; // Truncated hash + + /** + * @brief Message persistence and conversation management for LXMF + * + * Stores LXMF messages on the filesystem organized by conversation (peer). + * Maintains an index of conversations and message order for efficient retrieval. + * + * Storage structure: + * / + * conversations.json - Conversation index + * messages/.json - Individual message files + * conversations// - Per-conversation metadata + * + * Usage: + * MessageStore store("/path/to/storage"); + * store.save_message(message); + * + * auto conversations = store.get_conversations(); + * auto messages = store.get_messages_for_conversation(peer_hash); + * LXMessage msg = store.load_message(message_hash); + */ + class MessageStore { + + public: + /** + * @brief Conversation metadata with fixed-size message hash storage + */ + struct ConversationInfo { + // Fixed arrays eliminate ~6KB Bytes metadata overhead per conversation + // (256 messages × 24 bytes metadata = 6.1KB saved per conversation) + uint8_t peer_hash[PEER_HASH_SIZE]; + uint8_t message_hashes[MAX_MESSAGES_PER_CONVERSATION][MESSAGE_HASH_SIZE]; + size_t message_count = 0; // Number of messages in this conversation + double last_activity = 0.0; // Timestamp of most recent message + size_t unread_count = 0; // Number of unread messages + uint8_t last_message_hash[MESSAGE_HASH_SIZE]; + + // Helper methods for accessing fixed arrays as Bytes + RNS::Bytes peer_hash_bytes() const { return RNS::Bytes(peer_hash, PEER_HASH_SIZE); } + RNS::Bytes message_hash_bytes(size_t idx) const { + if (idx >= message_count) return RNS::Bytes(); + return RNS::Bytes(message_hashes[idx], MESSAGE_HASH_SIZE); + } + RNS::Bytes last_message_hash_bytes() const { return RNS::Bytes(last_message_hash, MESSAGE_HASH_SIZE); } + + void set_peer_hash(const RNS::Bytes& b) { + size_t len = std::min(b.size(), PEER_HASH_SIZE); + memcpy(peer_hash, b.data(), len); + if (len < PEER_HASH_SIZE) memset(peer_hash + len, 0, PEER_HASH_SIZE - len); + } + void set_last_message_hash(const RNS::Bytes& b) { + size_t len = std::min(b.size(), MESSAGE_HASH_SIZE); + memcpy(last_message_hash, b.data(), len); + if (len < MESSAGE_HASH_SIZE) memset(last_message_hash + len, 0, MESSAGE_HASH_SIZE - len); + } + bool peer_hash_equals(const RNS::Bytes& b) const { + if (b.size() != PEER_HASH_SIZE) return false; + return memcmp(peer_hash, b.data(), PEER_HASH_SIZE) == 0; + } + + /** + * @brief Add a message hash to this conversation + * @param hash Message hash to add + * @return True if added, false if already exists or pool full + */ + bool add_message_hash(const RNS::Bytes& hash); + + /** + * @brief Check if conversation has a specific message + * @param hash Message hash to check + * @return True if message exists in this conversation + */ + bool has_message(const RNS::Bytes& hash) const; + + /** + * @brief Remove a message hash from this conversation + * @param hash Message hash to remove + * @return True if removed, false if not found + */ + bool remove_message_hash(const RNS::Bytes& hash); + + /** + * @brief Clear all data in this conversation info + */ + void clear(); + }; + + /** + * @brief Fixed-size slot for conversation storage + */ + struct ConversationSlot { + bool in_use = false; + uint8_t peer_hash[PEER_HASH_SIZE]; + ConversationInfo info; + + // Helper methods + RNS::Bytes peer_hash_bytes() const { return RNS::Bytes(peer_hash, PEER_HASH_SIZE); } + void set_peer_hash(const RNS::Bytes& b) { + size_t len = std::min(b.size(), PEER_HASH_SIZE); + memcpy(peer_hash, b.data(), len); + if (len < PEER_HASH_SIZE) memset(peer_hash + len, 0, PEER_HASH_SIZE - len); + } + bool peer_hash_equals(const RNS::Bytes& b) const { + if (b.size() != PEER_HASH_SIZE) return false; + return memcmp(peer_hash, b.data(), PEER_HASH_SIZE) == 0; + } + + /** + * @brief Clear this slot and mark as not in use + */ + void clear(); + }; + + /** + * @brief Lightweight message metadata for fast loading + * + * Contains only fields needed for chat list display, avoiding + * expensive msgpack unpacking. + */ + struct MessageMetadata { + RNS::Bytes hash; + std::string content; + double timestamp; + bool incoming; + int state; // Type::Message::State as int + bool valid; // True if loaded successfully + }; + + public: + /** + * @brief Construct MessageStore + * + * @param base_path Base directory for message storage + */ + MessageStore(const std::string& base_path); + + ~MessageStore(); + + public: + /** + * @brief Save a message to storage + * + * Saves the message and updates the conversation index. + * Messages are organized by peer (the other party in the conversation). + * + * @param message Message to save + * @return True if saved successfully + */ + bool save_message(const LXMessage& message); + + /** + * @brief Load a message from storage + * + * @param message_hash Hash of the message to load + * @return LXMessage object (or empty if not found) + */ + LXMessage load_message(const RNS::Bytes& message_hash); + + /** + * @brief Load only message metadata (fast path for chat list) + * + * Reads content/timestamp/state directly from JSON without msgpack unpacking. + * Much faster than load_message() for displaying message lists. + * + * @param message_hash Hash of the message to load + * @return MessageMetadata struct (check .valid field) + */ + MessageMetadata load_message_metadata(const RNS::Bytes& message_hash); + + /** + * @brief Update message state in storage + * + * Updates just the state field of a stored message. + * + * @param message_hash Hash of the message to update + * @param state New state value + * @return True if updated successfully + */ + bool update_message_state(const RNS::Bytes& message_hash, Type::Message::State state); + + /** + * @brief Delete a message from storage + * + * Removes the message file and updates the conversation index. + * + * @param message_hash Hash of the message to delete + * @return True if deleted successfully + */ + bool delete_message(const RNS::Bytes& message_hash); + + /** + * @brief Get list of all conversation peer hashes + * + * Returns peer hashes sorted by last activity (most recent first). + * + * @return Vector of peer hashes + */ + std::vector get_conversations(); + + /** + * @brief Get conversation info for a peer + * + * @param peer_hash Hash of the peer + * @return ConversationInfo (or empty if not found) + */ + ConversationInfo get_conversation_info(const RNS::Bytes& peer_hash); + + /** + * @brief Get all message hashes for a conversation + * + * Returns messages in chronological order (oldest first). + * + * @param peer_hash Hash of the peer + * @return Vector of message hashes + */ + std::vector get_messages_for_conversation(const RNS::Bytes& peer_hash); + + /** + * @brief Mark all messages in conversation as read + * + * @param peer_hash Hash of the peer + */ + void mark_conversation_read(const RNS::Bytes& peer_hash); + + /** + * @brief Delete entire conversation + * + * Removes all messages and conversation metadata. + * + * @param peer_hash Hash of the peer + * @return True if deleted successfully + */ + bool delete_conversation(const RNS::Bytes& peer_hash); + + /** + * @brief Get total number of stored messages + * + * @return Message count + */ + size_t get_message_count() const; + + /** + * @brief Get total number of conversations + * + * @return Conversation count + */ + size_t get_conversation_count() const; + + /** + * @brief Get total unread message count across all conversations + * + * @return Unread message count + */ + size_t get_unread_count() const; + + /** + * @brief Clear all stored messages and conversations + * + * WARNING: This permanently deletes all data. + * + * @return True if cleared successfully + */ + bool clear_all(); + + private: + /** + * @brief Initialize storage directories + * + * Creates base_path, messages/, and conversations/ directories if needed. + * + * @return True if initialized successfully + */ + bool initialize_storage(); + + /** + * @brief Load conversation index from disk + * + * Loads conversations.json into _conversations_pool. + */ + void load_index(); + + /** + * @brief Save conversation index to disk + * + * Persists _conversations_pool to conversations.json. + * + * @return True if saved successfully + */ + bool save_index(); + + /** + * @brief Get filesystem path for a message file + * + * @param message_hash Hash of the message + * @return Full path to message JSON file + */ + std::string get_message_path(const RNS::Bytes& message_hash) const; + + /** + * @brief Get filesystem path for conversation directory + * + * @param peer_hash Hash of the peer + * @return Full path to conversation directory + */ + std::string get_conversation_path(const RNS::Bytes& peer_hash) const; + + /** + * @brief Determine peer hash from message + * + * For incoming messages: peer = source + * For outgoing messages: peer = destination + * + * @param message The message + * @param our_hash Our local identity hash + * @return Peer hash + */ + RNS::Bytes get_peer_hash(const LXMessage& message, const RNS::Bytes& our_hash) const; + + /** + * @brief Find a conversation slot by peer hash + * + * @param peer_hash Hash of the peer + * @return Pointer to ConversationSlot or nullptr if not found + */ + ConversationSlot* find_conversation(const RNS::Bytes& peer_hash); + const ConversationSlot* find_conversation(const RNS::Bytes& peer_hash) const; + + /** + * @brief Get or create a conversation slot for a peer + * + * @param peer_hash Hash of the peer + * @return Pointer to ConversationSlot or nullptr if pool is full + */ + ConversationSlot* get_or_create_conversation(const RNS::Bytes& peer_hash); + + /** + * @brief Count number of active conversations in pool + * + * @return Number of in-use conversation slots + */ + size_t count_conversations() const; + + private: + std::string _base_path; + ConversationSlot _conversations_pool[MAX_CONVERSATIONS]; + bool _initialized; + + // Reusable JSON document to reduce heap fragmentation + // Note: This class is assumed to be used from a single thread (main loop). + // If called from multiple threads, this would need per-thread documents or locking. + JsonDocument _json_doc; + }; + +} // namespace LXMF diff --git a/src-shim/LXMF/PropagationNodeManager.cpp b/src-shim/LXMF/PropagationNodeManager.cpp new file mode 100644 index 00000000..06315791 --- /dev/null +++ b/src-shim/LXMF/PropagationNodeManager.cpp @@ -0,0 +1,326 @@ +#include "PropagationNodeManager.h" +#include +#include + +#include +#include + +using namespace LXMF; +using namespace RNS; + +PropagationNodeManager::PropagationNodeManager() + : AnnounceHandler("lxmf.propagation") +{ + INFO("PropagationNodeManager: initialized with aspect filter 'lxmf.propagation'"); +} + +void PropagationNodeManager::received_announce( + const Bytes& destination_hash, + const Identity& announced_identity, + const Bytes& app_data +) { + std::string hash_str = destination_hash.toHex().substr(0, 16); + TRACE("PropagationNodeManager::received_announce from " + hash_str + "..."); + + if (!app_data || app_data.size() == 0) { + WARNING("PropagationNodeManager: Received announce with empty app_data"); + return; + } + + PropagationNodeInfo info = parse_announce_data(app_data); + if (info.node_hash.size() == 0 && info.name.empty()) { + WARNING("PropagationNodeManager: Failed to parse announce app_data"); + return; + } + + info.node_hash = destination_hash; + info.last_seen = Utilities::OS::time(); + + // Get hop count from Transport + info.hops = Transport::hops_to(destination_hash); + + // Check if this is an update to an existing node + PropagationNodeSlot* existing_slot = find_node_slot(destination_hash); + bool is_update = (existing_slot != nullptr); + + // Store/update node + PropagationNodeSlot* slot = existing_slot; + if (!slot) { + slot = find_empty_node_slot(); + if (!slot) { + WARNING("PropagationNodeManager: Pool full, cannot add node " + hash_str); + return; + } + } + + slot->in_use = true; + slot->node_hash = destination_hash; + slot->info = info; + + std::string action = is_update ? "Updated" : "Discovered"; + if (info.enabled) { + INFO("PropagationNodeManager: " + action + + " propagation node '" + info.name + "' at " + + destination_hash.toHex().substr(0, 16) + "... (" + + std::to_string(info.hops) + " hops)"); + } else { + INFO("PropagationNodeManager: Node " + destination_hash.toHex().substr(0, 16) + + "... reports propagation disabled"); + } + + // Notify listeners + if (_update_callback) { + _update_callback(); + } +} + +PropagationNodeInfo PropagationNodeManager::parse_announce_data(const Bytes& app_data) { + PropagationNodeInfo info; + + try { + MsgPack::Unpacker unpacker; + unpacker.feed(app_data.data(), app_data.size()); + + // Expect array with 7 elements + MsgPack::arr_size_t arr_size; + unpacker.deserialize(arr_size); + if (arr_size.size() < 7) { + WARNING("PropagationNodeManager: Invalid app_data array size: " + std::to_string(arr_size.size())); + return {}; + } + + // [0] Legacy flag (bool) - skip + bool legacy_flag; + unpacker.deserialize(legacy_flag); + + // [1] Node timebase (int64) + int64_t timebase; + unpacker.deserialize(timebase); + info.timebase = static_cast(timebase); + + // [2] Propagation enabled (bool) + unpacker.deserialize(info.enabled); + + // [3] Per-transfer limit (int) + int64_t transfer_limit; + unpacker.deserialize(transfer_limit); + info.transfer_limit = static_cast(transfer_limit); + + // [4] Per-sync limit (int) + int64_t sync_limit; + unpacker.deserialize(sync_limit); + info.sync_limit = static_cast(sync_limit); + + // [5] Stamp costs array: [cost, flexibility, peering_cost] + MsgPack::arr_size_t costs_size; + unpacker.deserialize(costs_size); + if (costs_size.size() >= 3) { + int64_t cost, flexibility, peering; + unpacker.deserialize(cost); + unpacker.deserialize(flexibility); + unpacker.deserialize(peering); + info.stamp_cost = static_cast(cost); + info.stamp_flexibility = static_cast(flexibility); + info.peering_cost = static_cast(peering); + } + + // [6] Metadata dict + MsgPack::map_size_t map_size; + unpacker.deserialize(map_size); + for (size_t i = 0; i < map_size.size(); i++) { + int64_t key; + unpacker.deserialize(key); + + if (key == PN_META_NAME) { + // Name is a binary/string + MsgPack::bin_t name_bin; + unpacker.deserialize(name_bin); + info.name = std::string(name_bin.begin(), name_bin.end()); + } else { + // Skip other metadata fields by reading and discarding + // Try to read as binary first (most common), fall back to int + try { + MsgPack::bin_t skip_bin; + unpacker.deserialize(skip_bin); + } catch (...) { + try { + int64_t skip_int; + unpacker.deserialize(skip_int); + } catch (...) { + // Give up - might be complex type + } + } + } + } + + // Default name if none provided + if (info.name.empty()) { + info.name = "Propagation Node"; + } + + // Mark as valid by setting a dummy value that will be overwritten + info.hops = 0; + return info; + + } catch (const std::exception& e) { + WARNING("PropagationNodeManager: Exception parsing app_data: " + std::string(e.what())); + return {}; + } +} + +std::vector PropagationNodeManager::get_nodes() const { + std::vector result; + result.reserve(nodes_count()); + + for (size_t i = 0; i < MAX_PROPAGATION_NODES; ++i) { + if (_nodes_pool[i].in_use) { + result.push_back(_nodes_pool[i].info); + } + } + + // Sort by hops (closest first), then by last_seen (most recent first) + std::sort(result.begin(), result.end(), [](const PropagationNodeInfo& a, const PropagationNodeInfo& b) { + if (a.hops != b.hops) { + return a.hops < b.hops; + } + return a.last_seen > b.last_seen; + }); + + return result; +} + +PropagationNodeInfo PropagationNodeManager::get_node(const Bytes& hash) const { + const PropagationNodeSlot* slot = find_node_slot(hash); + if (slot) { + return slot->info; + } + return {}; +} + +bool PropagationNodeManager::has_node(const Bytes& hash) const { + return find_node_slot(hash) != nullptr; +} + +void PropagationNodeManager::set_selected_node(const Bytes& hash) { + if (hash.size() == 0) { + _selected_node = {}; + INFO("PropagationNodeManager: Cleared manual node selection"); + return; + } + + if (!has_node(hash)) { + WARNING("PropagationNodeManager: Cannot select unknown node " + hash.toHex().substr(0, 16)); + return; + } + + _selected_node = hash; + PropagationNodeInfo node = get_node(hash); + INFO("PropagationNodeManager: Selected node '" + node.name + "' (" + + hash.toHex().substr(0, 16) + "...)"); +} + +Bytes PropagationNodeManager::get_best_node() const { + PropagationNodeInfo best; + best.hops = 0xFF; + best.last_seen = 0; + + for (size_t i = 0; i < MAX_PROPAGATION_NODES; ++i) { + if (!_nodes_pool[i].in_use) { + continue; + } + const PropagationNodeInfo& node = _nodes_pool[i].info; + + // Skip disabled nodes + if (!node.enabled) { + continue; + } + + // Check if this node is better + bool is_better = false; + if (node.hops < best.hops) { + is_better = true; + } else if (node.hops == best.hops && node.last_seen > best.last_seen) { + is_better = true; + } + + if (is_better) { + best = node; + } + } + + if (best.node_hash.size() > 0) { + TRACE("PropagationNodeManager: Best node is '" + best.name + "' (" + + std::to_string(best.hops) + " hops)"); + } + + return best.node_hash; +} + +Bytes PropagationNodeManager::get_effective_node() const { + if (_selected_node.size() > 0) { + // Verify selected node is still valid + const PropagationNodeSlot* slot = find_node_slot(_selected_node); + if (slot && slot->info.enabled) { + return _selected_node; + } + } + + // Fall back to auto-selection + return get_best_node(); +} + +void PropagationNodeManager::clean_stale_nodes() { + double now = Utilities::OS::time(); + bool removed_any = false; + + for (size_t i = 0; i < MAX_PROPAGATION_NODES; ++i) { + if (_nodes_pool[i].in_use && + now - _nodes_pool[i].info.last_seen > NODE_STALE_TIMEOUT) { + INFO("PropagationNodeManager: Removing stale node " + + _nodes_pool[i].node_hash.toHex().substr(0, 16) + "..."); + _nodes_pool[i].clear(); + removed_any = true; + } + } + + if (removed_any && _update_callback) { + _update_callback(); + } +} + +PropagationNodeSlot* PropagationNodeManager::find_node_slot(const Bytes& hash) { + for (size_t i = 0; i < MAX_PROPAGATION_NODES; ++i) { + if (_nodes_pool[i].in_use && _nodes_pool[i].node_hash == hash) { + return &_nodes_pool[i]; + } + } + return nullptr; +} + +const PropagationNodeSlot* PropagationNodeManager::find_node_slot(const Bytes& hash) const { + for (size_t i = 0; i < MAX_PROPAGATION_NODES; ++i) { + if (_nodes_pool[i].in_use && _nodes_pool[i].node_hash == hash) { + return &_nodes_pool[i]; + } + } + return nullptr; +} + +PropagationNodeSlot* PropagationNodeManager::find_empty_node_slot() { + for (size_t i = 0; i < MAX_PROPAGATION_NODES; ++i) { + if (!_nodes_pool[i].in_use) { + return &_nodes_pool[i]; + } + } + return nullptr; +} + +size_t PropagationNodeManager::nodes_count() const { + size_t count = 0; + for (size_t i = 0; i < MAX_PROPAGATION_NODES; ++i) { + if (_nodes_pool[i].in_use) { + ++count; + } + } + return count; +} diff --git a/src-shim/LXMF/PropagationNodeManager.h b/src-shim/LXMF/PropagationNodeManager.h new file mode 100644 index 00000000..1189160e --- /dev/null +++ b/src-shim/LXMF/PropagationNodeManager.h @@ -0,0 +1,233 @@ +#pragma once + +#include "Type.h" +#include +#include +#include + +#include +#include +#include + +namespace LXMF { + + // Maximum number of propagation nodes to track (fixed pool size) + static constexpr size_t MAX_PROPAGATION_NODES = 32; + + /** + * @brief Information about a discovered propagation node + */ + struct PropagationNodeInfo { + RNS::Bytes node_hash; // Destination hash of the propagation node + std::string name; // Display name from metadata + double timebase = 0; // Node's timebase from announce + bool enabled = false; // Whether propagation is enabled + uint32_t transfer_limit = 0; // Per-transfer limit in KB + uint32_t sync_limit = 0; // Per-sync limit in KB + uint8_t stamp_cost = 0; // Required stamp cost + uint8_t stamp_flexibility = 0; // Stamp cost flexibility + uint8_t peering_cost = 0; // Cost for peering + uint8_t hops = 0xFF; // Hop count (0xFF = unknown) + double last_seen = 0; // Timestamp of last announce + + // Check if this node info is valid + operator bool() const { return node_hash.size() > 0; } + }; + + /** + * @brief Fixed-size slot for propagation node storage + * + * Used to avoid heap fragmentation from std::map operations. + */ + struct PropagationNodeSlot { + bool in_use = false; + RNS::Bytes node_hash; // key for lookup + PropagationNodeInfo info; + + void clear() { + in_use = false; + node_hash = RNS::Bytes(); + info = PropagationNodeInfo(); + } + }; + + /** + * @brief Manages discovery and tracking of LXMF propagation nodes + * + * Listens for propagation node announces (aspect "lxmf.propagation") and + * maintains a list of known nodes. Supports manual selection and auto-selection + * of the best available node. + * + * Usage: + * PropagationNodeManager manager; + * Transport::register_announce_handler(HAnnounceHandler(&manager)); + * + * // Get list of known nodes + * auto nodes = manager.get_nodes(); + * + * // Auto-select best node + * Bytes best = manager.get_best_node(); + */ + class PropagationNodeManager : public RNS::AnnounceHandler { + public: + using NodeUpdateCallback = std::function; + + // Propagation node metadata keys (from Python LXMF) + static constexpr uint8_t PN_META_VERSION = 0x00; + static constexpr uint8_t PN_META_NAME = 0x01; + static constexpr uint8_t PN_META_SYNC_STRATUM = 0x02; + static constexpr uint8_t PN_META_SYNC_THROTTLE = 0x03; + static constexpr uint8_t PN_META_AUTH_BAND = 0x04; + static constexpr uint8_t PN_META_UTIL_PRESSURE = 0x05; + static constexpr uint8_t PN_META_CUSTOM = 0xFF; + + // Stale node timeout (1 hour) + static constexpr double NODE_STALE_TIMEOUT = 3600.0; + + public: + /** + * @brief Construct PropagationNodeManager + * + * Registers for "lxmf.propagation" announces. + */ + PropagationNodeManager(); + virtual ~PropagationNodeManager() = default; + + /** + * @brief AnnounceHandler callback + * + * Called by Transport when a propagation node announce is received. + */ + virtual void received_announce( + const RNS::Bytes& destination_hash, + const RNS::Identity& announced_identity, + const RNS::Bytes& app_data + ) override; + + /** + * @brief Get list of all known propagation nodes + * + * @return Vector of node info, sorted by hops (closest first) + */ + std::vector get_nodes() const; + + /** + * @brief Get info for a specific node + * + * @param hash Destination hash of the node + * @return Node info (or empty if not found) + */ + PropagationNodeInfo get_node(const RNS::Bytes& hash) const; + + /** + * @brief Check if a node is known + * + * @param hash Destination hash of the node + * @return True if node is known + */ + bool has_node(const RNS::Bytes& hash) const; + + /** + * @brief Get number of known nodes + * + * @return Number of nodes + */ + size_t node_count() const { return nodes_count(); } + + /** + * @brief Manually select a propagation node + * + * @param hash Destination hash of the node to select (empty to clear) + */ + void set_selected_node(const RNS::Bytes& hash); + + /** + * @brief Get the manually selected node + * + * @return Destination hash of selected node (or empty if none) + */ + RNS::Bytes get_selected_node() const { return _selected_node; } + + /** + * @brief Auto-select the best available propagation node + * + * Selection criteria: + * 1. Node must be enabled + * 2. Prefer nodes with fewer hops + * 3. Prefer nodes seen more recently + * + * @return Destination hash of best node (or empty if none available) + */ + RNS::Bytes get_best_node() const; + + /** + * @brief Get the effective propagation node (selected or best) + * + * Returns the manually selected node if set, otherwise auto-selects. + * + * @return Destination hash of effective node + */ + RNS::Bytes get_effective_node() const; + + /** + * @brief Set callback for node list updates + * + * @param callback Function to call when nodes are added/updated + */ + void set_update_callback(NodeUpdateCallback callback) { _update_callback = callback; } + + /** + * @brief Clean up stale nodes + * + * Removes nodes that haven't been seen recently. + */ + void clean_stale_nodes(); + + private: + /** + * @brief Parse announce app_data into PropagationNodeInfo + * + * App data format (msgpack): + * [0] Legacy flag (bool) + * [1] Node timebase (timestamp) + * [2] Propagation enabled (bool) + * [3] Per-transfer limit (KB) + * [4] Per-sync limit (KB) + * [5] [stamp_cost, flexibility, peering_cost] + * [6] Metadata dict + * + * @param app_data Raw announce app_data + * @return Parsed node info + */ + PropagationNodeInfo parse_announce_data(const RNS::Bytes& app_data); + + /** + * @brief Find a node slot by hash + * + * @param hash Destination hash to search for + * @return Pointer to slot if found, nullptr otherwise + */ + PropagationNodeSlot* find_node_slot(const RNS::Bytes& hash); + const PropagationNodeSlot* find_node_slot(const RNS::Bytes& hash) const; + + /** + * @brief Find an empty slot in the pool + * + * @return Pointer to empty slot, nullptr if pool is full + */ + PropagationNodeSlot* find_empty_node_slot(); + + /** + * @brief Get the number of nodes currently in use + * + * @return Number of active nodes in the pool + */ + size_t nodes_count() const; + + private: + PropagationNodeSlot _nodes_pool[MAX_PROPAGATION_NODES]; + RNS::Bytes _selected_node; + NodeUpdateCallback _update_callback; + }; + +} // namespace LXMF diff --git a/src-shim/LXMF/Type.h b/src-shim/LXMF/Type.h new file mode 100644 index 00000000..0e468261 --- /dev/null +++ b/src-shim/LXMF/Type.h @@ -0,0 +1,161 @@ +#pragma once + +#include // microReticulum's Type.h, resolved via -I deps/microReticulum/src + +#include + +namespace LXMF { + + /** + * @brief LXMF (Lightweight Extensible Messaging Format) type definitions + * + * This namespace contains all constants, enums, and type definitions for + * the LXMF messaging layer built on top of Reticulum Network Stack. + */ + namespace Type { + + /** + * @brief Message states during lifecycle + */ + namespace Message { + enum State : uint8_t { + GENERATING = 0x00, ///< Message is being generated + OUTBOUND = 0x01, ///< Message queued for sending + SENDING = 0x02, ///< Message is being sent + SENT = 0x04, ///< Message sent successfully + DELIVERED = 0x08, ///< Message delivered and confirmed + REJECTED = 0xFD, ///< Message rejected by recipient + CANCELLED = 0xFE, ///< Message sending cancelled + FAILED = 0xFF ///< Message sending failed + }; + + /** + * @brief Message representation format + */ + enum Representation : uint8_t { + UNKNOWN = 0x00, ///< Unknown representation + PACKET = 0x01, ///< Single packet + RESOURCE = 0x02 ///< Multi-packet resource + }; + + /** + * @brief Message delivery methods + */ + enum Method : uint8_t { + OPPORTUNISTIC = 0x01, ///< Single packet, fire-and-forget + DIRECT = 0x02, ///< Via established link (Phase 1 MVP) + PROPAGATED = 0x03, ///< Store-and-forward via propagation nodes + PAPER = 0x05 ///< QR code / paper-based transfer + }; + + /** + * @brief Reasons why message verification failed + */ + enum UnverifiedReason : uint8_t { + SOURCE_UNKNOWN = 0x01, ///< Source identity unknown + SIGNATURE_INVALID = 0x02 ///< Ed25519 signature invalid + }; + } + + /** + * @brief LXMF protocol constants + * + * These constants define the structure and size limits of LXMF messages. + * They are derived from the underlying Reticulum network stack parameters. + */ + namespace Constants { + // Identity and signature sizes (from RNS) + static const uint8_t DESTINATION_LENGTH = RNS::Type::Identity::TRUNCATED_HASHLENGTH / 8; // 16 bytes + static const uint8_t SIGNATURE_LENGTH = RNS::Type::Identity::SIGLENGTH / 8; // 64 bytes + static const uint8_t TICKET_LENGTH = RNS::Type::Identity::TRUNCATED_HASHLENGTH / 8; // 16 bytes + + // Ticket expiry and renewal (in seconds) + static const uint32_t TICKET_EXPIRY = 21 * 24 * 60 * 60; // 3 weeks + static const uint32_t TICKET_GRACE = 5 * 24 * 60 * 60; // 5 days grace period + static const uint32_t TICKET_RENEW = 14 * 24 * 60 * 60; // Renew when <14 days remain + static const uint32_t TICKET_INTERVAL = 1 * 24 * 60 * 60; // Check daily + static const uint16_t COST_TICKET = 0x100; + + // LXMF message structure overhead + static const uint8_t TIMESTAMP_SIZE = 8; // 8 bytes for timestamp + static const uint8_t STRUCT_OVERHEAD = 8; // 8 bytes for msgpack structure + + /** + * @brief Total LXMF overhead per message: 112 bytes + * + * Breakdown: + * - 16 bytes: destination hash + * - 16 bytes: source hash + * - 64 bytes: Ed25519 signature + * - 8 bytes: timestamp + * - 8 bytes: msgpack structure overhead + */ + static const uint8_t LXMF_OVERHEAD = 2 * DESTINATION_LENGTH + SIGNATURE_LENGTH + TIMESTAMP_SIZE + STRUCT_OVERHEAD; + + // Maximum Data Units (MDU) for different transport modes + // With an MTU of 500, encrypted packet MDU is 391 bytes + static const uint16_t ENCRYPTED_PACKET_MDU = RNS::Type::Packet::ENCRYPTED_MDU + TIMESTAMP_SIZE; // 391 bytes + + /** + * @brief Max content in single encrypted packet: 295 bytes + * + * Calculation: ENCRYPTED_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH + * We add DESTINATION_LENGTH because we can infer it from the packet header. + */ + static const uint16_t ENCRYPTED_PACKET_MAX_CONTENT = ENCRYPTED_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH; + + /** + * @brief Link packet MDU: 431 bytes + * + * Links have less overhead per packet than standalone packets. + */ + static const uint16_t LINK_PACKET_MDU = RNS::Type::Link::MDU; + + /** + * @brief Max content in single link packet: 319 bytes (Phase 1 MVP limit) + * + * Calculation: LINK_PACKET_MDU - LXMF_OVERHEAD + * Messages larger than 319 bytes will use Resource transfer. + */ + static const uint16_t LINK_PACKET_MAX_CONTENT = LINK_PACKET_MDU - LXMF_OVERHEAD; + + // Plain (unencrypted) packet MDU + static const uint16_t PLAIN_PACKET_MDU = RNS::Type::Packet::PLAIN_MDU; + + /** + * @brief Max content in plain packet: 368 bytes + * + * For unencrypted messages (rarely used in practice). + */ + static const uint16_t PLAIN_PACKET_MAX_CONTENT = PLAIN_PACKET_MDU - LXMF_OVERHEAD + DESTINATION_LENGTH; + + // QR/Paper encoding constants + static const uint16_t QR_MAX_STORAGE = 2953; // Max bytes in QR code + static const uint16_t PAPER_MDU = ((QR_MAX_STORAGE - 6) * 6) / 8; // "lxm://" = 6 chars + + // Stamp proof-of-work constants + static const uint8_t STAMP_SIZE = 32; // 256-bit stamp + static const uint16_t STAMP_WORKBLOCK_ROUNDS = 3000; // Direct message stamps + static const uint16_t STAMP_WORKBLOCK_ROUNDS_PN = 1000; // Propagation node stamps + } + + /** + * @brief Encryption type descriptions for user display + */ + namespace Encryption { + static const char* DESCRIPTION_AES = "AES-128"; + static const char* DESCRIPTION_EC = "Curve25519"; + static const char* DESCRIPTION_UNENCRYPTED = "Unencrypted"; + } + + /** + * @brief URI schema for QR/paper encoding + */ + namespace URI { + static const char* SCHEMA = "lxm"; + static const char* QR_ERROR_CORRECTION = "ERROR_CORRECT_L"; + } + + } // namespace Type + +} // namespace LXMF diff --git a/src-shim/MessageBase.h b/src-shim/MessageBase.h new file mode 100644 index 00000000..8b5c91d7 --- /dev/null +++ b/src-shim/MessageBase.h @@ -0,0 +1,26 @@ +#pragma once + +#ifndef RNS_MESSAGE_BASE_H +#define RNS_MESSAGE_BASE_H + +#include "Bytes.h" + +namespace RNS { + +class MessageBase { +public: + virtual ~MessageBase() = default; + + // Get the message type identifier (unique per message class) + virtual uint16_t msgtype() const = 0; + + // Serialize message data to bytes (for wire transmission) + virtual Bytes pack() const = 0; + + // Deserialize message data from bytes + virtual void unpack(const Bytes& data) = 0; +}; + +} // namespace RNS + +#endif // RNS_MESSAGE_BASE_H diff --git a/src-shim/ObjectPool.h b/src-shim/ObjectPool.h new file mode 100644 index 00000000..63a4b337 --- /dev/null +++ b/src-shim/ObjectPool.h @@ -0,0 +1,160 @@ +#pragma once + +#include +#include +#include +#include + +// FreeRTOS spinlock support - only on ESP32 +#if defined(ESP_PLATFORM) || defined(ARDUINO) +#include "freertos/FreeRTOS.h" +#include "freertos/portmacro.h" +#define OBJECTPOOL_USE_SPINLOCK 1 +#else +// Native build - use std::mutex instead +#include +#define OBJECTPOOL_USE_SPINLOCK 0 +#endif + +namespace RNS { + +/** + * Fixed-size object pool with O(1) allocate/deallocate. + * Thread-safe via spinlock (ESP32) or mutex (native). + * Falls back to nullptr on exhaustion. + * + * Template parameters: + * T - Object type to pool + * N - Pool capacity (number of slots) + * + * Usage: + * ObjectPool pool; + * MyClass* obj = pool.allocate(); // nullptr if exhausted + * if (obj) { + * // use object... + * pool.deallocate(obj); + * } + * + * Design notes: + * - Freelist stored in-place using union with storage + * - O(1) allocate: pop from freelist head + * - O(1) deallocate: push to freelist head + * - Placement new for construction, explicit destructor for cleanup + * - Pool exhaustion returns nullptr (caller should fall back to heap) + */ +template +class ObjectPool { +public: + ObjectPool() : _first_free(0), _allocated_count(0) { +#if OBJECTPOOL_USE_SPINLOCK + portMUX_INITIALIZE(&_mux); +#endif + // Initialize freelist chain + for (size_t i = 0; i < N - 1; i++) { + _slots[i].next_free = i + 1; + } + _slots[N - 1].next_free = INVALID_SLOT; + } + + /** + * Allocate object from pool. + * Returns nullptr if pool exhausted (caller should fall back to heap). + * Thread-safe. + * + * Supports variadic constructor arguments via perfect forwarding. + * Example: pool.allocate(arg1, arg2) calls T(arg1, arg2) + */ + template + T* allocate(Args&&... args) { +#if OBJECTPOOL_USE_SPINLOCK + portENTER_CRITICAL(&_mux); +#else + std::lock_guard lock(_mutex); +#endif + if (_first_free == INVALID_SLOT) { +#if OBJECTPOOL_USE_SPINLOCK + portEXIT_CRITICAL(&_mux); +#endif + return nullptr; // Pool exhausted + } + size_t slot = _first_free; + _first_free = _slots[slot].next_free; + _allocated_count++; +#if OBJECTPOOL_USE_SPINLOCK + portEXIT_CRITICAL(&_mux); +#endif + + // Placement new to construct object with forwarded arguments + return new (&_slots[slot].storage) T(std::forward(args)...); + } + + /** + * Return object to pool. + * Pointer must have been obtained from this pool's allocate(). + * Thread-safe. + */ + void deallocate(T* ptr) { + if (!ptr) return; + + // Calculate slot index from pointer + uintptr_t offset = reinterpret_cast(ptr) - + reinterpret_cast(_slots); + size_t slot = offset / sizeof(Slot); + + if (slot >= N) { + // Not from this pool - ignore (caller's responsibility) + return; + } + + // Explicit destructor call + ptr->~T(); + +#if OBJECTPOOL_USE_SPINLOCK + portENTER_CRITICAL(&_mux); +#else + std::lock_guard lock(_mutex); +#endif + _slots[slot].next_free = _first_free; + _first_free = slot; + _allocated_count--; +#if OBJECTPOOL_USE_SPINLOCK + portEXIT_CRITICAL(&_mux); +#endif + } + + /** + * Check if pointer was allocated from this pool. + */ + bool owns(T* ptr) const { + if (!ptr) return false; + uintptr_t start = reinterpret_cast(_slots); + uintptr_t end = start + sizeof(_slots); + uintptr_t addr = reinterpret_cast(ptr); + return addr >= start && addr < end; + } + + size_t allocated() const { return _allocated_count; } + size_t capacity() const { return N; } + size_t available() const { return N - _allocated_count; } + +private: + static constexpr size_t INVALID_SLOT = ~size_t(0); + + struct Slot { + union { + alignas(T) char storage[sizeof(T)]; + size_t next_free; + }; + }; + + Slot _slots[N]; + size_t _first_free; + size_t _allocated_count; +#if OBJECTPOOL_USE_SPINLOCK + portMUX_TYPE _mux; +#else + std::mutex _mutex; +#endif +}; + +} // namespace RNS diff --git a/src-shim/PSRAMAllocator.h b/src-shim/PSRAMAllocator.h new file mode 100644 index 00000000..3ab66d1e --- /dev/null +++ b/src-shim/PSRAMAllocator.h @@ -0,0 +1,87 @@ +/* + * PSRAMAllocator.h - Custom STL allocator for ESP32 PSRAM + * + * Routes std::vector allocations to PSRAM (external RAM) when available, + * falling back to internal heap if PSRAM allocation fails. + * + * This helps prevent internal heap fragmentation by keeping large + * data structures in PSRAM. + * + * Based on: https://github.com/atanisoft/esp32usb/blob/master/include/psram_allocator.h + */ +#pragma once + +#include +#include + +#ifdef ARDUINO +#include +#else +// Non-ESP32 platforms: use standard allocation +#include +#endif + +template +class PSRAMAllocator +{ +public: + using value_type = T; + + PSRAMAllocator() noexcept {} + + template + constexpr PSRAMAllocator(const PSRAMAllocator&) noexcept {} + + [[nodiscard]] value_type* allocate(std::size_t n) + { +#ifdef ARDUINO +#if CONFIG_SPIRAM || defined(BOARD_HAS_PSRAM) + // Attempt to allocate in PSRAM first + auto p = static_cast( + heap_caps_malloc(n * sizeof(value_type), MALLOC_CAP_SPIRAM)); + if (p) + { + return p; + } +#endif // CONFIG_SPIRAM || BOARD_HAS_PSRAM + + // If PSRAM allocation failed (or not available), try default memory pool + auto p2 = static_cast( + heap_caps_malloc(n * sizeof(value_type), MALLOC_CAP_DEFAULT)); + if (p2) + { + return p2; + } +#else + // Non-Arduino: use standard malloc + auto p = static_cast(std::malloc(n * sizeof(value_type))); + if (p) + { + return p; + } +#endif + + throw std::bad_alloc(); + } + + void deallocate(value_type* p, std::size_t) noexcept + { +#ifdef ARDUINO + heap_caps_free(p); +#else + std::free(p); +#endif + } +}; + +template +bool operator==(const PSRAMAllocator&, const PSRAMAllocator&) +{ + return true; +} + +template +bool operator!=(const PSRAMAllocator& x, const PSRAMAllocator& y) +{ + return !(x == y); +} diff --git a/src-shim/SegmentAccumulator.cpp b/src-shim/SegmentAccumulator.cpp new file mode 100644 index 00000000..f9566376 --- /dev/null +++ b/src-shim/SegmentAccumulator.cpp @@ -0,0 +1,231 @@ +#include "SegmentAccumulator.h" +#include "Resource.h" + +using namespace RNS; +using namespace RNS::Utilities; + +SegmentAccumulator::SegmentAccumulator(AccumulatedCallback callback) + : _accumulated_callback(callback) +{ +} + +void SegmentAccumulator::set_accumulated_callback(AccumulatedCallback callback) { + _accumulated_callback = callback; +} + +void SegmentAccumulator::set_segment_callback(SegmentCallback callback) { + _segment_callback = callback; +} + +SegmentAccumulator::PendingTransferSlot* SegmentAccumulator::find_slot(const Bytes& transfer_id) { + for (size_t i = 0; i < MAX_PENDING_TRANSFERS; i++) { + if (_pending_pool[i].in_use && _pending_pool[i].transfer_id == transfer_id) { + return &_pending_pool[i]; + } + } + return nullptr; +} + +const SegmentAccumulator::PendingTransferSlot* SegmentAccumulator::find_slot(const Bytes& transfer_id) const { + for (size_t i = 0; i < MAX_PENDING_TRANSFERS; i++) { + if (_pending_pool[i].in_use && _pending_pool[i].transfer_id == transfer_id) { + return &_pending_pool[i]; + } + } + return nullptr; +} + +SegmentAccumulator::PendingTransferSlot* SegmentAccumulator::allocate_slot(const Bytes& transfer_id) { + for (size_t i = 0; i < MAX_PENDING_TRANSFERS; i++) { + if (!_pending_pool[i].in_use) { + _pending_pool[i].in_use = true; + _pending_pool[i].transfer_id = transfer_id; + return &_pending_pool[i]; + } + } + return nullptr; // Pool is full +} + +bool SegmentAccumulator::segment_completed(const Resource& resource) { + // Check if this is a multi-segment resource + if (!resource.is_segmented()) { + // Single-segment resource - caller should handle normally + return false; + } + + int segment_index = resource.segment_index(); + int total_segments = resource.total_segments(); + Bytes original_hash = resource.original_hash(); + + // Use resource hash as fallback if original_hash not set + if (!original_hash) { + original_hash = resource.hash(); + DEBUG("SegmentAccumulator: No original_hash, using resource hash as key"); + } + + std::string hash_short = original_hash.toHex().substr(0, 16); + DEBUGF("SegmentAccumulator: Received segment %d/%d for %s (%zu bytes)", + segment_index, total_segments, + hash_short.c_str(), + resource.data().size()); + + // Validate total_segments doesn't exceed our fixed array size + if (total_segments > static_cast(MAX_SEGMENTS_PER_TRANSFER)) { + ERRORF("SegmentAccumulator: Transfer has %d segments, exceeds max %zu", + total_segments, MAX_SEGMENTS_PER_TRANSFER); + return true; // We handled it (by rejecting) + } + + double now = OS::time(); + + // Find or create pending transfer + PendingTransferSlot* slot = find_slot(original_hash); + if (slot == nullptr) { + // New transfer - allocate a slot + slot = allocate_slot(original_hash); + if (slot == nullptr) { + ERRORF("SegmentAccumulator: Cannot track transfer %s, pool full (%zu max)", + hash_short.c_str(), MAX_PENDING_TRANSFERS); + return true; // We handled it (by rejecting due to pool exhaustion) + } + + // Initialize the transfer + PendingTransfer& transfer = slot->transfer; + transfer.original_hash = original_hash; + transfer.total_segments = total_segments; + transfer.received_count = 0; + transfer.segment_count = static_cast(total_segments); + transfer.started_at = now; + transfer.last_activity = now; + + // Initialize segment slots + for (int i = 0; i < total_segments; i++) { + transfer.segments[i].segment_index = i + 1; + transfer.segments[i].received = false; + } + + INFOF("SegmentAccumulator: Started tracking %d-segment transfer for %s", + total_segments, hash_short.c_str()); + } + + PendingTransfer& transfer = slot->transfer; + transfer.last_activity = now; + + // Validate segment index + if (segment_index < 1 || segment_index > transfer.total_segments) { + WARNINGF("SegmentAccumulator: Invalid segment_index %d (expected 1-%d)", + segment_index, transfer.total_segments); + return true; // We handled it (by rejecting) + } + + // Store segment data (segment_index is 1-based) + int idx = segment_index - 1; + if (!transfer.segments[idx].received) { + transfer.segments[idx].data = resource.data(); + transfer.segments[idx].data_size = resource.data().size(); + transfer.segments[idx].received = true; + transfer.received_count++; + + DEBUGF("SegmentAccumulator: Stored segment %d, %d/%d received", + segment_index, transfer.received_count, transfer.total_segments); + + // Fire per-segment callback if set + if (_segment_callback) { + _segment_callback(segment_index, total_segments, original_hash); + } + } else { + DEBUGF("SegmentAccumulator: Duplicate segment %d, ignoring", segment_index); + } + + // Check if all segments received + if (transfer.received_count == transfer.total_segments) { + std::string hash_short = original_hash.toHex().substr(0, 16); + INFOF("SegmentAccumulator: All %d segments received for %s, assembling...", + transfer.total_segments, hash_short.c_str()); + + // Assemble complete data + Bytes complete_data = assemble_segments(transfer); + + INFOF("SegmentAccumulator: Assembled %zu bytes from %d segments", + complete_data.size(), transfer.total_segments); + + // Fire accumulated callback + if (_accumulated_callback) { + _accumulated_callback(complete_data, original_hash); + } + + // Cleanup - clear the slot + slot->clear(); + } + + return true; // We handled this multi-segment resource +} + +Bytes SegmentAccumulator::assemble_segments(const PendingTransfer& transfer) { + // Calculate total size + size_t total_size = 0; + for (int i = 0; i < transfer.total_segments; i++) { + total_size += transfer.segments[i].data_size; + } + + // Concatenate in order + Bytes result; + result.reserve(total_size); + + for (int i = 0; i < transfer.total_segments; i++) { + const SegmentInfo& seg = transfer.segments[i]; + if (!seg.received) { + ERRORF("SegmentAccumulator: Missing segment %d during assembly!", i + 1); + continue; + } + result += seg.data; + } + + return result; +} + +void SegmentAccumulator::check_timeouts(double timeout_seconds) { + double now = OS::time(); + + for (size_t i = 0; i < MAX_PENDING_TRANSFERS; i++) { + if (!_pending_pool[i].in_use) { + continue; + } + + const PendingTransfer& transfer = _pending_pool[i].transfer; + double inactive_time = now - transfer.last_activity; + + if (inactive_time > timeout_seconds) { + std::string hash_short = transfer.original_hash.toHex().substr(0, 16); + WARNINGF("SegmentAccumulator: Transfer %s timed out (%.1fs inactive, %d/%d segments)", + hash_short.c_str(), + inactive_time, transfer.received_count, transfer.total_segments); + _pending_pool[i].clear(); + } + } +} + +void SegmentAccumulator::cleanup(const Bytes& original_hash) { + PendingTransferSlot* slot = find_slot(original_hash); + if (slot != nullptr) { + std::string hash_short = original_hash.toHex().substr(0, 16); + DEBUGF("SegmentAccumulator: Cleaning up transfer %s (%d/%d segments received)", + hash_short.c_str(), + slot->transfer.received_count, slot->transfer.total_segments); + slot->clear(); + } +} + +bool SegmentAccumulator::has_pending(const Bytes& original_hash) const { + return find_slot(original_hash) != nullptr; +} + +size_t SegmentAccumulator::pending_count() const { + size_t count = 0; + for (size_t i = 0; i < MAX_PENDING_TRANSFERS; i++) { + if (_pending_pool[i].in_use) { + count++; + } + } + return count; +} diff --git a/src-shim/SegmentAccumulator.h b/src-shim/SegmentAccumulator.h new file mode 100644 index 00000000..e298c01d --- /dev/null +++ b/src-shim/SegmentAccumulator.h @@ -0,0 +1,162 @@ +#pragma once + +#include "Bytes.h" +#include "Log.h" +#include "Utilities/OS.h" + +#include + +namespace RNS { + + // Forward declaration to avoid circular includes + class Resource; + + /** + * SegmentAccumulator collects multi-segment resources and fires a single callback + * when all segments have been received. + * + * Python RNS splits large resources (>MAX_EFFICIENT_SIZE, ~1MB) into multiple segments. + * Each segment is transferred as a separate Resource with its own hash/proof. + * Segments share the same original_hash and have segment_index 1..total_segments. + * + * This class: + * - Tracks incoming segments by original_hash + * - Stores segment data until all segments arrive + * - Fires the accumulated callback once with the complete concatenated data + * - Handles timeout cleanup for stale transfers + */ + class SegmentAccumulator { + + public: + // Fixed-size pool limits to eliminate heap fragmentation + static constexpr size_t MAX_PENDING_TRANSFERS = 8; + static constexpr size_t MAX_SEGMENTS_PER_TRANSFER = 64; + + // Callback fired when all segments are received + // Parameters: complete_data, original_hash + using AccumulatedCallback = std::function; + + // Callback for individual segment completion (optional, for progress tracking) + using SegmentCallback = std::function; + + public: + SegmentAccumulator() = default; + explicit SegmentAccumulator(AccumulatedCallback callback); + + /** + * Set the callback for completed multi-segment resources. + */ + void set_accumulated_callback(AccumulatedCallback callback); + + /** + * Set optional per-segment progress callback. + */ + void set_segment_callback(SegmentCallback callback); + + /** + * Called when a Resource segment completes. + * + * @param resource The completed resource (may be single or multi-segment) + * @return true if this was a multi-segment resource that was handled, + * false if it was a single-segment resource (caller should invoke normal callback) + */ + bool segment_completed(const Resource& resource); + + /** + * Check for timed-out transfers and clean them up. + * Should be called periodically (e.g., from watchdog). + * + * @param timeout_seconds Maximum time since last activity before cleanup + */ + void check_timeouts(double timeout_seconds = 600.0); + + /** + * Manually cleanup a specific transfer. + */ + void cleanup(const Bytes& original_hash); + + /** + * Check if a transfer is in progress for the given original_hash. + */ + bool has_pending(const Bytes& original_hash) const; + + /** + * Get the number of pending (incomplete) transfers. + */ + size_t pending_count() const; + + private: + struct SegmentInfo { + int segment_index = 0; + size_t data_size = 0; + Bytes data; + bool received = false; + + void clear() { + segment_index = 0; + data_size = 0; + data.clear(); + received = false; + } + }; + + struct PendingTransfer { + Bytes original_hash; + int total_segments = 0; + int received_count = 0; + SegmentInfo segments[MAX_SEGMENTS_PER_TRANSFER]; // Fixed array instead of std::vector + size_t segment_count = 0; + double started_at = 0.0; + double last_activity = 0.0; + + void clear() { + original_hash.clear(); + total_segments = 0; + received_count = 0; + for (size_t i = 0; i < segment_count; i++) { + segments[i].clear(); + } + segment_count = 0; + started_at = 0.0; + last_activity = 0.0; + } + }; + + struct PendingTransferSlot { + bool in_use = false; + Bytes transfer_id; // key (original_hash) + PendingTransfer transfer; + + void clear() { + in_use = false; + transfer_id.clear(); + transfer.clear(); + } + }; + + // Fixed-size pool instead of std::map + PendingTransferSlot _pending_pool[MAX_PENDING_TRANSFERS]; + + AccumulatedCallback _accumulated_callback = nullptr; + SegmentCallback _segment_callback = nullptr; + + /** + * Find a slot by transfer_id (original_hash). + * @return pointer to slot if found, nullptr otherwise + */ + PendingTransferSlot* find_slot(const Bytes& transfer_id); + const PendingTransferSlot* find_slot(const Bytes& transfer_id) const; + + /** + * Allocate a new slot for a transfer. + * @return pointer to slot if available, nullptr if pool is full + */ + PendingTransferSlot* allocate_slot(const Bytes& transfer_id); + + /** + * Concatenate all segments in order and return complete data. + */ + Bytes assemble_segments(const PendingTransfer& transfer); + }; + +} diff --git a/src-shim/Utilities/Print.cpp b/src-shim/Utilities/Print.cpp new file mode 100644 index 00000000..7c2566d6 --- /dev/null +++ b/src-shim/Utilities/Print.cpp @@ -0,0 +1,354 @@ +/* + Print.cpp - Base class that provides print() and println() + Copyright (c) 2008 David A. Mellis. All right reserved. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + Modified 23 November 2006 by David A. Mellis + Modified December 2014 by Ivan Grokhotkov + Modified May 2015 by Michael C. Miller - ESP31B progmem support + */ + +#ifndef ARDUINO + +#include +#include +#include +#include +#include + +#include "Print.h" +extern "C" { + #include "time.h" +} + +// Public Methods ////////////////////////////////////////////////////////////// + +/* default implementation: may be overridden */ +size_t Print::write(const uint8_t *buffer, size_t size) +{ + size_t n = 0; + while(size--) { + n += write(*buffer++); + } + return n; +} + +size_t Print::printf(const char *format, ...) +{ + char loc_buf[64]; + char * temp = loc_buf; + va_list arg; + va_list copy; + va_start(arg, format); + va_copy(copy, arg); + int len = vsnprintf(temp, sizeof(loc_buf), format, copy); + va_end(copy); + if(len < 0) { + va_end(arg); + return 0; + } + if(len >= (int)sizeof(loc_buf)){ // comparation of same sign type for the compiler + temp = (char*) malloc(len+1); + if(temp == NULL) { + va_end(arg); + return 0; + } + len = vsnprintf(temp, len+1, format, arg); + } + va_end(arg); + len = write((uint8_t*)temp, len); + if(temp != loc_buf){ + free(temp); + } + return len; +} + +size_t Print::print(const std::string &s) +{ + return write(s.c_str(), s.length()); +} + +size_t Print::print(const char str[]) +{ + return write(str); +} + +size_t Print::print(char c) +{ + return write(c); +} + +size_t Print::print(unsigned char b, int base) +{ + return print((unsigned long) b, base); +} + +size_t Print::print(int n, int base) +{ + return print((long) n, base); +} + +size_t Print::print(unsigned int n, int base) +{ + return print((unsigned long) n, base); +} + +size_t Print::print(long n, int base) +{ + int t = 0; + if (base == 10 && n < 0) { + t = print('-'); + n = -n; + } + return printNumber(static_cast(n), base) + t; +} + +size_t Print::print(unsigned long n, int base) +{ + if(base == 0) { + return write(n); + } else { + return printNumber(n, base); + } +} + +size_t Print::print(long long n, int base) +{ + int t = 0; + if (base == 10 && n < 0) { + t = print('-'); + n = -n; + } + return printNumber(static_cast(n), base) + t; +} + +size_t Print::print(unsigned long long n, int base) +{ + if (base == 0) { + return write(n); + } else { + return printNumber(n, base); + } +} + +size_t Print::print(double n, int digits) +{ + return printFloat(n, digits); +} + +size_t Print::print(struct tm * timeinfo, const char * format) +{ + const char * f = format; + if(!f){ + f = "%c"; + } + char buf[64]; + size_t written = strftime(buf, 64, f, timeinfo); + if(written == 0){ + return written; + } + return print(buf); +} + +size_t Print::println(void) +{ + return print("\r\n"); +} + +size_t Print::println(const std::string &s) +{ + size_t n = print(s); + n += println(); + return n; +} + +size_t Print::println(const char c[]) +{ + size_t n = print(c); + n += println(); + return n; +} + +size_t Print::println(char c) +{ + size_t n = print(c); + n += println(); + return n; +} + +size_t Print::println(unsigned char b, int base) +{ + size_t n = print(b, base); + n += println(); + return n; +} + +size_t Print::println(int num, int base) +{ + size_t n = print(num, base); + n += println(); + return n; +} + +size_t Print::println(unsigned int num, int base) +{ + size_t n = print(num, base); + n += println(); + return n; +} + +size_t Print::println(long num, int base) +{ + size_t n = print(num, base); + n += println(); + return n; +} + +size_t Print::println(unsigned long num, int base) +{ + size_t n = print(num, base); + n += println(); + return n; +} + +size_t Print::println(long long num, int base) +{ + size_t n = print(num, base); + n += println(); + return n; +} + +size_t Print::println(unsigned long long num, int base) +{ + size_t n = print(num, base); + n += println(); + return n; +} + +size_t Print::println(double num, int digits) +{ + size_t n = print(num, digits); + n += println(); + return n; +} + +size_t Print::println(struct tm * timeinfo, const char * format) +{ + size_t n = print(timeinfo, format); + n += println(); + return n; +} + +// Private Methods ///////////////////////////////////////////////////////////// + +size_t Print::printNumber(unsigned long n, uint8_t base) +{ + char buf[8 * sizeof(n) + 1]; // Assumes 8-bit chars plus zero byte. + char *str = &buf[sizeof(buf) - 1]; + + *str = '\0'; + + // prevent crash if called with base == 1 + if(base < 2) { + base = 10; + } + + do { + char c = n % base; + n /= base; + + *--str = c < 10 ? c + '0' : c + 'A' - 10; + } while (n); + + return write(str); +} + +size_t Print::printNumber(unsigned long long n, uint8_t base) +{ + char buf[8 * sizeof(n) + 1]; // Assumes 8-bit chars plus zero byte. + char* str = &buf[sizeof(buf) - 1]; + + *str = '\0'; + + // prevent crash if called with base == 1 + if (base < 2) { + base = 10; + } + + do { + auto m = n; + n /= base; + char c = m - base * n; + + *--str = c < 10 ? c + '0' : c + 'A' - 10; + } while (n); + + return write(str); +} + +size_t Print::printFloat(double number, uint8_t digits) +{ + size_t n = 0; + + if(isnan(number)) { + return print("nan"); + } + if(isinf(number)) { + return print("inf"); + } + if(number > 4294967040.0) { + return print("ovf"); // constant determined empirically + } + if(number < -4294967040.0) { + return print("ovf"); // constant determined empirically + } + + // Handle negative numbers + if(number < 0.0) { + n += print('-'); + number = -number; + } + + // Round correctly so that print(1.999, 2) prints as "2.00" + double rounding = 0.5; + for(uint8_t i = 0; i < digits; ++i) { + rounding /= 10.0; + } + + number += rounding; + + // Extract the integer part of the number and print it + unsigned long int_part = (unsigned long) number; + double remainder = number - (double) int_part; + n += print(int_part); + + // Print the decimal point, but only if there are digits beyond + if(digits > 0) { + n += print("."); + } + + // Extract digits from the remainder one at a time + while(digits-- > 0) { + remainder *= 10.0; + int toPrint = int(remainder); + n += print(toPrint); + remainder -= toPrint; + } + + return n; +} + +#endif diff --git a/src-shim/Utilities/Print.h b/src-shim/Utilities/Print.h new file mode 100644 index 00000000..89b026c3 --- /dev/null +++ b/src-shim/Utilities/Print.h @@ -0,0 +1,121 @@ +/* + Print.h - Base class that provides print() and println() + Copyright (c) 2008 David A. Mellis. All right reserved. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef ARDUINO + +#ifndef Print_h +#define Print_h + +#include +#include +#include + + +#include + +//#define DEC 10 +//#define HEX 16 +//#define OCT 8 +//#define BIN 2 +static constexpr uint8_t DEC {10}; +static constexpr uint8_t HEX {16}; +static constexpr uint8_t OCT {8}; +static constexpr uint8_t BIN {2}; + +class Print +{ +private: + int write_error; + size_t printNumber(unsigned long, uint8_t); + size_t printNumber(unsigned long long, uint8_t); + size_t printFloat(double, uint8_t); +protected: + void setWriteError(int err = 1) + { + write_error = err; + } +public: + Print() : + write_error(0) + { + } + virtual ~Print() {} + int getWriteError() + { + return write_error; + } + void clearWriteError() + { + setWriteError(0); + } + + virtual size_t write(uint8_t) = 0; + size_t write(const char *str) + { + if(str == NULL) { + return 0; + } + return write((const uint8_t *) str, strlen(str)); + } + virtual size_t write(const uint8_t *buffer, size_t size); + size_t write(const char *buffer, size_t size) + { + return write((const uint8_t *) buffer, size); + } + + size_t printf(const char * format, ...) __attribute__ ((format (printf, 2, 3))); + + // add availableForWrite to make compatible with Arduino Print.h + // default to zero, meaning "a single write may block" + // should be overriden by subclasses with buffering + virtual int availableForWrite() { return 0; } + size_t print(const std::string &); + size_t print(const char[]); + size_t print(char); + size_t print(unsigned char, int = DEC); + size_t print(int, int = DEC); + size_t print(unsigned int, int = DEC); + size_t print(long, int = DEC); + size_t print(unsigned long, int = DEC); + size_t print(long long, int = DEC); + size_t print(unsigned long long, int = DEC); + size_t print(double, int = 2); + size_t print(struct tm * timeinfo, const char * format = NULL); + + size_t println(const std::string &s); + size_t println(const char[]); + size_t println(char); + size_t println(unsigned char, int = DEC); + size_t println(int, int = DEC); + size_t println(unsigned int, int = DEC); + size_t println(long, int = DEC); + size_t println(unsigned long, int = DEC); + size_t println(long long, int = DEC); + size_t println(unsigned long long, int = DEC); + size_t println(double, int = 2); + size_t println(struct tm * timeinfo, const char * format = NULL); + size_t println(void); + + virtual void flush() { /* Empty implementation for backward compatibility */ } + +}; + +#endif + +#endif diff --git a/src-shim/Utilities/Stream.cpp b/src-shim/Utilities/Stream.cpp new file mode 100644 index 00000000..56fc2a60 --- /dev/null +++ b/src-shim/Utilities/Stream.cpp @@ -0,0 +1,339 @@ +/* + Stream.cpp - adds parsing methods to Stream class + Copyright (c) 2008 David A. Mellis. All right reserved. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + Created July 2011 + parsing functions based on TextFinder library by Michael Margolis + */ + +#ifndef ARDUINO + +#include "Stream.h" +#include "OS.h" + +#define PARSE_TIMEOUT 1000 // default number of milli-seconds to wait +#define NO_SKIP_CHAR 1 // a magic char not found in a valid ASCII numeric field + +// private method to read stream with timeout +int Stream::timedRead() +{ + int c; + _startMillis = RNS::Utilities::OS::ltime(); + do { + c = read(); + if(c >= 0) { + return c; + } + } while(RNS::Utilities::OS::ltime() - _startMillis < _timeout); + return -1; // -1 indicates timeout +} + +// private method to peek stream with timeout +int Stream::timedPeek() +{ + int c; + _startMillis = RNS::Utilities::OS::ltime(); + do { + c = peek(); + if(c >= 0) { + return c; + } + } while(RNS::Utilities::OS::ltime() - _startMillis < _timeout); + return -1; // -1 indicates timeout +} + +// returns peek of the next digit in the stream or -1 if timeout +// discards non-numeric characters +int Stream::peekNextDigit() +{ + int c; + while(1) { + c = timedPeek(); + if(c < 0) { + return c; // timeout + } + if(c == '-') { + return c; + } + if(c >= '0' && c <= '9') { + return c; + } + read(); // discard non-numeric + } +} + +// Public Methods +////////////////////////////////////////////////////////////// + +void Stream::setTimeout(unsigned long timeout) // sets the maximum number of milliseconds to wait +{ + _timeout = timeout; +} +unsigned long Stream::getTimeout(void) { + return _timeout; +} + +// find returns true if the target string is found +bool Stream::find(const char *target) +{ + return findUntil(target, strlen(target), NULL, 0); +} + +// reads data from the stream until the target string of given length is found +// returns true if target string is found, false if timed out +bool Stream::find(const char *target, size_t length) +{ + return findUntil(target, length, NULL, 0); +} + +// as find but search ends if the terminator string is found +bool Stream::findUntil(const char *target, const char *terminator) +{ + return findUntil(target, strlen(target), terminator, strlen(terminator)); +} + +// reads data from the stream until the target string of the given length is found +// search terminated if the terminator string is found +// returns true if target string is found, false if terminated or timed out +bool Stream::findUntil(const char *target, size_t targetLen, const char *terminator, size_t termLen) +{ + if (terminator == NULL) { + MultiTarget t[1] = {{target, targetLen, 0}}; + return findMulti(t, 1) == 0 ? true : false; + } else { + MultiTarget t[2] = {{target, targetLen, 0}, {terminator, termLen, 0}}; + return findMulti(t, 2) == 0 ? true : false; + } +} + +int Stream::findMulti( struct Stream::MultiTarget *targets, int tCount) { + // any zero length target string automatically matches and would make + // a mess of the rest of the algorithm. + for (struct MultiTarget *t = targets; t < targets+tCount; ++t) { + if (t->len <= 0) + return t - targets; + } + + while (1) { + int c = timedRead(); + if (c < 0) + return -1; + + for (struct MultiTarget *t = targets; t < targets+tCount; ++t) { + // the simple case is if we match, deal with that first. + if (c == t->str[t->index]) { + if (++t->index == t->len) + return t - targets; + else + continue; + } + + // if not we need to walk back and see if we could have matched further + // down the stream (ie '1112' doesn't match the first position in '11112' + // but it will match the second position so we can't just reset the current + // index to 0 when we find a mismatch. + if (t->index == 0) + continue; + + int origIndex = t->index; + do { + --t->index; + // first check if current char works against the new current index + if (c != t->str[t->index]) + continue; + + // if it's the only char then we're good, nothing more to check + if (t->index == 0) { + t->index++; + break; + } + + // otherwise we need to check the rest of the found string + int diff = origIndex - t->index; + size_t i; + for (i = 0; i < t->index; ++i) { + if (t->str[i] != t->str[i + diff]) + break; + } + + // if we successfully got through the previous loop then our current + // index is good. + if (i == t->index) { + t->index++; + break; + } + + // otherwise we just try the next index + } while (t->index); + } + } + // unreachable + return -1; +} + +// returns the first valid (long) integer value from the current position. +// initial characters that are not digits (or the minus sign) are skipped +// function is terminated by the first character that is not a digit. +long Stream::parseInt() +{ + return parseInt(NO_SKIP_CHAR); // terminate on first non-digit character (or timeout) +} + +// as above but a given skipChar is ignored +// this allows format characters (typically commas) in values to be ignored +long Stream::parseInt(char skipChar) +{ + bool isNegative = false; + long value = 0; + int c; + + c = peekNextDigit(); + // ignore non numeric leading characters + if(c < 0) { + return 0; // zero returned if timeout + } + + do { + if(c == skipChar) { + } // ignore this charactor + else if(c == '-') { + isNegative = true; + } else if(c >= '0' && c <= '9') { // is c a digit? + value = value * 10 + c - '0'; + } + read(); // consume the character we got with peek + c = timedPeek(); + } while((c >= '0' && c <= '9') || c == skipChar); + + if(isNegative) { + value = -value; + } + return value; +} + +// as parseInt but returns a floating point value +float Stream::parseFloat() +{ + return parseFloat(NO_SKIP_CHAR); +} + +// as above but the given skipChar is ignored +// this allows format characters (typically commas) in values to be ignored +float Stream::parseFloat(char skipChar) +{ + bool isNegative = false; + bool isFraction = false; + long value = 0; + int c; + float fraction = 1.0; + + c = peekNextDigit(); + // ignore non numeric leading characters + if(c < 0) { + return 0; // zero returned if timeout + } + + do { + if(c == skipChar) { + } // ignore + else if(c == '-') { + isNegative = true; + } else if(c == '.') { + isFraction = true; + } else if(c >= '0' && c <= '9') { // is c a digit? + value = value * 10 + c - '0'; + if(isFraction) { + fraction *= 0.1f; + } + } + read(); // consume the character we got with peek + c = timedPeek(); + } while((c >= '0' && c <= '9') || c == '.' || c == skipChar); + + if(isNegative) { + value = -value; + } + if(isFraction) { + return value * fraction; + } else { + return value; + } +} + +// read characters from stream into buffer +// terminates if length characters have been read, or timeout (see setTimeout) +// returns the number of characters placed in the buffer +// the buffer is NOT null terminated. +// +size_t Stream::readBytes(char *buffer, size_t length) +{ + size_t count = 0; + while(count < length) { + int c = timedRead(); + if(c < 0) { + break; + } + *buffer++ = (char) c; + count++; + } + return count; +} + +// as readBytes with terminator character +// terminates if length characters have been read, timeout, or if the terminator character detected +// returns the number of characters placed in the buffer (0 means no valid data found) + +size_t Stream::readBytesUntil(char terminator, char *buffer, size_t length) +{ + if(length < 1) { + return 0; + } + size_t index = 0; + while(index < length) { + int c = timedRead(); + if(c < 0 || c == terminator) { + break; + } + *buffer++ = (char) c; + index++; + } + return index; // return number of characters, not including null terminator +} + +std::string Stream::readString() +{ + std::string ret; + int c = timedRead(); + while(c >= 0) { + ret += (char) c; + c = timedRead(); + } + return ret; +} + +std::string Stream::readStringUntil(char terminator) +{ + std::string ret; + int c = timedRead(); + while(c >= 0 && c != terminator) { + ret += (char) c; + c = timedRead(); + } + return ret; +} + +#endif diff --git a/src-shim/Utilities/Stream.h b/src-shim/Utilities/Stream.h new file mode 100644 index 00000000..c3c7a701 --- /dev/null +++ b/src-shim/Utilities/Stream.h @@ -0,0 +1,143 @@ +/* + Stream.h - base class for character-based streams. + Copyright (c) 2010 David A. Mellis. All right reserved. + + This library is free software; you can redistribute it and/or + modify it under the terms of the GNU Lesser General Public + License as published by the Free Software Foundation; either + version 2.1 of the License, or (at your option) any later version. + + This library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public + License along with this library; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + parsing functions based on TextFinder library by Michael Margolis + */ + +#ifndef ARDUINO + +#ifndef Stream_h +#define Stream_h + +#include +#include "Print.h" + +// compatability macros for testing +/* + #define getInt() parseInt() + #define getInt(skipChar) parseInt(skipchar) + #define getFloat() parseFloat() + #define getFloat(skipChar) parseFloat(skipChar) + #define getString( pre_string, post_string, buffer, length) + readBytesBetween( pre_string, terminator, buffer, length) + */ + +class Stream: public Print +{ +protected: + unsigned long _timeout; // number of milliseconds to wait for the next char before aborting timed read + unsigned long _startMillis; // used for timeout measurement + int timedRead(); // private method to read stream with timeout + int timedPeek(); // private method to peek stream with timeout + int peekNextDigit(); // returns the next numeric digit in the stream or -1 if timeout + +public: + virtual int available() = 0; + virtual int read() = 0; + virtual int peek() = 0; + + Stream():_startMillis(0) + { + _timeout = 1000; + } + virtual ~Stream() {} + +// parsing methods + + void setTimeout(unsigned long timeout); // sets maximum milliseconds to wait for stream data, default is 1 second + unsigned long getTimeout(void); + + bool find(const char *target); // reads data from the stream until the target string is found + bool find(uint8_t *target) + { + return find((char *) target); + } + // returns true if target string is found, false if timed out (see setTimeout) + + bool find(const char *target, size_t length); // reads data from the stream until the target string of given length is found + bool find(const uint8_t *target, size_t length) + { + return find((char *) target, length); + } + // returns true if target string is found, false if timed out + + bool find(char target) + { + return find (&target, 1); + } + + bool findUntil(const char *target, const char *terminator); // as find but search ends if the terminator string is found + bool findUntil(const uint8_t *target, const char *terminator) + { + return findUntil((char *) target, terminator); + } + + bool findUntil(const char *target, size_t targetLen, const char *terminate, size_t termLen); // as above but search ends if the terminate string is found + bool findUntil(const uint8_t *target, size_t targetLen, const char *terminate, size_t termLen) + { + return findUntil((char *) target, targetLen, terminate, termLen); + } + + long parseInt(); // returns the first valid (long) integer value from the current position. + // initial characters that are not digits (or the minus sign) are skipped + // integer is terminated by the first character that is not a digit. + + float parseFloat(); // float version of parseInt + + virtual size_t readBytes(char *buffer, size_t length); // read chars from stream into buffer + virtual size_t readBytes(uint8_t *buffer, size_t length) + { + return readBytes((char *) buffer, length); + } + // terminates if length characters have been read or timeout (see setTimeout) + // returns the number of characters placed in the buffer (0 means no valid data found) + + size_t readBytesUntil(char terminator, char *buffer, size_t length); // as readBytes with terminator character + size_t readBytesUntil(char terminator, uint8_t *buffer, size_t length) + { + return readBytesUntil(terminator, (char *) buffer, length); + } + // terminates if length characters have been read, timeout, or if the terminator character detected + // returns the number of characters placed in the buffer (0 means no valid data found) + + // Arduino String functions to be added here + virtual std::string readString(); + std::string readStringUntil(char terminator); + +protected: + long parseInt(char skipChar); // as above but the given skipChar is ignored + // as above but the given skipChar is ignored + // this allows format characters (typically commas) in values to be ignored + + float parseFloat(char skipChar); // as above but the given skipChar is ignored + + struct MultiTarget { + const char *str; // string you're searching for + size_t len; // length of string you're searching for + size_t index; // index used by the search routine. + }; + + // This allows you to search for an arbitrary number of strings. + // Returns index of the target that is found first or -1 if timeout occurs. + int findMulti(struct MultiTarget *targets, int tCount); + +}; + +#endif + +#endif