From b7ab97d624d42f5eec4ee46c8cfca3b0154c2716 Mon Sep 17 00:00:00 2001 From: Torlando <281092095+torlando-agent[bot]@users.noreply.github.com> Date: Fri, 28 Aug 2026 19:26:10 +0000 Subject: [PATCH] test: add host regression for MapTileStoreSD mount-path overflow A 31-character pack ID makes mounted tile paths (/sd/pyxis-map/packs//tiles///.png) exceed the 68-byte mount buffer once either x or y is two digits, so beginRead returns INVALID_ARGUMENT and the UI shows 'Tile I/O error' at z4+ even though the file exists and is intact. Compiles the unmodified MapTilePack/MapTileStoreSD/SDAccess/codec/manifest sources against small Arduino/FreeRTOS host shims. The core section (no card needed) drives the real store with a model of the mount-prefix arithmetic and asserts the 31-char-ID boundary loads at z4/z5; an optional end-to-end section runs the real MapTilePack over a bound card root when /sd is writable. Fails on the current 64-byte PATH_CAPACITY: red by design. --- tests/native/sdhostshim/Arduino.h | 48 ++ tests/native/sdhostshim/FS.h | 8 + tests/native/sdhostshim/SD.h | 199 ++++++++ tests/native/sdhostshim/SPI.h | 15 + tests/native/sdhostshim/esp_heap_caps.h | 16 + tests/native/sdhostshim/freertos/FreeRTOS.h | 22 + tests/native/sdhostshim/freertos/semphr.h | 11 + tests/native/sdhostshim/microReticulum/Log.h | 13 + .../test_map_tile_store_path_capacity.cpp | 464 ++++++++++++++++++ .../test_map_tile_store_path_capacity.py | 109 ++++ 10 files changed, 905 insertions(+) create mode 100644 tests/native/sdhostshim/Arduino.h create mode 100644 tests/native/sdhostshim/FS.h create mode 100644 tests/native/sdhostshim/SD.h create mode 100644 tests/native/sdhostshim/SPI.h create mode 100644 tests/native/sdhostshim/esp_heap_caps.h create mode 100644 tests/native/sdhostshim/freertos/FreeRTOS.h create mode 100644 tests/native/sdhostshim/freertos/semphr.h create mode 100644 tests/native/sdhostshim/microReticulum/Log.h create mode 100644 tests/native/test_map_tile_store_path_capacity.cpp create mode 100644 tests/native/test_map_tile_store_path_capacity.py diff --git a/tests/native/sdhostshim/Arduino.h b/tests/native/sdhostshim/Arduino.h new file mode 100644 index 00000000..b462eb3f --- /dev/null +++ b/tests/native/sdhostshim/Arduino.h @@ -0,0 +1,48 @@ +// Host shim for — compiles the real T-Deck SD storage sources +// (MapTileStoreSD.cpp, SDAccess.cpp) unmodified on x86. +// +// MapTileStoreSD.cpp uses Arduino.h only for fs::File (pulled via SD.h); +// SDAccess.cpp uses pinMode/digitalWrite/Serial, modelled here as no-ops. +// The ARDUINO macro itself is supplied by the test driver (-DARDUINO=100). +#ifndef SDHOSTSHIM_ARDUINO_H +#define SDHOSTSHIM_ARDUINO_H + +#include +#include +#include +#include +#include + +#define HIGH 1 +#define LOW 0 +#define INPUT 0 +#define OUTPUT 1 + +namespace { + +struct HostSerial { + template + void println(const char* const format, Args&&...) { + std::fputs(format ? format : "\n", stderr); + std::fputc('\n', stderr); + } + void println() { std::fputc('\n', stderr); } + void printf(const char* format, ...) { + va_list args; + va_start(args, format); + std::vfprintf(stderr, format, args); + va_end(args); + } +}; + +HostSerial __attribute__((unused)) host_serial; + +} // namespace + +#define Serial host_serial + +inline void pinMode(std::uint8_t, std::uint8_t) {} +inline void digitalWrite(std::uint8_t, std::uint8_t) {} +inline void delay(std::uint32_t) {} + +#endif diff --git a/tests/native/sdhostshim/FS.h b/tests/native/sdhostshim/FS.h new file mode 100644 index 00000000..0847d5ac --- /dev/null +++ b/tests/native/sdhostshim/FS.h @@ -0,0 +1,8 @@ +// Host shim for — MapTileStoreSD.h includes it for the fs::File type, +// which the SD.h shim provides. +#ifndef SDHOSTSHIM_FS_H +#define SDHOSTSHIM_FS_H + +#include + +#endif diff --git a/tests/native/sdhostshim/SD.h b/tests/native/sdhostshim/SD.h new file mode 100644 index 00000000..12f0bdca --- /dev/null +++ b/tests/native/sdhostshim/SD.h @@ -0,0 +1,199 @@ +// Host shim for — models the SD card as a host-filesystem tree rooted +// at HostSD::root(), mounted under the same "/sd" prefix the firmware uses. +// +// HostFile mirrors the Arduino sdio File semantics the production code relies +// on: copy constructs duplicate the open handle (both stay usable and close +// independently), read() returns data in bounded SPI bursts, and close() +// releases the underlying descriptor. Directory handles expose the +// openNextFile()/isDirectory()/path() iteration API used by +// MapTileStoreSD::nextList. +#ifndef SDHOSTSHIM_SD_H +#define SDHOSTSHIM_SD_H + +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +#include "SPI.h" + +#define FILE_READ 1 +#define FILE_WRITE 2 +#define CARD_NONE 0 +#define CARD_MMC 1 +#define CARD_SD 2 +#define CARD_SDHC 3 + +class HostFile; + +class HostSD { +public: + static void set_root(const char* path) { root_ = path; present_ = true; } + static const char* root() { return root_.empty() ? "/tmp" : root_.c_str(); } + static bool ready() { return present_; } + + static bool begin(std::uint8_t, const HostSPI&, std::uint32_t, + const char* mount, std::uint8_t, bool) { + (void)mount; + return true; + } + static std::uint8_t cardType() { return present_ ? CARD_SDHC : CARD_NONE; } + static std::uint64_t cardSize() { return 119850ULL * 1024ULL * 1024ULL; } + + static std::string absolute(const char* path) { return std::string(root()) + path; } + static bool exists(const char* path) { + struct stat st; + return ::stat(absolute(path).c_str(), &st) == 0; + } + static bool mkdir(const char* path) { return ::mkdir(absolute(path).c_str(), 0755) == 0; } + static bool remove(const char* path) { return ::unlink(absolute(path).c_str()) == 0; } + static bool rename(const char* from, const char* to) { + return ::rename(absolute(from).c_str(), absolute(to).c_str()) == 0; + } + static HostFile open(const char* path, int); + +private: + static std::string root_; + static bool present_; +}; + +extern HostSD SD; + +class HostFile { +public: + HostFile() : fd_(-1), dir_(NULL) {} + HostFile(const HostFile& other) + : fd_(other.fd_ >= 0 ? ::dup(other.fd_) : -1), + dir_(other.isDir() ? ::opendir(other.absolute().c_str()) : NULL), + path_(other.path_) {} + HostFile& operator=(const HostFile& other) { + if (this == &other) return *this; + close(); + fd_ = other.fd_ >= 0 ? ::dup(other.fd_) : -1; + dir_ = other.isDir() ? ::opendir(other.absolute().c_str()) : NULL; + path_ = other.path_; + return *this; + } + ~HostFile() { close(); } + + explicit operator bool() const { return fd_ >= 0 || dir_ != NULL; } + bool isDirectory() const { return dir_ != NULL; } + + // Device-relative path of this entry ("" for a closed handle). + const char* path() const { return path_.c_str(); } + std::string name() const { + const std::size_t slash = path_.find_last_of('/'); + return path_.substr(slash + 1); + } + + // Bounded SPI read burst: the SD layer never returns more than this many + // bytes per transfer, so callers always loop. Returns the number of bytes + // actually read (0 at end-of-file or error), matching Arduino File::read. + static const int SPI_BURST_BYTES = 512; + + std::size_t read(void* buffer, std::size_t count) { + if (fd_ < 0) return 0U; + const std::size_t burst = count < static_cast(SPI_BURST_BYTES) + ? count : static_cast(SPI_BURST_BYTES); + if (burst == 0U) return 0U; + const ssize_t got = ::read(fd_, buffer, burst); + return (got < 0) ? 0U : static_cast(got); + } + + std::size_t size() const { + if (fd_ < 0) return 0U; + struct stat st; + if (::fstat(fd_, &st) != 0) return 0U; + return static_cast(st.st_size); + } + + bool available() const { + if (fd_ < 0) return false; + const off_t cur = ::lseek(fd_, 0, SEEK_CUR); + if (cur < 0) return false; + const off_t end = ::lseek(fd_, 0, SEEK_END); + (void)::lseek(fd_, cur, SEEK_SET); + return end > cur; + } + + void close() { + if (fd_ >= 0) { + ::close(fd_); + fd_ = -1; + } + if (dir_ != NULL) { + ::closedir(dir_); + dir_ = NULL; + } + } + + // Next entry inside a directory handle (dotfiles skipped), or a closed + // handle at end-of-directory (errno=0). + HostFile openNextFile() const { + if (dir_ == NULL) { + errno = ENOTDIR; + return HostFile(); + } + struct dirent* entry = NULL; + while ((entry = ::readdir(dir_)) != NULL) { + if (entry->d_name[0] == '.') continue; + const std::string abs_parent = absolute(); + const std::string rel_child = path_ + "/" + entry->d_name; + const std::string abs_child = abs_parent + "/" + entry->d_name; + struct stat st; + if (::stat(abs_child.c_str(), &st) != 0) continue; + HostFile child; + child.path_ = rel_child; + if (S_ISDIR(st.st_mode)) { + child.dir_ = ::opendir(abs_child.c_str()); + if (child.dir_ == NULL) child = HostFile(); + } else { + child.fd_ = ::open(abs_child.c_str(), O_RDONLY); + if (child.fd_ < 0) child = HostFile(); + } + if (child) { + errno = 0; + return child; + } + } + errno = 0; + return HostFile(); + } + +private: + friend class HostSD; + explicit HostFile(int fd, const std::string& path) : fd_(fd), dir_(NULL), path_(path) {} + HostFile(DIR* dir, const std::string& path) : fd_(-1), dir_(dir), path_(path) {} + + std::string absolute() const { return HostSD::absolute(path_.c_str()); } + bool isDir() const { return dir_ != NULL; } + int fd_; + DIR* dir_; + std::string path_; +}; + +using File = HostFile; + +namespace fs { +using File = HostFile; +} + +inline HostFile HostSD::open(const char* path, int) { + const std::string absolute = HostSD::absolute(path); + struct stat st; + if (::stat(absolute.c_str(), &st) != 0) return HostFile(); + if (S_ISDIR(st.st_mode)) { + DIR* dir = ::opendir(absolute.c_str()); + return (dir == NULL) ? HostFile() : HostFile(dir, path); + } + const int fd = ::open(absolute.c_str(), O_RDONLY); + return (fd < 0) ? HostFile() : HostFile(fd, path); +} + +#endif diff --git a/tests/native/sdhostshim/SPI.h b/tests/native/sdhostshim/SPI.h new file mode 100644 index 00000000..8fbaa989 --- /dev/null +++ b/tests/native/sdhostshim/SPI.h @@ -0,0 +1,15 @@ +// Host shim for — compiles the real T-Deck SD sources (SDAccess.cpp) +// unmodified on x86. Only the call sites used by SDAccess::init are modelled. +#ifndef SDHOSTSHIM_SPI_H +#define SDHOSTSHIM_SPI_H + +#include + +class HostSPI { +public: + void begin(std::uint8_t, std::uint8_t, std::uint8_t) {} +}; + +extern HostSPI SPI; + +#endif diff --git a/tests/native/sdhostshim/esp_heap_caps.h b/tests/native/sdhostshim/esp_heap_caps.h new file mode 100644 index 00000000..30a2adc9 --- /dev/null +++ b/tests/native/sdhostshim/esp_heap_caps.h @@ -0,0 +1,16 @@ +// Host shim for — MapTilePack.cpp selects PSRAM-backed +// buffers only on ARDUINO_ARCH_ESP32; on the host the static fallback +// members are used and these are never called. Provided for completeness. +#ifndef SDHOSTSHIM_ESP_HEAP_CAPS_H +#define SDHOSTSHIM_ESP_HEAP_CAPS_H + +#include +#include + +#define MALLOC_CAP_SPIRAM 1 +#define MALLOC_CAP_8BIT 2 + +inline void* heap_caps_malloc(std::size_t size, std::uint32_t) { return std::malloc(size); } +inline void heap_caps_free(void* pointer) { std::free(pointer); } + +#endif diff --git a/tests/native/sdhostshim/freertos/FreeRTOS.h b/tests/native/sdhostshim/freertos/FreeRTOS.h new file mode 100644 index 00000000..cbe19e60 --- /dev/null +++ b/tests/native/sdhostshim/freertos/FreeRTOS.h @@ -0,0 +1,22 @@ +// Host shim for — compiles the real T-Deck SD sources +// (SDAccess.cpp) unmodified on x86. Models the shared SPI bus mutex as a +// plain always-acquirable semaphore; the tests exercise storage logic, not +// bus arbitration. +#ifndef SDHOSTSHIM_FREERTOS_FREERTOS_H +#define SDHOSTSHIM_FREERTOS_FREERTOS_H + +#include +#include + +typedef void* SemaphoreHandle_t; +typedef long BaseType_t; +typedef unsigned long TickType_t; + +#define pdTRUE 1 +#define pdFALSE 0 +#define pdMS_TO_TICKS(ms) (ms) +#define portMAX_DELAY 0xFFFFFFFFU + +extern SemaphoreHandle_t hostsd_create_mutex(void); + +#endif diff --git a/tests/native/sdhostshim/freertos/semphr.h b/tests/native/sdhostshim/freertos/semphr.h new file mode 100644 index 00000000..c20aade4 --- /dev/null +++ b/tests/native/sdhostshim/freertos/semphr.h @@ -0,0 +1,11 @@ +#ifndef SDHOSTSHIM_FREERTOS_SEMPHR_H +#define SDHOSTSHIM_FREERTOS_SEMPHR_H + +#include + +inline SemaphoreHandle_t xSemaphoreCreateMutex() { return hostsd_create_mutex(); } +// Single-threaded host build: the shared SPI bus mutex is always free. +inline BaseType_t xSemaphoreTake(SemaphoreHandle_t, TickType_t) { return pdTRUE; } +inline BaseType_t xSemaphoreGive(SemaphoreHandle_t) { return pdTRUE; } + +#endif diff --git a/tests/native/sdhostshim/microReticulum/Log.h b/tests/native/sdhostshim/microReticulum/Log.h new file mode 100644 index 00000000..69aabc84 --- /dev/null +++ b/tests/native/sdhostshim/microReticulum/Log.h @@ -0,0 +1,13 @@ +// Host shim for — SDAccess.cpp pulls in +// `using namespace RNS;` for its logging helpers. +#ifndef SDHOSTSHIM_MICRORETICULUM_LOG_H +#define SDHOSTSHIM_MICRORETICULUM_LOG_H + +namespace RNS { +inline void log(const char*) {} +inline void logf(const char*, ...) {} +inline void logError(const char*) {} +inline void logWarning(const char*) {} +} + +#endif diff --git a/tests/native/test_map_tile_store_path_capacity.cpp b/tests/native/test_map_tile_store_path_capacity.cpp new file mode 100644 index 00000000..d0a5876d --- /dev/null +++ b/tests/native/test_map_tile_store_path_capacity.cpp @@ -0,0 +1,464 @@ +// Regression test: MapTileStoreSD mount-path capacity with a maximum-length +// pack ID. +// +// The SD store maps every firmware path onto the mounted card by prepending +// the "/sd" prefix (makeMountedPath in MapTileStoreSD.cpp). With the real +// 31-character world pack ID ("world-osm-bright-z0-z9-20260801"), any tile +// whose x or y coordinate is two digits makes the mounted path longer than +// the PATH_CAPACITY + 4 mount buffer, so beginRead rejects it before the file +// lookup with INVALID_ARGUMENT -- which MapTilePack::beginGet surfaces on the +// device as "Tile I/O error". Observed on the physical T-Deck at zoom 5 +// (single-digit quadrants rendered, two-digit quadrants failed) and on x86 +// against the real firmware build. +// +// The core regression section drives the UNMODIFIED production MapTileStoreSD +// (compiled against the sdhostshim headers): a path whose mounted form +// overflowed the old 68-byte buffer must be accepted once the buffer is +// sized correctly. The result is file-independent: overflow is rejected +// before stat(), and the control path (which fits) reaches the file lookup +// and reports MISS when no card content is present. The optional +// end-to-end section additionally initializes a real MapTilePack and reads +// tiles byte-for-byte, but only when "/sd" is a live card mount (e.g. the +// bind mount set up for the device harness). +// +// NOTE: the test is written to the FIXED behavior. On the pre-fix tree it +// fails in the core section (the two-digit z4/z5 reads return +// INVALID_ARGUMENT instead of OK) -- that failure is the TDD red step. + +#include +#include +#include +#include +#include +#include +#include + +#include "Hardware/TDeck/MapTilePack.h" +#include "Hardware/TDeck/MapTileStore.h" +#include "Hardware/TDeck/MapTileStoreSD.h" +#include "Hardware/TDeck/SDAccess.h" +#include "Hardware/TDeck/ActiveMapSetCodec.h" +#include "UI/LXMF/MapPackManifest.h" +#include + +using Hardware::TDeck::ActiveMapSetCodec; +using Hardware::TDeck::MapTilePack; +using Hardware::TDeck::MapTilePackResult; +using Hardware::TDeck::MapTileStorage; +using Hardware::TDeck::MapTileStore; +using Hardware::TDeck::MapTileStoreSD; +using Hardware::TDeck::SDAccess; +using Hardware::TDeck::TileKey; +using Hardware::TDeck::TileStoreResult; +using Pyxis::MapPackManifest; + +namespace { + +std::size_t tests_run = 0U; +void fail(const char* expression, int line) { + std::fprintf(stderr, "line %d: %s\n", line, expression); + std::exit(1); +} +#define CHECK(expression) do { if (!(expression)) fail(#expression, __LINE__); } while (false) +void beginTest() { ++tests_run; } + +const char* const kPackId = "world-osm-bright-z0-z9-20260801"; // 31 chars (max) +const char* const kMapSetId = "osm-bright"; +const char* const kAttribution = "Map data (c) OpenStreetMap contributors"; + +// The production mount buffer (MapTileStoreSD.cpp): PATH_CAPACITY + 4 bytes +// for the "/sd" prefix. The model store below pins the pre-fix width so the +// overflow boundary stays exercised no matter how PATH_CAPACITY changes. +const std::size_t kModelMountCapacity = 68U; + +std::uint32_t crc32_ieee(const std::uint8_t* data, std::size_t length) { + std::uint32_t crc = 0xffffffffU; + for (std::size_t i = 0U; i < length; ++i) { + crc ^= data[i]; + for (std::uint8_t bit = 0U; bit < 8U; ++bit) { + crc = (crc >> 1U) ^ ((crc & 1U) != 0U ? 0xedb88320U : 0U); + } + } + return ~crc; +} + +// Storage model of the pre-fix MapTileStoreSD mount-prefix arithmetic: +// paths whose "/sd" form overflowed the 68-byte mount buffer are rejected +// with INVALID_ARGUMENT before stat(), exactly as the real store rejected +// them on the device. +class ModelSDStorage : public MapTileStorage { +public: + struct Entry { std::string path; std::vector bytes; }; + + ModelSDStorage() : open_index(-1), position(0U), open_length(0U) {} + + bool isAvailable() const override { return true; } + + TileStoreResult beginRead(const char* name, std::uint32_t& size) override { + if (!fits(name)) return TileStoreResult::INVALID_ARGUMENT; + const std::string n(name); + for (std::size_t i = 0U; i < files.size(); ++i) { + if (files[i].path == n) { + open_index = static_cast(i); + position = 0U; + open_length = files[i].bytes.size(); + size = static_cast(open_length); + return TileStoreResult::OK; + } + } + return TileStoreResult::MISS; + } + TileStoreResult readChunk(std::uint8_t* out, std::size_t capacity, std::size_t& count) override { + if (open_index < 0) return TileStoreResult::IO_ERROR; + const std::size_t remaining = open_length - position; + count = capacity < remaining ? capacity : remaining; + if (count != 0U) { + std::memcpy(out, &files[static_cast(open_index)].bytes[position], count); + } + position += count; + return TileStoreResult::OK; + } + void endRead() override { open_index = -1; } + TileStoreResult beginWrite(const char*) override { return TileStoreResult::IO_ERROR; } + TileStoreResult writeChunk(const std::uint8_t*, std::size_t, std::size_t&) override { + return TileStoreResult::IO_ERROR; + } + TileStoreResult commitWrite() override { return TileStoreResult::IO_ERROR; } + void abortWrite() override {} + TileStoreResult remove(const char*) override { return TileStoreResult::IO_ERROR; } + TileStoreResult rename(const char*, const char*) override { return TileStoreResult::IO_ERROR; } + TileStoreResult stat(const char* name, std::uint32_t& size) override { + if (!fits(name)) return TileStoreResult::INVALID_ARGUMENT; + const std::string n(name); + for (std::size_t i = 0U; i < files.size(); ++i) { + if (files[i].path == n) { + size = static_cast(files[i].bytes.size()); + return TileStoreResult::OK; + } + } + return TileStoreResult::MISS; + } + TileStoreResult beginList() override { return TileStoreResult::OK; } + TileStoreResult nextList(char*, std::size_t, bool& done) override { + done = true; + return TileStoreResult::OK; + } + void endList() override {} + + void add(const char* name, const std::uint8_t* bytes, std::size_t length) { + Entry e; + e.path = name; + e.bytes.assign(bytes, bytes + length); + files.push_back(e); + } + // Mirrors makeMountedPath: snprintf(buffer, PATH_CAPACITY+4, "/sd%s", name). + bool fits(const char* name) const { + return (std::strlen(name) + 3U) < kModelMountCapacity; + } + +private: + std::vector files; + int open_index; + std::size_t position; + std::size_t open_length; +}; + +std::string tilePathFor(const char* pack_id, std::uint8_t zoom, std::uint32_t x, std::uint32_t y) { + // Larger than MapTileStore::PATH_CAPACITY on purpose: with the 31-character + // pack ID, two-digit z5 tile paths are 69 bytes -- exactly the overflow + // class this test pins. + char buffer[128] = {}; + CHECK(MapTilePack::tilePath(pack_id, TileKey{zoom, x, y}, buffer, sizeof(buffer)) + == MapTilePackResult::OK); + return buffer; +} + +std::vector build_pmas_v3(const char* map_set_id, const char* attribution, + const char* pack_id) { + // PMAS v3 layout (must match ActiveMapSetCodec::decode byte-for-byte): + // 0..3 "PMAS" + // 4 format_version (3 = indexless) + // 5 reserved (0) + // 6..7 u16 total record length (little-endian) + // 8..11 u32 generation (non-zero) + // 12.. u8 len + map_set_id + // u8 len + attribution + // u8 pack_count + // [u8 len + pack_id] * pack_count (no spans in v3) + // tail u32 CRC-32 (IEEE) over everything before it + std::vector payload; + const auto put_str = [&payload](const char* text) { + const std::size_t n = std::strlen(text); + payload.push_back(static_cast(n)); + payload.insert(payload.end(), text, text + n); + }; + const auto put_u32 = [&payload](std::uint32_t v) { + payload.push_back(static_cast(v)); + payload.push_back(static_cast(v >> 8U)); + payload.push_back(static_cast(v >> 16U)); + payload.push_back(static_cast(v >> 24U)); + }; + put_u32(1U); // generation + put_str(map_set_id); + put_str(attribution); + payload.push_back(1U); // pack_count + put_str(pack_id); + + const std::size_t total = 12U + payload.size(); + CHECK(total <= ActiveMapSetCodec::MAX_SERIALIZED_SIZE); + + std::vector out; + out.reserve(total); + out.insert(out.end(), {'P', 'M', 'A', 'S', 3U, 0U}); + out.push_back(static_cast(total)); + out.push_back(static_cast(total >> 8U)); + out.insert(out.end(), payload.begin(), payload.end()); + const std::uint32_t crc = crc32_ieee(out.data(), out.size()); + out.push_back(static_cast(crc)); + out.push_back(static_cast(crc >> 8U)); + out.push_back(static_cast(crc >> 16U)); + out.push_back(static_cast(crc >> 24U)); + CHECK(out.size() == total); + return out; +} + +std::vector build_manifest_v3(const char* pack_id) { + MapPackManifest m = {}; + std::strcpy(m.pack_id, pack_id); + std::strcpy(m.name, "World OSM Bright z0-z9"); + std::strcpy(m.attribution, kAttribution); + std::strcpy(m.source, "Oxed's Map Tile Downloader (OSM Bright)"); + std::strcpy(m.license, "OSM ODbL; style CC-BY-4.0/BSD-3-Clause"); + m.min_zoom = 0U; + m.max_zoom = 9U; + m.tile_count = 83567U; + m.format_version = MapPackManifest::INDEXLESS_FORMAT_VERSION; + std::uint8_t out[MapPackManifest::MAX_SERIALIZED_SIZE] = {}; + std::size_t written = 0U; + CHECK(MapPackManifest::serializeIndexless(m, out, sizeof(out), written) + == Pyxis::ManifestResult::OK); + CHECK(written <= sizeof(out)); + return std::vector(out, out + written); +} + +void read_tile_via(MapTilePack& pack, const TileKey& key, std::vector& out) { + std::uint32_t size = 0U; + CHECK(pack.beginGet(key, size) == MapTilePackResult::OK); + out.clear(); + while (out.size() < static_cast(size)) { + std::uint8_t chunk[256]; + std::size_t count = 0U; + const MapTilePackResult r = pack.readGetChunk(chunk, sizeof(chunk), count); + CHECK(r == MapTilePackResult::OK); + CHECK(count > 0U); + out.insert(out.end(), chunk, chunk + count); + } + CHECK(out.size() == static_cast(size)); + pack.endGet(); +} + +std::string shell_dir(const std::string& dir) { + // The dir is generated by this test from a compiler-provided temp path. + std::string quoted; + for (std::size_t i = 0U; i < dir.size(); ++i) { + const char c = dir[i]; + if (c == '\'') quoted += "'\\''"; + else quoted += c; + } + return "'" + quoted + "'"; +} + +void write_file(const std::string& path, const std::vector& bytes) { + const std::size_t slash = path.find_last_of('/'); + const std::string dir = path.substr(0U, slash); + if (!dir.empty() && dir != "/") { + (void)system(("mkdir -p " + shell_dir(dir)).c_str()); + } + FILE* fp = std::fopen(path.c_str(), "wb"); + CHECK(fp != NULL); + if (!bytes.empty()) { + CHECK(std::fwrite(bytes.data(), 1U, bytes.size(), fp) == bytes.size()); + } + std::fclose(fp); +} + +// True when "/sd" is a writable directory the production store can use for +// the end-to-end card-content section (a bind mount set up by the test +// harness, or a user-owned dir). A root-owned empty dir (stat-able but not +// writable) is NOT ready: the fixture write would fail. +bool sd_mount_ready() { + struct stat st; + if (::stat("/sd", &st) != 0 || !S_ISDIR(st.st_mode)) return false; + return ::access("/sd", W_OK) == 0; +} + +} // namespace + +// Shim globals: SDAccess.cpp references the Arduino globals `SD` and `SPI`; +// the FreeRTOS shim needs a mutex factory. +HostSD SD; +HostSPI SPI; +std::string HostSD::root_ = "/tmp"; +bool HostSD::present_ = false; +SemaphoreHandle_t hostsd_create_mutex() { return (void*)1; } + +int main(int argc, char** argv) { + CHECK(argc == 2); + const std::string root = argv[1]; + HostSD::set_root(root.c_str()); + + CHECK(std::strlen(kPackId) == 31U); + CHECK(std::strlen(kPackId) == MapPackManifest::PACK_ID_CAPACITY - 1U); + + // -- Wire records decode through the production codecs ------------------- + std::vector pmas = build_pmas_v3(kMapSetId, kAttribution, kPackId); + beginTest(); + { + Hardware::TDeck::ActiveMapSetView view = {}; + CHECK(ActiveMapSetCodec::decode(pmas.data(), pmas.size(), view)); + CHECK(view.format_version == ActiveMapSetCodec::INDEXLESS_FORMAT_VERSION); + CHECK(view.generation == 1U); + CHECK(view.pack_count == 1U); + CHECK(std::strcmp(view.packs[0].pack_id, kPackId) == 0); + } + std::vector manifest = build_manifest_v3(kPackId); + beginTest(); + { + MapPackManifest parsed = {}; + CHECK(MapPackManifest::parse(manifest.data(), manifest.size(), parsed) + == Pyxis::ManifestResult::OK); + CHECK(std::strcmp(parsed.pack_id, kPackId) == 0); + } + + // -- The boundary itself, derived from the real path builders ------------ + beginTest(); + { + // Single-digit quadrants fit the pre-fix 68-byte mount buffer... + CHECK(tilePathFor(kPackId, 4U, 9U, 9U).size() + 3U < kModelMountCapacity); + // ...and the two-digit quadrants that failed on the device do not. + CHECK(tilePathFor(kPackId, 4U, 15U, 15U).size() + 3U >= kModelMountCapacity); + CHECK(tilePathFor(kPackId, 5U, 10U, 0U).size() + 3U >= kModelMountCapacity); + } + + // The fix: the production mount buffer must hold the worst case. With a + // 31-character pack ID the longest z0-z22 mounted tile path is 82 bytes, + // so PATH_CAPACITY must be at least 80 (giving an 84-byte buffer). Pre-fix + // PATH_CAPACITY is 64, so this section is red until the fix lands. + beginTest(); + { + CHECK(MapTileStore::PATH_CAPACITY >= 80U); + CHECK(MapTileStore::PATH_CAPACITY + 4U >= 84U); + } + + // -- Model store: the pre-fix arithmetic, pinned -------------------------- + beginTest(); + { + ModelSDStorage store; + const std::vector tile(64U, 0x5AU); + store.add(tilePathFor(kPackId, 4U, 9U, 9U).c_str(), tile.data(), tile.size()); + std::uint32_t size = 0U; + CHECK(store.beginRead(tilePathFor(kPackId, 4U, 9U, 9U).c_str(), size) + == TileStoreResult::OK); + CHECK(size == tile.size()); + store.endRead(); + // Two-digit x or y: rejected before the file lookup, pre-fix. + CHECK(store.beginRead(tilePathFor(kPackId, 4U, 15U, 15U).c_str(), size) + == TileStoreResult::INVALID_ARGUMENT); + CHECK(store.beginRead(tilePathFor(kPackId, 5U, 10U, 0U).c_str(), size) + == TileStoreResult::INVALID_ARGUMENT); + } + + // -- CORE REGRESSION: the real, unmodified MapTileStoreSD ------------------ + // Production makeMountedPath rejects an overflowing mounted path with + // INVALID_ARGUMENT before stat(); a path that fits reaches the file + // lookup. Neither depends on card content, so this section runs + // everywhere. Pre-fix, the two-digit reads below return + // INVALID_ARGUMENT instead of OK and this test fails -- the TDD red. + beginTest(); + { + CHECK(SDAccess::init(xSemaphoreCreateMutex())); + MapTileStoreSD store; + CHECK(store.isAvailable()); + std::uint32_t size = 0U; + // Fits the old buffer: passes the overflow check, reaches the lookup, + // and reports MISS because no file exists on the (empty) mount. + CHECK(store.beginRead(tilePathFor(kPackId, 4U, 9U, 9U).c_str(), size) + == TileStoreResult::MISS); + // The device repro class. With the fixed buffer width these pass the + // overflow check and reach the lookup as well. + CHECK(store.beginRead(tilePathFor(kPackId, 4U, 15U, 15U).c_str(), size) + == TileStoreResult::MISS); + CHECK(store.beginRead(tilePathFor(kPackId, 5U, 10U, 0U).c_str(), size) + == TileStoreResult::MISS); + CHECK(store.beginRead(tilePathFor(kPackId, 5U, 0U, 10U).c_str(), size) + == TileStoreResult::MISS); + CHECK(store.beginRead(tilePathFor(kPackId, 5U, 31U, 31U).c_str(), size) + == TileStoreResult::MISS); + } + + // -- Optional end-to-end: real MapTilePack over card content --------------- + // Only meaningful when "/sd" holds a pack (device-harness bind mount). + if (sd_mount_ready()) { + // The production store stats the literal "/sd" prefix and opens via + // SD.open(device_path); point the shim at the same mount point. + HostSD::set_root("/sd"); + const auto make_tile = [](std::uint8_t seed) { + std::vector v(1024U); + for (std::size_t i = 0U; i < v.size(); ++i) { + v[i] = static_cast((i & 0xffU) ^ seed); + } + return v; + }; + const std::string pmap = "/sd/pyxis-map"; + { + std::string cmd = "mkdir -p " + shell_dir(pmap + "/map-sets"); + CHECK(system(cmd.c_str()) == 0); + } + write_file(pmap + "/active-pack.0", pmas); + write_file(pmap + "/map-sets/" + std::string(kMapSetId) + ".pmas", pmas); + write_file(pmap + "/packs/" + kPackId + "/manifest.pmp", manifest); + const auto write_tile = [&](std::uint8_t z, std::uint32_t x, std::uint32_t y, + std::uint8_t seed) { + // The production store reads the literal "/sd" mount point. + write_file(tilePathFor(kPackId, z, x, y), make_tile(seed)); + }; + write_tile(3U, 7U, 7U, 0xA1U); + write_tile(4U, 9U, 9U, 0xA4U); + write_tile(4U, 15U, 15U, 0xA5U); + write_tile(5U, 10U, 0U, 0xA6U); + write_tile(5U, 0U, 10U, 0xA7U); + write_tile(5U, 31U, 31U, 0xA8U); + + MapTileStoreSD sd_store; + MapTilePack pack(sd_store); + beginTest(); + CHECK(pack.initialize() == MapTilePackResult::OK); + CHECK(pack.hasSelection()); + CHECK(pack.selectionGeneration() == 1U); + + beginTest(); + { + std::vector data; + read_tile_via(pack, TileKey{3U, 7U, 7U}, data); + CHECK(data[0] == static_cast(0x00U ^ 0xA1U)); + } + beginTest(); + { + std::vector data; + read_tile_via(pack, TileKey{4U, 15U, 15U}, data); + CHECK(data[0] == static_cast(0x00U ^ 0xA5U)); + } + beginTest(); + { + std::vector data; + read_tile_via(pack, TileKey{5U, 31U, 31U}, data); + CHECK(data[0] == static_cast(0x00U ^ 0xA8U)); + } + } else { + std::fprintf(stderr, "note: /sd not present; end-to-end card section skipped\n"); + } + + std::printf("map tile store path capacity: %lu tests passed\n", + static_cast(tests_run)); + return 0; +} diff --git a/tests/native/test_map_tile_store_path_capacity.py b/tests/native/test_map_tile_store_path_capacity.py new file mode 100644 index 00000000..be30a9a1 --- /dev/null +++ b/tests/native/test_map_tile_store_path_capacity.py @@ -0,0 +1,109 @@ +from __future__ import annotations + +from pathlib import Path +import os +import shutil +import subprocess + +import pytest + +from native_test import find_cxx + +ROOT = Path(__file__).resolve().parents[2] +TEST_SOURCE = ROOT / "tests/native/test_map_tile_store_path_capacity.cpp" +PACK_SOURCE = ROOT / "lib/tdeck_ui/Hardware/TDeck/MapTilePack.cpp" +STORE_SOURCE = ROOT / "lib/tdeck_ui/Hardware/TDeck/MapTileStoreSD.cpp" +SDACCESS_SOURCE = ROOT / "lib/tdeck_ui/Hardware/TDeck/SDAccess.cpp" +CODEC_SOURCE = ROOT / "lib/tdeck_ui/Hardware/TDeck/ActiveMapSetCodec.cpp" +MANIFEST_SOURCE = ROOT / "lib/tdeck_ui/UI/LXMF/MapPackManifest.cpp" +SHIM_DIR = ROOT / "tests/native/sdhostshim" + + +def sudo_ok() -> bool: + return shutil.which("sudo") is not None and subprocess.run( + ["sudo", "-n", "true"], capture_output=True, timeout=10 + ).returncode == 0 + + +def sd_writable() -> bool: + return os.access("/sd", os.W_OK) + + +def run_test(binary: Path, card_root: Path, env: dict, timeout: int) -> subprocess.CompletedProcess[str]: + """Run the test binary, arranging a /sd bind mount for the end-to-end + section when possible. + + The production store reads the literal "/sd" mount point, so the + end-to-end card-content section only runs when "/sd" is a writable + directory. Without a usable password-less sudo we still run the binary + normally: the core regression section (mount-independent) executes, and + the binary prints a skip note for the card section. + """ + command = [str(binary), str(card_root)] + mounted = False + run_as_root = False + if not sd_writable() and sudo_ok(): + probe = subprocess.run(["mountpoint", "-q", "/sd"], capture_output=True, timeout=10) + if probe.returncode == 0: + # A live mount we did not create: leave it alone, run normally. + pass + else: + subprocess.run( + ["sudo", "-n", "mkdir", "-p", "/sd"], capture_output=True, timeout=10) + mnt = subprocess.run( + ["sudo", "-n", "mount", "--bind", str(card_root), "/sd"], + capture_output=True, text=True, timeout=30, + ) + if mnt.returncode == 0: + mounted = True + run_as_root = True # bind mount content is root-owned + else: + raise pytest.skip(f"could not bind-mount card root at /sd: {mnt.stderr.strip()}") + if run_as_root: + command = ["sudo", "-n"] + command + try: + return subprocess.run(command, capture_output=True, text=True, + timeout=timeout, env=env) + finally: + if mounted: + subprocess.run(["sudo", "-n", "umount", "/sd"], capture_output=True, timeout=30) + + +@pytest.mark.parametrize("sanitize", [False, True], ids=["strict-cxx11", "asan-ubsan"]) +def test_map_tile_store_path_capacity(tmp_path: Path, sanitize: bool) -> None: + # Note: -Wconversion/-Wsign-conversion are deliberately omitted here (and + # only here): SDAccess.cpp is ARDUINO-gated production code that only ever + # compiles under PlatformIO's ESP32 flag set, where the uint8_t/int + # Arduino API conversions are routine. The other flags above are kept so + # the unmodified store/pack sources stay strictly checked. + binary = tmp_path / "test_map_tile_store_path_capacity" + card_root = tmp_path / "card" + card_root.mkdir() + command = [ + find_cxx(), "-std=c++11", "-Wall", "-Wextra", "-Werror", "-pedantic", + "-DARDUINO=100", + f"-I{SHIM_DIR}", f"-I{ROOT / 'lib/tdeck_ui'}", + str(TEST_SOURCE), str(PACK_SOURCE), str(STORE_SOURCE), + str(SDACCESS_SOURCE), str(CODEC_SOURCE), str(MANIFEST_SOURCE), + "-o", str(binary), + ] + if sanitize: + command[1:1] = ["-fsanitize=address,undefined", "-fno-omit-frame-pointer"] + compiled = subprocess.run(command, capture_output=True, text=True, timeout=120, + cwd=ROOT) + assert compiled.returncode == 0, compiled.stdout + compiled.stderr + env = os.environ.copy() + if sanitize: + env["ASAN_OPTIONS"] = "detect_leaks=1:halt_on_error=1" + env["UBSAN_OPTIONS"] = "halt_on_error=1:print_stacktrace=1" + ran = run_test(binary, card_root, env, timeout=120) + assert ran.returncode == 0, ran.stdout + ran.stderr + # The end-to-end section (4 tests) is conditional on a writable /sd mount; + # the core regression section (6 tests) always runs. + for expected in ( + "map tile store path capacity: 6 tests passed\n", + "map tile store path capacity: 10 tests passed\n", + ): + if ran.stdout == expected: + return + raise AssertionError(f"unexpected output: {ran.stdout!r}")