diff --git a/lib/tdeck_ui/Hardware/TDeck/ActiveMapSetCodec.cpp b/lib/tdeck_ui/Hardware/TDeck/ActiveMapSetCodec.cpp index f9cfcecd..1dc075f7 100644 --- a/lib/tdeck_ui/Hardware/TDeck/ActiveMapSetCodec.cpp +++ b/lib/tdeck_ui/Hardware/TDeck/ActiveMapSetCodec.cpp @@ -66,12 +66,15 @@ bool ActiveMapSetCodec::decode(const std::uint8_t* input, std::size_t length, ActiveMapSetView& output) { static const std::uint8_t magic[4] = {'P', 'M', 'A', 'S'}; if (input == NULL || length < 16U || length > MAX_SERIALIZED_SIZE || - std::memcmp(input, magic, sizeof(magic)) != 0 || input[4] != 2U || input[5] != 0U || + std::memcmp(input, magic, sizeof(magic)) != 0 || + (input[4] != SPAN_FORMAT_VERSION && input[4] != INDEXLESS_FORMAT_VERSION) || + input[5] != 0U || readU16(input + 6U) != length || readU32(input + length - 4U) != crc32(input, length - 4U)) { return false; } ActiveMapSetView candidate = {}; + candidate.format_version = input[4]; candidate.generation = readU32(input + 8U); if (candidate.generation == 0U) return false; const std::size_t end = length - 4U; @@ -90,6 +93,11 @@ bool ActiveMapSetCodec::decode(const std::uint8_t* input, std::size_t length, for (std::uint8_t previous = 0U; previous < pack_index; ++previous) { if (std::strcmp(pack.pack_id, candidate.packs[previous].pack_id) == 0) return false; } + if (candidate.format_version == INDEXLESS_FORMAT_VERSION) { + pack.span_count = 0U; + pack.span_bytes = NULL; + continue; + } if (position > end || end - position < 2U) return false; pack.span_count = readU16(input + position); position += 2U; if (pack.span_count == 0U || pack.span_count > MAX_ROW_SPANS || diff --git a/lib/tdeck_ui/Hardware/TDeck/ActiveMapSetCodec.h b/lib/tdeck_ui/Hardware/TDeck/ActiveMapSetCodec.h index 7aff7137..2e928bcf 100644 --- a/lib/tdeck_ui/Hardware/TDeck/ActiveMapSetCodec.h +++ b/lib/tdeck_ui/Hardware/TDeck/ActiveMapSetCodec.h @@ -19,6 +19,7 @@ struct ActiveMapSetPackView { }; struct ActiveMapSetView { + std::uint8_t format_version; std::uint32_t generation; char map_set_id[Pyxis::MapPackManifest::PACK_ID_CAPACITY]; char attribution[Pyxis::MapPackManifest::ATTRIBUTION_CAPACITY]; @@ -27,9 +28,11 @@ struct ActiveMapSetView { ActiveMapSetPackView packs[8]; }; -/** Strict, allocation-free codec for canonical PMAS version-2 records. */ +/** Strict, allocation-free codec for span-indexed v2 and indexless v3 PMAS records. */ class ActiveMapSetCodec { public: + static const std::uint8_t SPAN_FORMAT_VERSION = 2U; + static const std::uint8_t INDEXLESS_FORMAT_VERSION = 3U; static const std::size_t MAX_SERIALIZED_SIZE = 7105U; static const std::size_t MAX_PACKS = 8U; static const std::size_t MAX_ROW_SPANS = 512U; diff --git a/lib/tdeck_ui/Hardware/TDeck/MapTilePack.cpp b/lib/tdeck_ui/Hardware/TDeck/MapTilePack.cpp index 781a8bd5..175d26d8 100644 --- a/lib/tdeck_ui/Hardware/TDeck/MapTilePack.cpp +++ b/lib/tdeck_ui/Hardware/TDeck/MapTilePack.cpp @@ -95,7 +95,8 @@ const char MapTilePack::ACTIVE_PACK_SLOT_1_PATH[] = "/pyxis-map/active-pack.1"; MapTilePack::MapTilePack(MapTileStorage& storage) : storage_(storage), manifest_(), active_packs_(), active_pack_count_(0U), - map_set_active_(false), selection_generation_(0U), + map_set_active_(false), map_set_indexless_(false), resolution_cache_(), + selection_generation_(0U), status_(MapTilePackStatus::UNINITIALIZED), stream_open_(false), stream_remaining_(0U), selection_buffers_(), active_selection_buffer_(0U) { @@ -104,6 +105,7 @@ MapTilePack::MapTilePack(MapTileStorage& storage) 2U * ACTIVE_SELECTION_SIZE + MANIFEST_BUFFER_CAPACITY, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT)); #endif + clearResolutionCache(); } MapTilePack::~MapTilePack() { @@ -232,15 +234,23 @@ MapTilePackResult MapTilePack::validateMapSet(MapTileStorage& storage, : MapTilePackResult::IO_ERROR; } Pyxis::MapPackManifest manifest = {}; - if (Pyxis::MapPackManifest::parse(scratch, length, manifest) != - Pyxis::ManifestResult::OK || - manifest.format_version != Pyxis::MapPackManifest::FORMAT_VERSION || + const bool indexless = + view.format_version == ActiveMapSetCodec::INDEXLESS_FORMAT_VERSION; + const Pyxis::ManifestResult parsed = + Pyxis::MapPackManifest::parse(scratch, length, manifest); + const bool compatible_manifest_version = indexless + ? (manifest.format_version == Pyxis::MapPackManifest::FORMAT_VERSION || + manifest.format_version == Pyxis::MapPackManifest::INDEXLESS_FORMAT_VERSION) + : manifest.format_version == Pyxis::MapPackManifest::FORMAT_VERSION; + if (parsed != Pyxis::ManifestResult::OK || + !compatible_manifest_version || std::strcmp(manifest.pack_id, view.packs[index].pack_id) != 0 || std::strcmp(manifest.attribution, view.attribution) != 0 || - manifest.row_span_count != view.packs[index].span_count || - std::memcmp(manifest.row_span_bytes, view.packs[index].span_bytes, - static_cast(manifest.row_span_count) * - ActiveMapSetCodec::ROW_SPAN_SIZE) != 0) { + (!indexless && + (manifest.row_span_count != view.packs[index].span_count || + std::memcmp(manifest.row_span_bytes, view.packs[index].span_bytes, + static_cast(manifest.row_span_count) * + ActiveMapSetCodec::ROW_SPAN_SIZE) != 0))) { return MapTilePackResult::INVALID_MANIFEST; } if (legacy_bright) { @@ -259,11 +269,13 @@ MapTilePackResult MapTilePack::validateMapSet(MapTileStorage& storage, bool MapTilePack::parseMapSetSelection(const std::uint8_t* input, std::size_t length, std::uint32_t& generation, Pyxis::MapPackManifest& metadata, - ActivePackView* packs, std::uint8_t& pack_count) { + ActivePackView* packs, std::uint8_t& pack_count, + bool& indexless) { if (packs == NULL) return false; ActiveMapSetView view = {}; if (!ActiveMapSetCodec::decode(input, length, view)) return false; generation = view.generation; + indexless = view.format_version == ActiveMapSetCodec::INDEXLESS_FORMAT_VERSION; metadata = Pyxis::MapPackManifest(); std::strcpy(metadata.pack_id, view.map_set_id); std::strncpy(metadata.name, metadata.pack_id, sizeof(metadata.name) - 1U); @@ -294,6 +306,110 @@ bool MapTilePack::spanCovers(const ActivePackView& pack, const TileKey& key) { return false; } +bool MapTilePack::sameKey(const TileKey& left, const TileKey& right) { + return left.zoom == right.zoom && left.x == right.x && left.y == right.y; +} + +void MapTilePack::clearResolutionCache() { + for (std::size_t index = 0U; index < RESOLUTION_CACHE_CAPACITY; ++index) { + resolution_cache_[index].valid = false; + resolution_cache_[index].pack_index = 0xffU; + resolution_cache_[index].rank = 0U; + resolution_cache_[index].key = TileKey{0U, 0U, 0U}; + } +} + +int MapTilePack::findResolution(const TileKey& key) { + for (std::size_t index = 0U; index < RESOLUTION_CACHE_CAPACITY; ++index) { + if (!resolution_cache_[index].valid || + !sameKey(resolution_cache_[index].key, key)) continue; + const std::uint8_t previous_rank = resolution_cache_[index].rank; + for (std::size_t other = 0U; other < RESOLUTION_CACHE_CAPACITY; ++other) { + if (other != index && resolution_cache_[other].valid && + resolution_cache_[other].rank < previous_rank) { + ++resolution_cache_[other].rank; + } + } + resolution_cache_[index].rank = 0U; + return static_cast(index); + } + return -1; +} + +void MapTilePack::rememberResolution(const TileKey& key, std::uint8_t pack_index) { + std::size_t target = 0U; + bool found = false; + std::uint8_t oldest = 0U; + for (std::size_t index = 0U; index < RESOLUTION_CACHE_CAPACITY; ++index) { + if (!resolution_cache_[index].valid) { + target = index; found = true; break; + } + if (resolution_cache_[index].rank >= oldest) { + oldest = resolution_cache_[index].rank; target = index; + } + } + (void)found; + for (std::size_t index = 0U; index < RESOLUTION_CACHE_CAPACITY; ++index) { + if (index != target && resolution_cache_[index].valid && + resolution_cache_[index].rank < 0xffU) { + ++resolution_cache_[index].rank; + } + } + resolution_cache_[target].key = key; + resolution_cache_[target].pack_index = pack_index; + resolution_cache_[target].rank = 0U; + resolution_cache_[target].valid = true; +} + +MapTilePackResult MapTilePack::beginIndexless(const TileKey& key, std::uint32_t& size) { + const int cached = findResolution(key); + if (cached >= 0) { + const std::uint8_t pack_index = + resolution_cache_[static_cast(cached)].pack_index; + if (pack_index == 0xffU) return MapTilePackResult::UNCOVERED; + if (pack_index >= active_pack_count_) { + resolution_cache_[static_cast(cached)].valid = false; + } else { + char path[PATH_CAPACITY]; + MapTilePackResult result = tilePath(active_packs_[pack_index].pack_id, + key, path, sizeof(path)); + if (result != MapTilePackResult::OK) return result; + std::uint32_t candidate_size = 0U; + const TileStoreResult begin = storage_.beginRead(path, candidate_size); + if (begin == TileStoreResult::OK) { + stream_open_ = true; stream_remaining_ = candidate_size; size = candidate_size; + return MapTilePackResult::OK; + } + if (begin != TileStoreResult::MISS) { + if (begin == TileStoreResult::STORAGE_UNAVAILABLE) + return MapTilePackResult::STORAGE_UNAVAILABLE; + if (begin == TileStoreResult::BUSY) return MapTilePackResult::BUSY; + return MapTilePackResult::IO_ERROR; + } + resolution_cache_[static_cast(cached)].valid = false; + } + } + + for (std::uint8_t index = 0U; index < active_pack_count_; ++index) { + char path[PATH_CAPACITY]; + MapTilePackResult result = tilePath(active_packs_[index].pack_id, + key, path, sizeof(path)); + if (result != MapTilePackResult::OK) return result; + std::uint32_t candidate_size = 0U; + const TileStoreResult begin = storage_.beginRead(path, candidate_size); + if (begin == TileStoreResult::MISS) continue; + if (begin == TileStoreResult::STORAGE_UNAVAILABLE) + return MapTilePackResult::STORAGE_UNAVAILABLE; + if (begin == TileStoreResult::BUSY) return MapTilePackResult::BUSY; + if (begin != TileStoreResult::OK) return MapTilePackResult::IO_ERROR; + rememberResolution(key, index); + stream_open_ = true; stream_remaining_ = candidate_size; size = candidate_size; + return MapTilePackResult::OK; + } + rememberResolution(key, 0xffU); + return MapTilePackResult::UNCOVERED; +} + MapTilePackResult MapTilePack::makePath(const char* pack_id, const TileKey* key, bool manifest, char* output, std::size_t capacity) { if (!isValidPackId(pack_id)) return MapTilePackResult::INVALID_PACK_ID; @@ -405,11 +521,13 @@ MapTilePackResult MapTilePack::initialize() { Pyxis::MapPackManifest map_set_metadata = {}; ActivePackView map_set_packs[MAX_ACTIVE_PACKS] = {}; std::uint8_t map_set_pack_count = 0U; + bool map_set_indexless = false; if (decodeSelection(selectionBuffer(candidate_buffer), record_length, legacy_id, slot_generations[slot]) || parseMapSetSelection(selectionBuffer(candidate_buffer), record_length, slot_generations[slot], map_set_metadata, - map_set_packs, map_set_pack_count)) { + map_set_packs, map_set_pack_count, + map_set_indexless)) { slot_valid[slot] = true; slot_lengths[slot] = record_length; } @@ -457,11 +575,13 @@ MapTilePackResult MapTilePack::initialize() { Pyxis::MapPackManifest map_set_metadata = {}; ActivePackView map_set_packs[MAX_ACTIVE_PACKS] = {}; std::uint8_t map_set_pack_count = 0U; + bool map_set_indexless = false; ActiveMapSetView map_set_view = {}; if (ActiveMapSetCodec::decode(selectionBuffer(candidate_buffer), record_length, map_set_view) && parseMapSetSelection(selectionBuffer(candidate_buffer), record_length, generation, - map_set_metadata, map_set_packs, map_set_pack_count)) { + map_set_metadata, map_set_packs, map_set_pack_count, + map_set_indexless)) { result = validateMapSet(storage_, map_set_view, manifestBuffer(), MANIFEST_BUFFER_CAPACITY); if (result == MapTilePackResult::OK) { @@ -470,6 +590,8 @@ MapTilePackResult MapTilePack::initialize() { static_cast(map_set_pack_count) * sizeof(ActivePackView)); active_pack_count_ = map_set_pack_count; map_set_active_ = true; + map_set_indexless_ = map_set_indexless; + clearResolutionCache(); selection_generation_ = generation; active_selection_buffer_ = candidate_buffer; status_ = MapTilePackStatus::READY; @@ -503,6 +625,8 @@ MapTilePackResult MapTilePack::initialize() { manifest_ = candidate; active_pack_count_ = 0U; map_set_active_ = false; + map_set_indexless_ = false; + clearResolutionCache(); selection_generation_ = generation; active_selection_buffer_ = candidate_buffer; status_ = MapTilePackStatus::READY; @@ -539,6 +663,8 @@ MapTilePackResult MapTilePack::initialize() { manifest_ = Pyxis::MapPackManifest(); active_pack_count_ = 0U; map_set_active_ = false; + map_set_indexless_ = false; + clearResolutionCache(); selection_generation_ = 0U; status_ = MapTilePackStatus::NO_SELECTION; return MapTilePackResult::NO_SELECTION; @@ -585,6 +711,8 @@ MapTilePackResult MapTilePack::initialize() { manifest_ = candidate; active_pack_count_ = 0U; map_set_active_ = false; + map_set_indexless_ = false; + clearResolutionCache(); selection_generation_ = selected_generation; active_selection_buffer_ = candidate_buffer; status_ = MapTilePackStatus::READY; @@ -598,6 +726,7 @@ MapTilePackResult MapTilePack::beginGet(const TileKey& key, std::uint32_t& size) if (!storage_.isAvailable()) return MapTilePackResult::STORAGE_UNAVAILABLE; if (map_set_active_) { + if (map_set_indexless_) return beginIndexless(key, size); bool covered = false; for (std::uint8_t index = 0U; index < active_pack_count_; ++index) { if (!spanCovers(active_packs_[index], key)) continue; diff --git a/lib/tdeck_ui/Hardware/TDeck/MapTilePack.h b/lib/tdeck_ui/Hardware/TDeck/MapTilePack.h index e7ffb70f..56c1fba5 100644 --- a/lib/tdeck_ui/Hardware/TDeck/MapTilePack.h +++ b/lib/tdeck_ui/Hardware/TDeck/MapTilePack.h @@ -62,6 +62,7 @@ public: static const std::size_t ACTIVE_SELECTION_SIZE = ActiveMapSetCodec::MAX_SERIALIZED_SIZE; static const std::size_t MAX_ACTIVE_PACKS = ActiveMapSetCodec::MAX_PACKS; static const std::size_t MAX_ACTIVE_ROW_SPANS = ActiveMapSetCodec::MAX_ROW_SPANS; + static const std::size_t RESOLUTION_CACHE_CAPACITY = 32U; static const std::size_t PATH_CAPACITY = 80U; static const std::size_t MANIFEST_BUFFER_CAPACITY = Pyxis::MapPackManifest::MAX_SERIALIZED_SIZE; @@ -114,11 +115,20 @@ private: const std::uint8_t* span_bytes; }; + struct ResolutionEntry { + TileKey key; + std::uint8_t pack_index; + std::uint8_t rank; + bool valid; + }; + MapTileStorage& storage_; Pyxis::MapPackManifest manifest_; ActivePackView active_packs_[MAX_ACTIVE_PACKS]; std::uint8_t active_pack_count_; bool map_set_active_; + bool map_set_indexless_; + ResolutionEntry resolution_cache_[RESOLUTION_CACHE_CAPACITY]; std::uint32_t selection_generation_; MapTilePackStatus status_; bool stream_open_; @@ -147,8 +157,14 @@ private: static bool parseMapSetSelection(const std::uint8_t* input, std::size_t length, std::uint32_t& generation, Pyxis::MapPackManifest& metadata, - ActivePackView* packs, std::uint8_t& pack_count); + ActivePackView* packs, std::uint8_t& pack_count, + bool& indexless); static bool spanCovers(const ActivePackView& pack, const TileKey& key); + static bool sameKey(const TileKey& left, const TileKey& right); + void clearResolutionCache(); + int findResolution(const TileKey& key); + void rememberResolution(const TileKey& key, std::uint8_t pack_index); + MapTilePackResult beginIndexless(const TileKey& key, std::uint32_t& size); static MapTilePackResult makePath(const char* pack_id, const TileKey* key, bool manifest, char* output, std::size_t capacity); }; diff --git a/lib/tdeck_ui/UI/LXMF/MapPackManifest.cpp b/lib/tdeck_ui/UI/LXMF/MapPackManifest.cpp index e31397a7..5a331f0e 100644 --- a/lib/tdeck_ui/UI/LXMF/MapPackManifest.cpp +++ b/lib/tdeck_ui/UI/LXMF/MapPackManifest.cpp @@ -201,6 +201,21 @@ ManifestResult validateSparse(const MapPackManifest& manifest, return ManifestResult::OK; } +ManifestResult validateIndexless(const MapPackManifest& manifest) { + std::size_t ignored = 0U; + if (checkedStringLength(manifest.pack_id, sizeof(manifest.pack_id), true, ignored) != ManifestResult::OK || + checkedStringLength(manifest.name, sizeof(manifest.name), false, ignored) != ManifestResult::OK || + checkedStringLength(manifest.attribution, sizeof(manifest.attribution), false, ignored) != ManifestResult::OK || + checkedStringLength(manifest.source, sizeof(manifest.source), false, ignored) != ManifestResult::OK || + checkedStringLength(manifest.license, sizeof(manifest.license), false, ignored) != ManifestResult::OK) { + return ManifestResult::INVALID_STRING; + } + if (manifest.min_zoom > manifest.max_zoom || manifest.max_zoom > MapPackManifest::MAX_ZOOM) { + return ManifestResult::INVALID_ZOOM; + } + return manifest.tile_count == 0U ? ManifestResult::INVALID_TILE_COUNT : ManifestResult::OK; +} + void writeString(std::uint8_t* output, std::size_t& position, const char* text, std::size_t length) { output[position++] = static_cast(length); @@ -331,13 +346,54 @@ ManifestResult MapPackManifest::serializeSparse(const MapPackManifest& manifest, return ManifestResult::OK; } +ManifestResult MapPackManifest::serializeIndexless(const MapPackManifest& manifest, + std::uint8_t* output, + std::size_t capacity, + std::size_t& written) { + written = 0U; + const ManifestResult validity = validateIndexless(manifest); + if (validity != ManifestResult::OK) return validity; + std::size_t lengths[5] = {0U, 0U, 0U, 0U, 0U}; + (void)checkedStringLength(manifest.pack_id, sizeof(manifest.pack_id), true, lengths[0]); + (void)checkedStringLength(manifest.name, sizeof(manifest.name), false, lengths[1]); + (void)checkedStringLength(manifest.attribution, sizeof(manifest.attribution), false, lengths[2]); + (void)checkedStringLength(manifest.source, sizeof(manifest.source), false, lengths[3]); + (void)checkedStringLength(manifest.license, sizeof(manifest.license), false, lengths[4]); + std::size_t required = HEADER_SIZE + CRC_SIZE + 6U; + for (std::size_t index = 0U; index < 5U; ++index) required += 1U + lengths[index]; + if (output == 0 || capacity < required) return ManifestResult::INSUFFICIENT_CAPACITY; + + std::uint8_t temporary[MAX_SERIALIZED_SIZE] = {}; + std::memcpy(temporary, MAGIC, sizeof(MAGIC)); + temporary[4] = INDEXLESS_FORMAT_VERSION; + temporary[5] = 0U; + putU16(temporary + 6U, static_cast(HEADER_SIZE)); + putU32(temporary + 8U, static_cast(required)); + putU32(temporary + 12U, 0U); + std::size_t position = HEADER_SIZE; + writeString(temporary, position, manifest.pack_id, lengths[0]); + writeString(temporary, position, manifest.name, lengths[1]); + writeString(temporary, position, manifest.attribution, lengths[2]); + writeString(temporary, position, manifest.source, lengths[3]); + writeString(temporary, position, manifest.license, lengths[4]); + temporary[position++] = manifest.min_zoom; + temporary[position++] = manifest.max_zoom; + putU32(temporary + position, manifest.tile_count); position += 4U; + putU32(temporary + position, crc32(temporary, position)); position += CRC_SIZE; + if (position != required) return ManifestResult::BAD_LENGTH; + std::memcpy(output, temporary, required); + written = required; + return ManifestResult::OK; +} + ManifestResult MapPackManifest::parse(const std::uint8_t* input, std::size_t length, MapPackManifest& output) { if (input == 0 || length < HEADER_SIZE + CRC_SIZE) return ManifestResult::BAD_LENGTH; if (std::memcmp(input, MAGIC, sizeof(MAGIC)) != 0) return ManifestResult::BAD_MAGIC; const std::uint8_t version = input[4]; - if (version != LEGACY_FORMAT_VERSION && version != FORMAT_VERSION) { + if (version != LEGACY_FORMAT_VERSION && version != FORMAT_VERSION && + version != INDEXLESS_FORMAT_VERSION) { return ManifestResult::UNSUPPORTED_VERSION; } if (input[5] != 0U || getU16(input + 6U) != HEADER_SIZE || getU32(input + 12U) != 0U) { @@ -383,7 +439,7 @@ ManifestResult MapPackManifest::parse(const std::uint8_t* input, } if (position != payload_end) return ManifestResult::BAD_LENGTH; result = validate(candidate); - } else { + } else if (version == FORMAT_VERSION) { if (payload_end - position < 8U) return ManifestResult::BAD_LENGTH; candidate.min_zoom = input[position++]; candidate.max_zoom = input[position++]; @@ -433,6 +489,16 @@ ManifestResult MapPackManifest::parse(const std::uint8_t* input, result = zoom_mask != expected_mask ? ManifestResult::INVALID_ZOOM : (candidate.tile_count == 0U || total != candidate.tile_count ? ManifestResult::INVALID_TILE_COUNT : ManifestResult::OK); + } else { + if (payload_end - position != 6U) return ManifestResult::BAD_LENGTH; + candidate.min_zoom = input[position++]; + candidate.max_zoom = input[position++]; + candidate.tile_count = getU32(input + position); position += 4U; + candidate.extent_count = 0U; + candidate.row_span_count = 0U; + candidate.row_span_bytes = 0; + result = position != payload_end ? ManifestResult::BAD_LENGTH + : validateIndexless(candidate); } if (result != ManifestResult::OK) return result; output = candidate; @@ -444,6 +510,7 @@ bool MapPackManifest::covers(const Hardware::TDeck::TileKey& key) const { const std::uint32_t world_size = UINT32_C(1) << key.zoom; if (key.x >= world_size || key.y >= world_size || key.zoom < min_zoom || key.zoom > max_zoom) return false; + if (format_version == INDEXLESS_FORMAT_VERSION) return false; if (format_version == FORMAT_VERSION) { if (row_span_bytes == 0 || row_span_count == 0U || row_span_count > MAX_ROW_SPANS) return false; for (std::size_t index = 0U; index < row_span_count; ++index) { diff --git a/lib/tdeck_ui/UI/LXMF/MapPackManifest.h b/lib/tdeck_ui/UI/LXMF/MapPackManifest.h index 9fd726e3..80cd4025 100644 --- a/lib/tdeck_ui/UI/LXMF/MapPackManifest.h +++ b/lib/tdeck_ui/UI/LXMF/MapPackManifest.h @@ -58,6 +58,7 @@ struct RowSpan { struct MapPackManifest { static const std::uint8_t LEGACY_FORMAT_VERSION = 1U; static const std::uint8_t FORMAT_VERSION = 2U; + static const std::uint8_t INDEXLESS_FORMAT_VERSION = 3U; static const std::uint8_t MAX_ZOOM = 22U; static const std::size_t MAX_ZOOM_LEVELS = 23U; static const std::size_t PACK_ID_CAPACITY = 32U; @@ -94,6 +95,10 @@ struct MapPackManifest { std::uint8_t* output, std::size_t capacity, std::size_t& written); + static ManifestResult serializeIndexless(const MapPackManifest& manifest, + std::uint8_t* output, + std::size_t capacity, + std::size_t& written); static ManifestResult parse(const std::uint8_t* input, std::size_t length, MapPackManifest& output); diff --git a/tests/native/test_map_pack_manifest.cpp b/tests/native/test_map_pack_manifest.cpp index 3ad3dfe5..29050afd 100644 --- a/tests/native/test_map_pack_manifest.cpp +++ b/tests/native/test_map_pack_manifest.cpp @@ -136,7 +136,7 @@ void testMagicVersionHeaderAndLengthRejected() { std::vector bad = valid; bad[index] ^= 1U; CHECK(MapPackManifest::parse(&bad[0], bad.size(), output) == ManifestResult::BAD_MAGIC); } - std::vector bad = valid; bad[4] = 3U; refreshCrc(bad); + std::vector bad = valid; bad[4] = 4U; refreshCrc(bad); CHECK(MapPackManifest::parse(&bad[0], bad.size(), output) == ManifestResult::UNSUPPORTED_VERSION); bad = valid; bad[6] = 15U; refreshCrc(bad); CHECK(MapPackManifest::parse(&bad[0], bad.size(), output) == ManifestResult::BAD_HEADER); @@ -363,6 +363,29 @@ void testSparseRowSpansMustBeCanonicalAndBounded() { CHECK(MapPackManifest::serializeSparse(manifest, reversed, 2U, storage, sizeof(storage), written) == ManifestResult::INVALID_EXTENT); } + +void testIndexlessManifestRoundTripKeepsMetadataWithoutClaimingCoverage() { + beginTest(); + MapPackManifest manifest = sample(); + manifest.min_zoom = 0U; + manifest.max_zoom = 9U; + manifest.tile_count = 83567U; + manifest.extent_count = 0U; + std::uint8_t storage[MapPackManifest::MAX_SERIALIZED_SIZE] = {}; + std::size_t written = 0U; + CHECK(MapPackManifest::serializeIndexless( + manifest, storage, sizeof(storage), written) == ManifestResult::OK); + CHECK(written < 512U); + + MapPackManifest parsed = {}; + CHECK(MapPackManifest::parse(storage, written, parsed) == ManifestResult::OK); + CHECK(parsed.format_version == MapPackManifest::INDEXLESS_FORMAT_VERSION); + CHECK(parsed.min_zoom == 0U); + CHECK(parsed.max_zoom == 9U); + CHECK(parsed.tile_count == 83567U); + CHECK(parsed.row_span_count == 0U); + CHECK(!parsed.covers(TileKey{9U, 150U, 100U})); +} } // namespace int main() { @@ -381,6 +404,7 @@ int main() { testMaximumSerializedSizeRoundTrip(); testSparseRowSpanRoundTripAndExactCoverage(); testSparseRowSpansMustBeCanonicalAndBounded(); + testIndexlessManifestRoundTripKeepsMetadataWithoutClaimingCoverage(); std::cout << "map pack manifest: " << tests_run << " tests passed\n"; return 0; } diff --git a/tests/native/test_map_pack_manifest.py b/tests/native/test_map_pack_manifest.py index 63e483ba..8fe866aa 100644 --- a/tests/native/test_map_pack_manifest.py +++ b/tests/native/test_map_pack_manifest.py @@ -31,4 +31,4 @@ def test_map_pack_manifest(tmp_path: Path, sanitize: bool) -> None: env["UBSAN_OPTIONS"] = "halt_on_error=1:print_stacktrace=1" ran = subprocess.run([str(binary)], capture_output=True, text=True, timeout=60, env=env) assert ran.returncode == 0, ran.stdout + ran.stderr - assert ran.stdout == "map pack manifest: 15 tests passed\n" + assert ran.stdout == "map pack manifest: 16 tests passed\n" diff --git a/tests/native/test_map_tile_pack.cpp b/tests/native/test_map_tile_pack.cpp index 8d4c593f..f69bf36f 100644 --- a/tests/native/test_map_tile_pack.cpp +++ b/tests/native/test_map_tile_pack.cpp @@ -54,8 +54,9 @@ class FakeStorage : public MapTileStorage { public: FakeStorage() : available(true), file_count(0U), open_file(NULL), position(0U), - read_calls(0U), end_calls(0U), fail_read_call(0U), zero_read_call(0U), - fail_begin_path(NULL), fail_begin_result(TileStoreResult::IO_ERROR) {} + begin_attempts(0U), read_calls(0U), end_calls(0U), fail_read_call(0U), + zero_read_call(0U), fail_begin_path(NULL), + fail_begin_result(TileStoreResult::IO_ERROR) {} void clear() { file_count = 0U; open_file = NULL; position = 0U; } void add(const char* path, const std::uint8_t* bytes, std::size_t size) { @@ -83,6 +84,7 @@ public: virtual bool isAvailable() const { return available; } virtual TileStoreResult beginRead(const char* path, std::uint32_t& size) { + ++begin_attempts; if (!available) return TileStoreResult::STORAGE_UNAVAILABLE; if (fail_begin_path != NULL && std::strcmp(path, fail_begin_path) == 0) return fail_begin_result; @@ -121,6 +123,7 @@ public: std::size_t file_count; File* open_file; std::size_t position; + std::size_t begin_attempts; std::size_t read_calls; std::size_t end_calls; std::size_t fail_read_call; @@ -207,6 +210,24 @@ void addSparseManifest(FakeStorage& storage, const char* id, const char* attribu storage.add(path, bytes, written); } +void addIndexlessManifest(FakeStorage& storage, const char* id, const char* attribution, + const char* source, const char* license, + std::uint8_t min_zoom, std::uint8_t max_zoom, + std::uint32_t tile_count) { + MapPackManifest manifest = {}; + std::strcpy(manifest.pack_id, id); std::strcpy(manifest.name, "Test Pack"); + std::strcpy(manifest.attribution, attribution); std::strcpy(manifest.source, source); + std::strcpy(manifest.license, license); + manifest.min_zoom = min_zoom; manifest.max_zoom = max_zoom; + manifest.tile_count = tile_count; + std::uint8_t bytes[MapPackManifest::MAX_SERIALIZED_SIZE]; std::size_t written = 0U; + CHECK(MapPackManifest::serializeIndexless( + manifest, bytes, sizeof(bytes), written) == Pyxis::ManifestResult::OK); + char path[MapTilePack::PATH_CAPACITY]; + CHECK(MapTilePack::manifestPath(id, path, sizeof(path)) == MapTilePackResult::OK); + storage.add(path, bytes, written); +} + void addSlot(FakeStorage& storage, const char* path, const char* id, std::uint32_t generation, bool corrupt = false) { std::uint8_t record[MapTilePack::LEGACY_ACTIVE_SELECTION_SIZE] = {}; @@ -255,6 +276,32 @@ void addMapSetSlot(FakeStorage& storage, const char* path, std::uint32_t generat storage.add(path, record, total_length); } +void addIndexlessMapSetSlot(FakeStorage& storage, const char* path, + std::uint32_t generation, + const char* map_set = "osm-bright", + const char* attribution = + "(c) OpenMapTiles (c) OpenStreetMap contributors") { + std::uint8_t record[256] = {}; + record[0] = 'P'; record[1] = 'M'; record[2] = 'A'; record[3] = 'S'; record[4] = 3U; + testPutU32(record + 8U, generation); + std::size_t position = 12U; + record[position++] = static_cast(std::strlen(map_set)); + std::memcpy(record + position, map_set, std::strlen(map_set)); position += std::strlen(map_set); + record[position++] = static_cast(std::strlen(attribution)); + std::memcpy(record + position, attribution, std::strlen(attribution)); position += std::strlen(attribution); + record[position++] = 2U; + const char* ids[2] = {"detail", "overview"}; + for (std::size_t index = 0U; index < 2U; ++index) { + const std::size_t length = std::strlen(ids[index]); + record[position++] = static_cast(length); + std::memcpy(record + position, ids[index], length); position += length; + } + const std::size_t total_length = position + 4U; + testPutU16(record + 6U, static_cast(total_length)); + testPutU32(record + position, testCrc32(record, position)); + storage.add(path, record, total_length); +} + void addLegacyBrightMapSetManifests(FakeStorage& storage) { const RowSpan detail[] = {{2U, 1U, 1U, 1U}, {3U, 5U, 5U, 5U}}; const RowSpan state[] = {{2U, 1U, 1U, 2U}}; @@ -554,6 +601,68 @@ void testActiveMapSetComposesPacksByPriorityAndCoverage() { CHECK(fallback_pack.readGetChunk(&output, 1U, count) == MapTilePackResult::OK); CHECK(output == state_overlap); } + +void testIndexlessMapSetResolvesVisiblePathsByPriorityAndCachesWinnersAndMisses() { + beginTest(); FakeStorage storage; + const char* attribution = "(c) OpenMapTiles (c) OpenStreetMap contributors"; + const char* source = "Oxed's Map Tile Downloader (OSM Bright)"; + const char* license = "OSM ODbL; style CC-BY-4.0/BSD-3-Clause"; + addIndexlessMapSetSlot(storage, MapTilePack::ACTIVE_PACK_SLOT_0_PATH, 9U); + addIndexlessManifest(storage, "detail", attribution, source, license, 0U, 22U, 1U); + addIndexlessManifest(storage, "overview", attribution, source, license, 0U, 9U, 2U); + const std::uint8_t detail = 0x11U, overview = 0x22U; + storage.add("/pyxis-map/packs/detail/tiles/9/150/100.png", &detail, 1U); + storage.add("/pyxis-map/packs/overview/tiles/9/150/100.png", &overview, 1U); + storage.add("/pyxis-map/packs/overview/tiles/9/151/100.png", &overview, 1U); + MapTilePack pack(storage); CHECK(pack.initialize() == MapTilePackResult::OK); + + std::uint32_t size = 0U; std::uint8_t output = 0U; std::size_t count = 0U; + const std::size_t before_priority = storage.begin_attempts; + CHECK(pack.beginGet(TileKey{9U,150U,100U}, size) == MapTilePackResult::OK); + CHECK(storage.begin_attempts == before_priority + 1U); + CHECK(pack.readGetChunk(&output, 1U, count) == MapTilePackResult::OK); + CHECK(output == detail); + + const std::size_t before_fallback = storage.begin_attempts; + CHECK(pack.beginGet(TileKey{9U,151U,100U}, size) == MapTilePackResult::OK); + CHECK(storage.begin_attempts == before_fallback + 2U); + CHECK(pack.readGetChunk(&output, 1U, count) == MapTilePackResult::OK); + CHECK(output == overview); + const std::size_t before_cached_winner = storage.begin_attempts; + CHECK(pack.beginGet(TileKey{9U,151U,100U}, size) == MapTilePackResult::OK); + CHECK(storage.begin_attempts == before_cached_winner + 1U); + pack.endGet(); + + const TileKey missing = {9U,152U,100U}; + const std::size_t before_missing = storage.begin_attempts; + CHECK(pack.beginGet(missing, size) == MapTilePackResult::UNCOVERED); + CHECK(storage.begin_attempts == before_missing + 2U); + const std::size_t before_cached_missing = storage.begin_attempts; + CHECK(pack.beginGet(missing, size) == MapTilePackResult::UNCOVERED); + CHECK(storage.begin_attempts == before_cached_missing); +} + +void testIndexlessSelectionMigratesExistingSparseManifests() { + beginTest(); FakeStorage storage; + addIndexlessMapSetSlot(storage, MapTilePack::ACTIVE_PACK_SLOT_0_PATH, 10U, + "osm-bright", "Map data (c) OpenStreetMap contributors"); + const RowSpan detail_spans[] = {{2U, 1U, 1U, 1U}}; + const RowSpan overview_spans[] = {{2U, 1U, 1U, 2U}}; + addSparseManifest(storage, "detail", "Map data (c) OpenStreetMap contributors", + "Coalition MUI OSM Bright user download", "ODbL-1.0", + detail_spans, 1U); + addSparseManifest(storage, "overview", "Map data (c) OpenStreetMap contributors", + "Coalition MUI OSM Bright user download", "ODbL-1.0", + overview_spans, 1U); + const std::uint8_t tile = 0x33U; + storage.add("/pyxis-map/packs/overview/tiles/2/2/1.png", &tile, 1U); + MapTilePack pack(storage); CHECK(pack.initialize() == MapTilePackResult::OK); + std::uint32_t size = 0U; + CHECK(pack.beginGet(TileKey{2U,2U,1U}, size) == MapTilePackResult::OK); + std::uint8_t output = 0U; std::size_t count = 0U; + CHECK(pack.readGetChunk(&output, 1U, count) == MapTilePackResult::OK); + CHECK(output == tile); +} void testRebootFallsBackFromNewerSemanticallyInvalidMapSet() { beginTest(); FakeStorage storage; addMapSetSlot(storage, MapTilePack::ACTIVE_PACK_SLOT_0_PATH, 4U); @@ -619,6 +728,8 @@ int main() { testActiveMapSetRequiresAllowlistedStyleAndMatchingImmutableManifests(); testNewCanonicalProfilesPassAndCrossStyleCompositionFails(); testActiveMapSetComposesPacksByPriorityAndCoverage(); + testIndexlessMapSetResolvesVisiblePathsByPriorityAndCachesWinnersAndMisses(); + testIndexlessSelectionMigratesExistingSparseManifests(); testRebootFallsBackFromNewerSemanticallyInvalidMapSet(); testNewerManifestIndeterminateDoesNotFallBackAndPreservesSelection(); testBothSemanticallyInvalidSlotsDoNotUseLegacyMarker(); diff --git a/tests/native/test_map_tile_pack.py b/tests/native/test_map_tile_pack.py index 634ee5d6..83cf2873 100644 --- a/tests/native/test_map_tile_pack.py +++ b/tests/native/test_map_tile_pack.py @@ -34,4 +34,4 @@ def test_map_tile_pack(tmp_path: Path, sanitize: bool) -> None: env["UBSAN_OPTIONS"] = "halt_on_error=1:print_stacktrace=1" ran = subprocess.run([str(binary)], capture_output=True, text=True, timeout=60, env=env) assert ran.returncode == 0, ran.stdout + ran.stderr - assert ran.stdout == "map tile pack: 26 tests passed\n" + assert ran.stdout == "map tile pack: 28 tests passed\n"