diff --git a/lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.cpp b/lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.cpp index 71d7e031..80a216cb 100644 --- a/lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.cpp +++ b/lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.cpp @@ -43,6 +43,12 @@ bool CompactPage::assign(const Document& document) { const std::size_t block_count = std::min(document.blocks.size(), block_limit); const std::size_t link_count = std::min(document.links.size(), MAX_LINKS); const std::size_t field_count = std::min(document.fields.size(), MAX_FIELDS); + const std::size_t partial_count = std::min(document.partials.size(), MAX_PARTIALS); + std::size_t partial_field_count = 0; + for (std::size_t i = 0; i < partial_count; ++i) { + partial_field_count += std::min(document.partials[i].fields.size(), + MAX_PARTIAL_FIELDS - std::min(partial_field_count, MAX_PARTIAL_FIELDS)); + } const std::size_t table_count = std::min(document.tables.size(), MAX_TABLES); std::size_t table_cell_count = 0; for (std::size_t i = 0; i < table_count; ++i) { @@ -90,6 +96,14 @@ bool CompactPage::assign(const Document& document) { if (bytes > MAX_ARENA_BYTES - std::min(arena_size, MAX_ARENA_BYTES)) return false; arena_size += bytes; } + for (std::size_t i = 0; i < partial_count; ++i) { + const auto& partial = document.partials[i]; + std::size_t bytes = partial.descriptor.size() + partial.url.size() + + partial.selectors.size() + partial.id.size() + 4; + for (const auto& field : partial.fields) bytes += field.size() + 1; + if (bytes > MAX_ARENA_BYTES - std::min(arena_size, MAX_ARENA_BYTES)) return false; + arena_size += bytes; + } std::size_t anchors_accounted = 0; for (const auto& anchor : document.anchors) { if (anchors_accounted >= anchor_count) break; @@ -108,6 +122,8 @@ bool CompactPage::assign(const Document& document) { _anchors.reserve(anchor_count); _tables.reserve(table_count); _table_cells.reserve(table_cell_count); + _partials.reserve(partial_count); + _partial_fields.reserve(partial_field_count); for (std::size_t i = 0; i < link_count; ++i) { LinkRecord link; @@ -139,6 +155,33 @@ bool CompactPage::assign(const Document& document) { _fields.push_back(field); } + for (std::size_t i = 0; i < partial_count; ++i) { + const auto& source = document.partials[i]; + PartialRecord partial; + partial.first_field = static_cast(_partial_fields.size()); + if (!append(source.descriptor, partial.descriptor_offset, partial.descriptor_length) || + !append(source.url, partial.url_offset, partial.url_length) || + !append(source.selectors, partial.selectors_offset, partial.selectors_length) || + !append(source.id, partial.id_offset, partial.id_length)) { + clear(); + return false; + } + partial.refresh_interval_ms = source.refresh_interval_ms; + partial.descriptor_hash = source.descriptor_hash; + for (const auto& source_field : source.fields) { + if (_partial_fields.size() >= MAX_PARTIAL_FIELDS || + partial.field_count == std::numeric_limits::max()) break; + PartialFieldRecord field; + if (!append(source_field, field.value_offset, field.value_length)) { + clear(); + return false; + } + _partial_fields.push_back(field); + ++partial.field_count; + } + _partials.push_back(partial); + } + for (const auto& source_anchor : document.anchors) { if (_anchors.size() >= anchor_count) break; if (source_anchor.block_index >= block_count || @@ -167,6 +210,9 @@ bool CompactPage::assign(const Document& document) { block.table_index = source_block.table_index >= 0 && static_cast(source_block.table_index) < table_count ? source_block.table_index : -1; + block.partial_index = source_block.partial_index >= 0 && + static_cast(source_block.partial_index) < partial_count + ? source_block.partial_index : -1; for (const auto& source_run : source_block.runs) { if (_runs.size() >= run_limit || block.run_count == std::numeric_limits::max()) { _truncated = true; @@ -264,6 +310,7 @@ bool CompactPage::assign(const Document& document) { _truncated = _truncated || document.truncated || document.blocks.size() > block_count || document.links.size() > link_count || document.anchors.size() > anchor_count || document.fields.size() > field_count || document.tables.size() > table_count || + document.partials.size() > partial_count || document.table_cells.size() > table_cell_count || document.table_runs.size() > table_run_count; _unsupported = document.unsupported; @@ -283,6 +330,8 @@ void CompactPage::clear() { ExternalVector().swap(_tables); ExternalVector().swap(_table_cells); ExternalVector().swap(_fields); + ExternalVector().swap(_partials); + ExternalVector().swap(_partial_fields); _has_background = false; _background = 0; _has_foreground = false; @@ -362,6 +411,40 @@ CompactPage::TextView CompactPage::field_label(std::size_t index) const { return {_arena.data() + field.label_offset, field.label_length}; } +CompactPage::TextView CompactPage::partial_descriptor(const PartialRecord& partial) const { + if (partial.descriptor_offset > _arena.size() || + partial.descriptor_length > _arena.size() - partial.descriptor_offset) return {}; + return {_arena.data() + partial.descriptor_offset, partial.descriptor_length}; +} + +CompactPage::TextView CompactPage::partial_url(const PartialRecord& partial) const { + if (partial.url_offset > _arena.size() || + partial.url_length > _arena.size() - partial.url_offset) return {}; + return {_arena.data() + partial.url_offset, partial.url_length}; +} + +CompactPage::TextView CompactPage::partial_selectors(const PartialRecord& partial) const { + if (partial.selectors_offset > _arena.size() || + partial.selectors_length > _arena.size() - partial.selectors_offset) return {}; + return {_arena.data() + partial.selectors_offset, partial.selectors_length}; +} + +CompactPage::TextView CompactPage::partial_id(const PartialRecord& partial) const { + if (partial.id_offset > _arena.size() || + partial.id_length > _arena.size() - partial.id_offset) return {}; + return {_arena.data() + partial.id_offset, partial.id_length}; +} + +CompactPage::TextView CompactPage::partial_field(const PartialRecord& partial, + std::size_t index) const { + if (index >= partial.field_count || partial.first_field > _partial_fields.size() || + index >= _partial_fields.size() - partial.first_field) return {}; + const auto& field = _partial_fields[partial.first_field + index]; + if (field.value_offset > _arena.size() || + field.value_length > _arena.size() - field.value_offset) return {}; + return {_arena.data() + field.value_offset, field.value_length}; +} + bool CompactPage::find_anchor(const std::string& name, uint16_t& block_index) const { if (name.size() > DocumentParser::MAX_ANCHOR_NAME_BYTES) return false; for (const auto& anchor : _anchors) { diff --git a/lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.h b/lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.h index d3e6e678..8b32bc7b 100644 --- a/lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.h +++ b/lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.h @@ -98,6 +98,9 @@ public: static constexpr std::size_t MAX_TABLES = DocumentParser::MAX_TABLES; static constexpr std::size_t MAX_TABLE_CELLS = DocumentParser::MAX_TOTAL_TABLE_CELLS; static constexpr std::size_t MAX_FIELDS = DocumentParser::MAX_FIELDS; + static constexpr std::size_t MAX_PARTIALS = DocumentParser::MAX_PARTIALS; + static constexpr std::size_t MAX_PARTIAL_FIELDS = + MAX_PARTIALS * DocumentParser::MAX_PARTIAL_FIELDS; // Text runs, link targets, and anchor names originate in the bounded source. // Link targets also remain visible in runs, while anchor declarations are // zero-width. Account for both bounded copies and one terminator per record. @@ -106,6 +109,7 @@ public: DocumentParser::MAX_DOCUMENT_BYTES * 2 + DocumentParser::MAX_FORM_BYTES + MAX_FIELDS * 3 + MAX_ANCHORS * (DocumentParser::MAX_ANCHOR_NAME_BYTES + 1) + + DocumentParser::MAX_PARTIAL_BYTES + MAX_RUNS + MAX_LINKS + MAX_NOTICE_BYTES + 1; enum Style : uint8_t { @@ -124,6 +128,7 @@ public: Alignment alignment = Alignment::LEFT; uint32_t divider_codepoint = 0x2500; int16_t table_index = -1; + int16_t partial_index = -1; }; struct RunRecord { @@ -174,6 +179,26 @@ public: bool masked = false; }; + struct PartialRecord { + uint32_t descriptor_offset = 0; + uint32_t url_offset = 0; + uint32_t selectors_offset = 0; + uint32_t id_offset = 0; + uint32_t first_field = 0; + uint16_t descriptor_length = 0; + uint16_t url_length = 0; + uint16_t selectors_length = 0; + uint16_t id_length = 0; + uint16_t field_count = 0; + uint32_t refresh_interval_ms = 0; + std::array descriptor_hash{}; + }; + + struct PartialFieldRecord { + uint32_t value_offset = 0; + uint16_t value_length = 0; + }; + struct TextView { const char* value = nullptr; std::size_t length = 0; @@ -202,11 +227,17 @@ public: const ExternalVector& tables() const { return _tables; } const ExternalVector& table_cells() const { return _table_cells; } const ExternalVector& fields() const { return _fields; } + const ExternalVector& partials() const { return _partials; } TextView text(const RunRecord& run) const; TextView target(std::size_t index) const; TextView field_name(std::size_t index) const; TextView field_value(std::size_t index) const; TextView field_label(std::size_t index) const; + TextView partial_descriptor(const PartialRecord& partial) const; + TextView partial_url(const PartialRecord& partial) const; + TextView partial_selectors(const PartialRecord& partial) const; + TextView partial_id(const PartialRecord& partial) const; + TextView partial_field(const PartialRecord& partial, std::size_t index) const; bool find_anchor(const std::string& name, uint16_t& block_index) const; bool has_background() const { return _has_background; } @@ -229,6 +260,8 @@ private: ExternalVector _tables; ExternalVector _table_cells; ExternalVector _fields; + ExternalVector _partials; + ExternalVector _partial_fields; bool _has_background = false; uint32_t _background = 0; bool _has_foreground = false; diff --git a/lib/tdeck_ui/UI/LXMF/NomadNetDocument.cpp b/lib/tdeck_ui/UI/LXMF/NomadNetDocument.cpp index 650f2e20..15a0d8d2 100644 --- a/lib/tdeck_ui/UI/LXMF/NomadNetDocument.cpp +++ b/lib/tdeck_ui/UI/LXMF/NomadNetDocument.cpp @@ -1,8 +1,10 @@ #include "NomadNetDocument.h" +#include "NomadNetPartialHash.h" #include #include #include +#include #include namespace UI::LXMF::NomadNet { @@ -642,6 +644,259 @@ bool append_table(Document& doc, const std::vector& lines, return true; } +struct DecimalDigitRange { + uint32_t first; + uint32_t last; +}; + +constexpr DecimalDigitRange PYTHON_DECIMAL_DIGIT_RANGES[] = { + {0x30U, 0x39U}, {0x660U, 0x669U}, {0x6F0U, 0x6F9U}, + {0x7C0U, 0x7C9U}, {0x966U, 0x96FU}, {0x9E6U, 0x9EFU}, + {0xA66U, 0xA6FU}, {0xAE6U, 0xAEFU}, {0xB66U, 0xB6FU}, + {0xBE6U, 0xBEFU}, {0xC66U, 0xC6FU}, {0xCE6U, 0xCEFU}, + {0xD66U, 0xD6FU}, {0xDE6U, 0xDEFU}, {0xE50U, 0xE59U}, + {0xED0U, 0xED9U}, {0xF20U, 0xF29U}, {0x1040U, 0x1049U}, + {0x1090U, 0x1099U}, {0x17E0U, 0x17E9U}, {0x1810U, 0x1819U}, + {0x1946U, 0x194FU}, {0x19D0U, 0x19D9U}, {0x1A80U, 0x1A89U}, + {0x1A90U, 0x1A99U}, {0x1B50U, 0x1B59U}, {0x1BB0U, 0x1BB9U}, + {0x1C40U, 0x1C49U}, {0x1C50U, 0x1C59U}, {0xA620U, 0xA629U}, + {0xA8D0U, 0xA8D9U}, {0xA900U, 0xA909U}, {0xA9D0U, 0xA9D9U}, + {0xA9F0U, 0xA9F9U}, {0xAA50U, 0xAA59U}, {0xABF0U, 0xABF9U}, + {0xFF10U, 0xFF19U}, {0x104A0U, 0x104A9U}, {0x10D30U, 0x10D39U}, + {0x11066U, 0x1106FU}, {0x110F0U, 0x110F9U}, {0x11136U, 0x1113FU}, + {0x111D0U, 0x111D9U}, {0x112F0U, 0x112F9U}, {0x11450U, 0x11459U}, + {0x114D0U, 0x114D9U}, {0x11650U, 0x11659U}, {0x116C0U, 0x116C9U}, + {0x11730U, 0x11739U}, {0x118E0U, 0x118E9U}, {0x11950U, 0x11959U}, + {0x11C50U, 0x11C59U}, {0x11D50U, 0x11D59U}, {0x11DA0U, 0x11DA9U}, + {0x11F50U, 0x11F59U}, {0x16A60U, 0x16A69U}, {0x16AC0U, 0x16AC9U}, + {0x16B50U, 0x16B59U}, {0x1D7CEU, 0x1D7FFU}, {0x1E140U, 0x1E149U}, + {0x1E2F0U, 0x1E2F9U}, {0x1E4F0U, 0x1E4F9U}, {0x1E950U, 0x1E959U}, + {0x1FBF0U, 0x1FBF9U}, +}; + +bool decode_utf8_codepoint(const std::string& input, std::size_t& position, + std::size_t end, uint32_t& codepoint) { + if (position >= end) return false; + const unsigned char lead = static_cast(input[position]); + std::size_t length = 0; + if (lead < 0x80U) { + codepoint = lead; + length = 1; + } else if (lead >= 0xC2U && lead <= 0xDFU) { + codepoint = lead & 0x1FU; + length = 2; + } else if (lead >= 0xE0U && lead <= 0xEFU) { + codepoint = lead & 0x0FU; + length = 3; + } else if (lead >= 0xF0U && lead <= 0xF4U) { + codepoint = lead & 0x07U; + length = 4; + } else { + return false; + } + if (length > end - position) return false; + for (std::size_t i = 1; i < length; ++i) { + const unsigned char continuation = static_cast(input[position + i]); + if ((continuation & 0xC0U) != 0x80U) return false; + codepoint = (codepoint << 6U) | (continuation & 0x3FU); + } + if ((length == 3 && codepoint < 0x800U) || + (length == 4 && codepoint < 0x10000U) || + (codepoint >= 0xD800U && codepoint <= 0xDFFFU) || codepoint > 0x10FFFFU) + return false; + position += length; + return true; +} + +bool decode_python_decimal_digit(const std::string& input, std::size_t& position, + std::size_t end, uint8_t& digit) { + std::size_t after_codepoint = position; + uint32_t codepoint = 0; + if (!decode_utf8_codepoint(input, after_codepoint, end, codepoint)) return false; + for (const auto& range : PYTHON_DECIMAL_DIGIT_RANGES) { + if (codepoint >= range.first && codepoint <= range.last) { + digit = static_cast((codepoint - range.first) % 10U); + position = after_codepoint; + return true; + } + } + return false; +} + +bool python_float_whitespace(uint32_t codepoint) { + return (codepoint >= 0x09U && codepoint <= 0x0DU) || + codepoint == 0x20U || codepoint == 0x85U || codepoint == 0xA0U || + codepoint == 0x1680U || + (codepoint >= 0x2000U && codepoint <= 0x200AU) || + codepoint == 0x2028U || codepoint == 0x2029U || codepoint == 0x202FU || + codepoint == 0x205FU || codepoint == 0x3000U; +} + +bool parse_python_decimal(const std::string& input, double& value) { + std::string canonical; + canonical.reserve(input.size()); + for (std::size_t position = 0; position < input.size();) { + std::size_t after_digit = position; + uint8_t digit = 0; + if (decode_python_decimal_digit(input, after_digit, input.size(), digit)) { + canonical.push_back(static_cast('0' + digit)); + position = after_digit; + continue; + } + uint32_t codepoint = 0; + std::size_t after_codepoint = position; + if (!decode_utf8_codepoint(input, after_codepoint, input.size(), codepoint)) return false; + if (python_float_whitespace(codepoint)) canonical.push_back(' '); + else if (codepoint <= 0x7FU) canonical.push_back(static_cast(codepoint)); + else return false; + position = after_codepoint; + } + + std::size_t begin = 0; + std::size_t end = canonical.size(); + while (begin < end && canonical[begin] == ' ') ++begin; + while (end > begin && canonical[end - 1] == ' ') --end; + if (begin == end) return false; + + std::size_t cursor = begin; + if (canonical[cursor] == '+' || canonical[cursor] == '-') ++cursor; + if (cursor == end) return false; + + auto consume_digits = [&](std::size_t& position) { + bool any = false; + bool previous_digit = false; + while (position < end) { + const unsigned char current = static_cast(canonical[position]); + if (std::isdigit(current)) { + any = true; + previous_digit = true; + ++position; + } else if (canonical[position] == '_' && previous_digit && + position + 1 < end) { + if (!std::isdigit(static_cast(canonical[position + 1]))) break; + previous_digit = false; + ++position; + } else { + break; + } + } + return any && previous_digit; + }; + + const bool integer_digits = consume_digits(cursor); + bool fraction_digits = false; + if (cursor < end && canonical[cursor] == '.') { + ++cursor; + fraction_digits = consume_digits(cursor); + } + if (!integer_digits && !fraction_digits) return false; + if (cursor < end && (canonical[cursor] == 'e' || canonical[cursor] == 'E')) { + ++cursor; + if (cursor < end && (canonical[cursor] == '+' || canonical[cursor] == '-')) ++cursor; + if (!consume_digits(cursor)) return false; + } + if (cursor != end) return false; + + std::string normalized; + normalized.reserve(end - begin); + for (std::size_t i = begin; i < end; ++i) { + if (canonical[i] != '_') normalized.push_back(canonical[i]); + } + char* parse_end = nullptr; + value = std::strtod(normalized.c_str(), &parse_end); + return parse_end && parse_end != normalized.c_str() && *parse_end == '\0'; +} + +bool parse_partial_descriptor(const std::string& line, Partial& partial, + TruncationReason& limit_reason) { + limit_reason = TruncationReason::NONE; + const std::size_t close = line.find('}', 2); + if (line.rfind("`{", 0) != 0 || close == std::string::npos) return false; + const std::size_t data_size = close - 2; + if (data_size > DocumentParser::MAX_PARTIAL_DESCRIPTOR_BYTES) { + limit_reason = TruncationReason::PARTIAL_DESCRIPTOR_BYTES; + return false; + } + const std::string data = line.substr(2, data_size); + const std::size_t first_separator = data.find('`'); + const std::size_t second_separator = first_separator == std::string::npos + ? std::string::npos : data.find('`', first_separator + 1); + if (second_separator != std::string::npos && + data.find('`', second_separator + 1) != std::string::npos) return false; + + const std::size_t url_end = first_separator == std::string::npos + ? data.size() : first_separator; + if (url_end == 0) return false; + if (url_end > DocumentParser::MAX_PARTIAL_URL_BYTES) { + limit_reason = TruncationReason::PARTIAL_DESCRIPTOR_BYTES; + return false; + } + partial.url = data.substr(0, url_end); + partial.descriptor = data; + std::string hash_input = partial.url; + + if (first_separator != std::string::npos) { + const std::size_t refresh_start = first_separator + 1; + const std::size_t refresh_size = (second_separator == std::string::npos + ? data.size() : second_separator) - refresh_start; + const std::string refresh = data.substr(refresh_start, refresh_size); + double seconds = 0.0; + if (!parse_python_decimal(refresh, seconds) || !std::isfinite(seconds)) return false; + if (seconds >= 1.0) { + const double milliseconds = seconds * 1000.0; + if (milliseconds > static_cast(DocumentParser::MAX_PARTIAL_REFRESH_MS)) + return false; + partial.refresh_interval_ms = static_cast(milliseconds); + } + hash_input += "|" + refresh; + } + + if (second_separator == std::string::npos) { + // Canonical split(\"|\") semantics retain one empty selector. + partial.fields.emplace_back(); + } else { + const std::size_t selectors_start = second_separator + 1; + const std::size_t selectors_size = data.size() - selectors_start; + if (selectors_size > DocumentParser::MAX_PARTIAL_FIELD_BYTES) { + limit_reason = TruncationReason::PARTIAL_FIELD_BYTES; + return false; + } + partial.selectors = data.substr(selectors_start, selectors_size); + hash_input += "|" + partial.selectors; + std::size_t field_start = 0; + while (field_start <= partial.selectors.size()) { + if (partial.fields.size() >= DocumentParser::MAX_PARTIAL_FIELDS) { + limit_reason = TruncationReason::PARTIAL_FIELDS; + return false; + } + const std::size_t field_end = partial.selectors.find('|', field_start); + const std::size_t field_size = (field_end == std::string::npos + ? partial.selectors.size() : field_end) - field_start; + if (field_size > DocumentParser::MAX_PARTIAL_FIELD_BYTES) { + limit_reason = TruncationReason::PARTIAL_FIELD_BYTES; + return false; + } + const std::string field = partial.selectors.substr(field_start, field_size); + partial.fields.push_back(field); + if (field.rfind("pid=", 0) == 0) { + const std::size_t value_end = field.find('=', 4); + const std::size_t id_size = (value_end == std::string::npos + ? field.size() : value_end) - 4; + if (id_size > DocumentParser::MAX_PARTIAL_ID_BYTES) { + limit_reason = TruncationReason::PARTIAL_FIELD_BYTES; + return false; + } + partial.id = field.substr(4, id_size); + } + if (field_end == std::string::npos) break; + field_start = field_end + 1; + } + } + partial.descriptor_hash = partial_descriptor_sha256( + hash_input.data(), hash_input.size()); + return true; +} + } // namespace Document DocumentParser::parse(const std::string& source) const { @@ -691,6 +946,7 @@ Document DocumentParser::parse(const char* source, std::size_t size) const { Alignment table_alignment = Alignment::LEFT; uint16_t table_width = DEFAULT_TABLE_WIDTH; std::size_t table_bytes = 0; + std::size_t partial_bytes = 0; std::vector table_lines; uint8_t section_depth = 0; std::size_t total_runs = 0; @@ -765,13 +1021,36 @@ Document DocumentParser::parse(const char* source, std::size_t size) const { continue; } if (line.empty()) continue; - bool pre_escaped_table_line = false; - if (!literal && line[0] == '\\' && - (table_mode || line.rfind("\\`t", 0) == 0)) { - line.erase(0, 1); - pre_escaped_table_line = true; + char classification_first_char = 0; + while (!line.empty()) { + if (!literal && line == "`=") { + literal = true; + line.clear(); + break; + } + classification_first_char = line[0]; + if (!literal && classification_first_char == '>' && + line.find("`<") != std::string::npos) { + const auto first_non_heading = line.find_first_not_of('>'); + line.erase(0, first_non_heading == std::string::npos + ? line.size() : first_non_heading); + if (line.empty()) break; + classification_first_char = line[0]; + } + if (!literal && !table_mode && classification_first_char == '<') { + section_depth = 0; + line.erase(0, 1); + continue; + } + break; } - if (!literal && !pre_escaped_table_line && !line.empty() && line[0] == '#') continue; + if (line.empty()) continue; + bool pre_escaped = false; + if (!literal && line[0] == '\\') { + line.erase(0, 1); + pre_escaped = true; + } + if (!literal && classification_first_char == '#') continue; if (!literal && line.rfind("`t", 0) == 0) { if (table_mode) { const bool canonical_table_rendered = table_lines.size() >= 2; @@ -824,17 +1103,6 @@ Document DocumentParser::parse(const char* source, std::size_t size) const { } continue; } - while (!line.empty() && line[0] == '<') { - section_depth = 0; - line.erase(0, 1); - } - if (line.empty()) continue; - if (!literal && line[0] == '>' && line.find("`<") != std::string::npos) { - const auto first_non_heading = line.find_first_not_of('>'); - line.erase(0, first_non_heading == std::string::npos - ? line.size() : first_non_heading); - if (line.empty()) continue; - } if (doc.blocks.size() >= MAX_BLOCKS) { doc.mark_truncated(TruncationReason::BLOCKS); break; @@ -847,7 +1115,7 @@ Document DocumentParser::parse(const char* source, std::size_t size) const { Run run; run.text = line; block.runs.push_back(std::move(run)); - } else if (line[0] == '>') { + } else if (classification_first_char == '>') { block.type = BlockType::HEADING; std::size_t depth = 0; while (depth < line.size() && line[depth] == '>') ++depth; @@ -856,7 +1124,7 @@ Document DocumentParser::parse(const char* source, std::size_t size) const { const std::string heading = line.substr(depth); parse_inline(doc, block, heading, style); add_anchor(doc, heading_slug(heading), doc.blocks.size()); - } else if (line[0] == '-') { + } else if (classification_first_char == '-') { block.type = BlockType::DIVIDER; uint32_t codepoint = 0; std::size_t codepoint_bytes = 0; @@ -865,12 +1133,70 @@ Document DocumentParser::parse(const char* source, std::size_t size) const { block.divider_codepoint = codepoint; } } else if (line.rfind("`{", 0) == 0) { - block.type = BlockType::UNSUPPORTED; - Run run; - run.text = "[Unsupported Micron content]"; - block.runs.push_back(std::move(run)); - doc.unsupported = true; + Partial partial; + TruncationReason partial_limit = TruncationReason::NONE; + if (parse_partial_descriptor(line, partial, partial_limit)) { + bool admitted = true; + if (doc.partials.size() >= MAX_PARTIALS) { + doc.mark_truncated(TruncationReason::PARTIALS); + admitted = false; + } + if (partial.descriptor.size() > MAX_PARTIAL_DESCRIPTOR_BYTES || + partial.url.size() > MAX_PARTIAL_URL_BYTES) { + doc.mark_truncated(TruncationReason::PARTIAL_DESCRIPTOR_BYTES); + admitted = false; + } + if (partial.fields.size() > MAX_PARTIAL_FIELDS) { + doc.mark_truncated(TruncationReason::PARTIAL_FIELDS); + admitted = false; + } + if (partial.selectors.size() > MAX_PARTIAL_FIELD_BYTES || + partial.id.size() > MAX_PARTIAL_ID_BYTES) { + doc.mark_truncated(TruncationReason::PARTIAL_FIELD_BYTES); + admitted = false; + } + for (const auto& field : partial.fields) { + if (field.size() > MAX_PARTIAL_FIELD_BYTES) { + doc.mark_truncated(TruncationReason::PARTIAL_FIELD_BYTES); + admitted = false; + break; + } + } + std::size_t bytes = partial.descriptor.size() + partial.url.size() + + partial.selectors.size() + partial.id.size() + partial.fields.size() + 4; + for (const auto& field : partial.fields) bytes += field.size(); + if (bytes > MAX_PARTIAL_BYTES - std::min(partial_bytes, MAX_PARTIAL_BYTES)) { + doc.mark_truncated(TruncationReason::PARTIAL_DESCRIPTOR_BYTES); + admitted = false; + } + if (admitted) { + partial_bytes += bytes; + block.type = BlockType::PARTIAL; + doc.partials.push_back(std::move(partial)); + block.partial_index = static_cast(doc.partials.size() - 1); + Run run; + run.text = "[Dynamic content loading]"; + block.runs.push_back(std::move(run)); + } else { + block.type = BlockType::UNSUPPORTED; + Run run; + run.text = "[Partial omitted: limits exceeded]"; + block.runs.push_back(std::move(run)); + } + } else { + block.type = BlockType::UNSUPPORTED; + Run run; + if (partial_limit != TruncationReason::NONE) { + doc.mark_truncated(partial_limit); + run.text = "[Partial omitted: limits exceeded]"; + } else { + run.text = "[Invalid Micron partial]"; + doc.malformed = true; + } + block.runs.push_back(std::move(run)); + } } else { + if (pre_escaped) line.insert(line.begin(), '\\'); parse_inline(doc, block, line, style); } const bool total_runs_exceeded = @@ -938,6 +1264,15 @@ std::string truncation_notice(const Document& document) { if (document.has_truncation(TruncationReason::TABLES)) return "[Page truncated: more than " + std::to_string(DocumentParser::MAX_TABLES) + " tables]"; + if (document.has_truncation(TruncationReason::PARTIALS)) + return "[Page truncated: more than " + + std::to_string(DocumentParser::MAX_PARTIALS) + " dynamic partials]"; + if (document.has_truncation(TruncationReason::PARTIAL_DESCRIPTOR_BYTES)) + return "[Page truncated: dynamic partial metadata exceeds limits]"; + if (document.has_truncation(TruncationReason::PARTIAL_FIELDS)) + return "[Page truncated: too many dynamic partial fields]"; + if (document.has_truncation(TruncationReason::PARTIAL_FIELD_BYTES)) + return "[Page truncated: dynamic partial field exceeds limits]"; if (document.has_truncation(TruncationReason::FORM_NAME_BYTES)) return "[Page truncated: form field name exceeds " + std::to_string(DocumentParser::MAX_FIELD_NAME_BYTES) + " bytes]"; diff --git a/lib/tdeck_ui/UI/LXMF/NomadNetDocument.h b/lib/tdeck_ui/UI/LXMF/NomadNetDocument.h index 10abc2d4..e3945dba 100644 --- a/lib/tdeck_ui/UI/LXMF/NomadNetDocument.h +++ b/lib/tdeck_ui/UI/LXMF/NomadNetDocument.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -7,12 +8,13 @@ namespace UI::LXMF::NomadNet { -enum class BlockType { TEXT, HEADING, DIVIDER, TABLE, UNSUPPORTED }; +enum class BlockType { TEXT, HEADING, DIVIDER, TABLE, PARTIAL, UNSUPPORTED }; enum class Alignment { LEFT, CENTER, RIGHT }; enum class FormFieldType : uint8_t { TEXT, PASSWORD, CHECKBOX, RADIO }; enum class ParseStatus : uint8_t { OK, INVALID_INPUT, ALLOCATION_FAILED }; enum class TruncationReason : uint32_t { + NONE = 0, DOCUMENT_BYTES = 1 << 0, SOURCE_LINES = 1 << 1, SOURCE_LINE_BYTES = 1 << 2, @@ -34,6 +36,10 @@ enum class TruncationReason : uint32_t { FORM_VALUE_BYTES = 1u << 18, FORM_LABEL_BYTES = 1u << 19, FORM_BYTES = 1u << 20, + PARTIALS = 1u << 21, + PARTIAL_DESCRIPTOR_BYTES = 1u << 22, + PARTIAL_FIELDS = 1u << 23, + PARTIAL_FIELD_BYTES = 1u << 24, }; struct Run { @@ -74,12 +80,23 @@ struct Table { uint16_t max_width = 100; }; +struct Partial { + std::string descriptor; + std::string url; + std::string selectors; + std::string id; + std::vector fields; + std::array descriptor_hash{}; + uint32_t refresh_interval_ms = 0; +}; + struct Block { BlockType type = BlockType::TEXT; uint8_t depth = 0; Alignment alignment = Alignment::LEFT; uint32_t divider_codepoint = 0x2500; int16_t table_index = -1; + int16_t partial_index = -1; std::vector runs; }; @@ -112,6 +129,7 @@ struct Document { std::vector table_cells; std::vector table_runs; std::vector fields; + std::vector partials; uint32_t cache_seconds = 12U * 60U * 60U; bool has_cache_directive = false; bool cache_directive_valid = true; @@ -161,6 +179,14 @@ public: static constexpr std::size_t MAX_FIELD_VALUE_BYTES = 512; static constexpr std::size_t MAX_FIELD_LABEL_BYTES = 256; static constexpr std::size_t MAX_FORM_BYTES = 16 * 1024; + static constexpr std::size_t MAX_PARTIALS = 16; + static constexpr std::size_t MAX_PARTIAL_DESCRIPTOR_BYTES = 1024; + static constexpr std::size_t MAX_PARTIAL_URL_BYTES = 512; + static constexpr std::size_t MAX_PARTIAL_FIELDS = 64; + static constexpr std::size_t MAX_PARTIAL_FIELD_BYTES = 511; + static constexpr std::size_t MAX_PARTIAL_ID_BYTES = 64; + static constexpr uint32_t MAX_PARTIAL_REFRESH_MS = 604800000U; + static constexpr std::size_t MAX_PARTIAL_BYTES = 8 * 1024; static constexpr uint16_t DEFAULT_FIELD_WIDTH = 24; static constexpr uint16_t MAX_FIELD_WIDTH = 256; static constexpr uint16_t DEFAULT_TABLE_WIDTH = 100; diff --git a/lib/tdeck_ui/UI/LXMF/NomadNetPartialHash.h b/lib/tdeck_ui/UI/LXMF/NomadNetPartialHash.h new file mode 100644 index 00000000..3790daba --- /dev/null +++ b/lib/tdeck_ui/UI/LXMF/NomadNetPartialHash.h @@ -0,0 +1,113 @@ +#pragma once + +#include +#include +#include +#include + +namespace UI::LXMF::NomadNet { +namespace PartialHashDetail { + +constexpr std::array ROUND_CONSTANTS{{ + 0x428a2f98U, 0x71374491U, 0xb5c0fbcfU, 0xe9b5dba5U, + 0x3956c25bU, 0x59f111f1U, 0x923f82a4U, 0xab1c5ed5U, + 0xd807aa98U, 0x12835b01U, 0x243185beU, 0x550c7dc3U, + 0x72be5d74U, 0x80deb1feU, 0x9bdc06a7U, 0xc19bf174U, + 0xe49b69c1U, 0xefbe4786U, 0x0fc19dc6U, 0x240ca1ccU, + 0x2de92c6fU, 0x4a7484aaU, 0x5cb0a9dcU, 0x76f988daU, + 0x983e5152U, 0xa831c66dU, 0xb00327c8U, 0xbf597fc7U, + 0xc6e00bf3U, 0xd5a79147U, 0x06ca6351U, 0x14292967U, + 0x27b70a85U, 0x2e1b2138U, 0x4d2c6dfcU, 0x53380d13U, + 0x650a7354U, 0x766a0abbU, 0x81c2c92eU, 0x92722c85U, + 0xa2bfe8a1U, 0xa81a664bU, 0xc24b8b70U, 0xc76c51a3U, + 0xd192e819U, 0xd6990624U, 0xf40e3585U, 0x106aa070U, + 0x19a4c116U, 0x1e376c08U, 0x2748774cU, 0x34b0bcb5U, + 0x391c0cb3U, 0x4ed8aa4aU, 0x5b9cca4fU, 0x682e6ff3U, + 0x748f82eeU, 0x78a5636fU, 0x84c87814U, 0x8cc70208U, + 0x90befffaU, 0xa4506cebU, 0xbef9a3f7U, 0xc67178f2U, +}}; + +inline uint32_t rotate_right(uint32_t value, uint8_t bits) noexcept { + return (value >> bits) | (value << (32U - bits)); +} + +inline void transform(const uint8_t block[64], + std::array& state) noexcept { + uint32_t words[64]{}; + for (std::size_t i = 0; i < 16; ++i) { + words[i] = (static_cast(block[i * 4]) << 24) | + (static_cast(block[i * 4 + 1]) << 16) | + (static_cast(block[i * 4 + 2]) << 8) | + static_cast(block[i * 4 + 3]); + } + for (std::size_t i = 16; i < 64; ++i) { + const uint32_t s0 = rotate_right(words[i - 15], 7) ^ + rotate_right(words[i - 15], 18) ^ (words[i - 15] >> 3); + const uint32_t s1 = rotate_right(words[i - 2], 17) ^ + rotate_right(words[i - 2], 19) ^ (words[i - 2] >> 10); + words[i] = words[i - 16] + s0 + words[i - 7] + s1; + } + + uint32_t a = state[0], b = state[1], c = state[2], d = state[3]; + uint32_t e = state[4], f = state[5], g = state[6], h = state[7]; + for (std::size_t i = 0; i < 64; ++i) { + const uint32_t s1 = rotate_right(e, 6) ^ rotate_right(e, 11) ^ + rotate_right(e, 25); + const uint32_t choice = (e & f) ^ ((~e) & g); + const uint32_t temp1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i]; + const uint32_t s0 = rotate_right(a, 2) ^ rotate_right(a, 13) ^ + rotate_right(a, 22); + const uint32_t majority = (a & b) ^ (a & c) ^ (b & c); + const uint32_t temp2 = s0 + majority; + h = g; + g = f; + f = e; + e = d + temp1; + d = c; + c = b; + b = a; + a = temp1 + temp2; + } + state[0] += a; state[1] += b; state[2] += c; state[3] += d; + state[4] += e; state[5] += f; state[6] += g; state[7] += h; +} + +} // namespace PartialHashDetail + +// Standalone so native parser harnesses and ESP32 builds use identical bytes +// without depending on a platform-specific crypto provider. +inline std::array partial_descriptor_sha256( + const char* data, std::size_t size) noexcept { + using namespace PartialHashDetail; + std::array state{{ + 0x6a09e667U, 0xbb67ae85U, 0x3c6ef372U, 0xa54ff53aU, + 0x510e527fU, 0x9b05688cU, 0x1f83d9abU, 0x5be0cd19U, + }}; + std::size_t offset = 0; + while (size - offset >= 64) { + transform(reinterpret_cast(data + offset), state); + offset += 64; + } + + uint8_t final_blocks[128]{}; + const std::size_t remainder = size - offset; + if (remainder != 0) std::memcpy(final_blocks, data + offset, remainder); + final_blocks[remainder] = 0x80; + const std::size_t final_size = remainder < 56 ? 64 : 128; + const uint64_t bit_length = static_cast(size) * 8U; + for (std::size_t i = 0; i < 8; ++i) + final_blocks[final_size - 1 - i] = static_cast(bit_length >> (i * 8)); + transform(final_blocks, state); + if (final_size == 128) transform(final_blocks + 64, state); + + std::array digest{}; + for (std::size_t i = 0; i < state.size(); ++i) { + digest[i * 4] = static_cast(state[i] >> 24); + digest[i * 4 + 1] = static_cast(state[i] >> 16); + digest[i * 4 + 2] = static_cast(state[i] >> 8); + digest[i * 4 + 3] = static_cast(state[i]); + } + return digest; +} + +} // namespace UI::LXMF::NomadNet diff --git a/lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.cpp b/lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.cpp new file mode 100644 index 00000000..fb1a9424 --- /dev/null +++ b/lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.cpp @@ -0,0 +1,134 @@ +#include "NomadNetPartialScheduler.h" + +#include + +namespace UI::LXMF::NomadNet { + +bool PartialScheduler::due(uint32_t now_ms, uint32_t due_at_ms) noexcept { + return static_cast(now_ms - due_at_ms) >= 0; +} + +uint32_t PartialScheduler::next_token() noexcept { + ++_token_sequence; + if (_token_sequence == 0) ++_token_sequence; + return _token_sequence; +} + +void PartialScheduler::configure(const CompactPage& page, + uint32_t page_generation, + uint32_t now_ms) noexcept { + _entries = {}; + _count = page_generation == 0 ? 0 : + std::min(page.partials().size(), _entries.size()); + _page_generation = page_generation; + _next_scan_index = 0; + _in_flight_token = 0; + _in_flight_index = 0; + for (std::size_t i = 0; i < _count; ++i) { + _entries[i].refresh_interval_ms = page.partials()[i].refresh_interval_ms; + _entries[i].descriptor_hash = page.partials()[i].descriptor_hash; + _entries[i].due_at_ms = now_ms; + _entries[i].pending = true; + } +} + +void PartialScheduler::configure(const Document& document, + uint32_t page_generation, + uint32_t now_ms) noexcept { + _entries = {}; + _count = page_generation == 0 ? 0 : + std::min(document.partials.size(), _entries.size()); + _page_generation = page_generation; + _next_scan_index = 0; + _in_flight_token = 0; + _in_flight_index = 0; + for (std::size_t i = 0; i < _count; ++i) { + _entries[i].refresh_interval_ms = document.partials[i].refresh_interval_ms; + _entries[i].descriptor_hash = document.partials[i].descriptor_hash; + _entries[i].due_at_ms = now_ms; + _entries[i].pending = true; + } +} + +void PartialScheduler::cancel(uint32_t page_generation) noexcept { + if (_page_generation != page_generation) return; + _entries = {}; + _count = 0; + _page_generation = 0; + _next_scan_index = 0; + _in_flight_token = 0; + _in_flight_index = 0; +} + +bool PartialScheduler::poll(uint32_t now_ms, bool browser_active, + bool owner_available, + PartialRequest& request) noexcept { + if (!browser_active || !owner_available || _page_generation == 0 || + _in_flight_token != 0) return false; + for (std::size_t offset = 0; offset < _count; ++offset) { + const std::size_t i = (_next_scan_index + offset) % _count; + Entry& entry = _entries[i]; + if (!entry.pending || entry.in_flight || !due(now_ms, entry.due_at_ms)) continue; + entry.pending = false; + entry.in_flight = true; + entry.started_at_ms = now_ms; + entry.request_token = next_token(); + ++entry.partial_generation; + if (entry.partial_generation == 0) ++entry.partial_generation; + _in_flight_token = entry.request_token; + _in_flight_index = static_cast(i); + request.partial_index = static_cast(i); + request.page_generation = _page_generation; + request.partial_generation = entry.partial_generation; + request.request_token = entry.request_token; + request.descriptor_hash = entry.descriptor_hash; + _next_scan_index = static_cast((i + 1U) % _count); + return true; + } + return false; +} + +bool PartialScheduler::complete(const PartialRequest& request, bool success, + uint32_t now_ms) noexcept { + if (_page_generation == 0 || request.page_generation != _page_generation || + request.partial_index >= _count || request.request_token == 0 || + request.request_token != _in_flight_token || + request.partial_index != _in_flight_index) return false; + Entry& entry = _entries[request.partial_index]; + if (!entry.in_flight || entry.request_token != request.request_token || + entry.partial_generation != request.partial_generation || + entry.descriptor_hash != request.descriptor_hash) return false; + + entry.in_flight = false; + entry.request_token = 0; + _in_flight_token = 0; + _in_flight_index = 0; + if (success) { + if (entry.refresh_interval_ms != 0) { + entry.due_at_ms = entry.started_at_ms + entry.refresh_interval_ms + 1U; + entry.pending = true; + } + return true; + } + + if (entry.refresh_interval_ms == 0) { + entry.pending = false; + return true; + } + entry.due_at_ms = now_ms + entry.refresh_interval_ms + 1U; + entry.pending = true; + return true; +} + +bool PartialScheduler::request_now(std::size_t partial_index, + uint32_t page_generation, + uint32_t now_ms) noexcept { + if (page_generation != _page_generation || partial_index >= _count || + _entries[partial_index].in_flight) return false; + Entry& entry = _entries[partial_index]; + entry.due_at_ms = now_ms; + entry.pending = true; + return true; +} + +} // namespace UI::LXMF::NomadNet diff --git a/lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.h b/lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.h new file mode 100644 index 00000000..7485ce65 --- /dev/null +++ b/lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.h @@ -0,0 +1,67 @@ +#pragma once + +#include "NomadNetCompactPage.h" + +#include +#include +#include + +namespace UI::LXMF::NomadNet { + +struct PartialRequest { + uint16_t partial_index = 0; + uint32_t page_generation = 0; + uint32_t partial_generation = 0; + uint32_t request_token = 0; + std::array descriptor_hash{}; +}; + +// Allocation-free, single-owner scheduler for bounded dynamic partial work. +// It never retains CompactPage pointers or response bytes. The caller owns +// transport and must validate completion through the returned generation/token. +class PartialScheduler { +public: + + void configure(const CompactPage& page, uint32_t page_generation, + uint32_t now_ms) noexcept; + void configure(const Document& document, uint32_t page_generation, + uint32_t now_ms) noexcept; + void cancel(uint32_t page_generation) noexcept; + + bool poll(uint32_t now_ms, bool browser_active, bool owner_available, + PartialRequest& request) noexcept; + bool complete(const PartialRequest& request, bool success, + uint32_t now_ms) noexcept; + bool request_now(std::size_t partial_index, uint32_t page_generation, + uint32_t now_ms) noexcept; + + bool empty() const noexcept { return _page_generation == 0 || _count == 0; } + bool in_flight() const noexcept { return _in_flight_token != 0; } + std::size_t size() const noexcept { return _count; } + uint32_t page_generation() const noexcept { return _page_generation; } + +private: + struct Entry { + uint32_t refresh_interval_ms = 0; + uint32_t due_at_ms = 0; + uint32_t started_at_ms = 0; + uint32_t request_token = 0; + uint32_t partial_generation = 0; + std::array descriptor_hash{}; + bool pending = false; + bool in_flight = false; + }; + + static bool due(uint32_t now_ms, uint32_t due_at_ms) noexcept; + uint32_t next_token() noexcept; + + std::array _entries{}; + std::size_t _count = 0; + uint32_t _page_generation = 0; + uint32_t _token_sequence = 0; + uint8_t _next_scan_index = 0; + uint32_t _in_flight_token = 0; + uint16_t _in_flight_index = 0; +}; + +} // namespace UI::LXMF::NomadNet diff --git a/lib/tdeck_ui/UI/LXMF/UIManager.cpp b/lib/tdeck_ui/UI/LXMF/UIManager.cpp index 4043d59f..14838069 100644 --- a/lib/tdeck_ui/UI/LXMF/UIManager.cpp +++ b/lib/tdeck_ui/UI/LXMF/UIManager.cpp @@ -2167,6 +2167,7 @@ bool UIManager::nomad_refresh_path_after_link_failure() { } uint32_t UIManager::nomad_advance_navigation_generation() { + _nomad_partial_scheduler.cancel(_nomad_navigation_generation); ++_nomad_navigation_generation; if (_nomad_navigation_generation == 0) ++_nomad_navigation_generation; return _nomad_navigation_generation; @@ -2271,7 +2272,7 @@ void UIManager::nomad_open(const std::string& address, bool add_history, if (local_result == NomadNet::LocalNavigationResult::APPLIED) { _nomad_url = std::move(next_url); _nomad_pending_scroll = -1; - nomad_advance_navigation_generation(); + // A fragment jump keeps the same published page and partials. return; } } catch (const std::bad_alloc&) { @@ -2553,6 +2554,8 @@ NomadNet::PageApplyResult UIManager::nomad_apply_page_document( _nomad_pending_history.clear(); return result; } + _nomad_partial_scheduler.configure( + document, _nomad_navigation_generation, millis()); if (library_changed) _nomad_library_dirty = true; _nomad_pending_scroll = -1; return result; diff --git a/lib/tdeck_ui/UI/LXMF/UIManager.h b/lib/tdeck_ui/UI/LXMF/UIManager.h index d647cb17..dda3067a 100644 --- a/lib/tdeck_ui/UI/LXMF/UIManager.h +++ b/lib/tdeck_ui/UI/LXMF/UIManager.h @@ -15,6 +15,7 @@ #include "NavigationStack.h" #include "NomadNetUrl.h" #include "NomadNetDocument.h" +#include "NomadNetPartialScheduler.h" #include "NomadNetProtocol.h" #include "NomadNetHistory.h" #include "NomadNetMailbox.h" @@ -411,6 +412,7 @@ private: NomadNet::Url _nomad_url; NomadNet::DocumentParser _nomad_parser; + NomadNet::PartialScheduler _nomad_partial_scheduler; NomadNet::ResponseBuffer _nomad_response; NomadNet::PageHistory _nomad_history; NomadNet::PageHistory::PendingOpen _nomad_pending_history; diff --git a/tests/native/nomadnet_x86_flow/CMakeLists.txt b/tests/native/nomadnet_x86_flow/CMakeLists.txt index 90e2ccb5..c1ff5367 100644 --- a/tests/native/nomadnet_x86_flow/CMakeLists.txt +++ b/tests/native/nomadnet_x86_flow/CMakeLists.txt @@ -27,8 +27,8 @@ target_compile_definitions(udp_interface PRIVATE set(PYXIS_NOMADNET_DIR "${CMAKE_CURRENT_LIST_DIR}/../../../lib/tdeck_ui/UI/LXMF") set(PYXIS_ROOT "${CMAKE_CURRENT_LIST_DIR}/../../..") -set(PYXIS_MANIFEST_BASE "3658c54cc76bdc16e7a514ff008f2cb70c69e192") -set(PYXIS_MANIFEST_BRANCH "feat/nomadnet-limits-observability") +set(PYXIS_MANIFEST_BASE "51e4b586c4c3867ae399f573557edda6f3b48a44") +set(PYXIS_MANIFEST_BRANCH "feat/nomadnet-partials-core") set(PYXIS_MANIFEST_MICRORETICULUM "cd0338e7fc07d3a7785a450656ba766491cbf6e8") execute_process(COMMAND git rev-parse HEAD WORKING_DIRECTORY "${RNS_SOURCE}" OUTPUT_VARIABLE _rns_commit OUTPUT_STRIP_TRAILING_WHITESPACE @@ -50,6 +50,9 @@ set(PYXIS_MANIFEST_FILES "lib/tdeck_ui/UI/LXMF/NomadNetDocument.h" "lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.cpp" "lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.h" + "lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.cpp" + "lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.h" + "lib/tdeck_ui/UI/LXMF/NomadNetPartialHash.h" "lib/tdeck_ui/UI/LXMF/NomadNetGlyphs.cpp" "lib/tdeck_ui/UI/LXMF/NomadNetGlyphs.h" "lib/tdeck_ui/UI/LXMF/NomadNetForm.cpp" @@ -73,6 +76,7 @@ add_executable(pyxis_nomadnet_x86_flow "${PYXIS_ROOT}/src/TCPClientInterface.cpp" "${PYXIS_NOMADNET_DIR}/NomadNetDocument.cpp" "${PYXIS_ROOT}/lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.cpp" + "${PYXIS_ROOT}/lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.cpp" "${PYXIS_ROOT}/lib/tdeck_ui/UI/LXMF/NomadNetForm.cpp" "${PYXIS_ROOT}/lib/tdeck_ui/UI/LXMF/NomadNetGlyphs.cpp" "${PYXIS_NOMADNET_DIR}/NomadNetLibrary.cpp" diff --git a/tests/native/nomadnet_x86_flow/run_flow.py b/tests/native/nomadnet_x86_flow/run_flow.py index ae015f5f..ef87ca4c 100644 --- a/tests/native/nomadnet_x86_flow/run_flow.py +++ b/tests/native/nomadnet_x86_flow/run_flow.py @@ -43,8 +43,8 @@ if os.environ.get("PYXIS_FLOW_SCENARIOS"): raise SystemExit("invalid PYXIS_FLOW_SCENARIOS") SCENARIOS = requested -MANIFEST_BASE = "3658c54cc76bdc16e7a514ff008f2cb70c69e192" -MANIFEST_BRANCH = "feat/nomadnet-limits-observability" +MANIFEST_BASE = "51e4b586c4c3867ae399f573557edda6f3b48a44" +MANIFEST_BRANCH = "feat/nomadnet-partials-core" MANIFEST_MICRORETICULUM = "cd0338e7fc07d3a7785a450656ba766491cbf6e8" MANIFEST_FILES = ( "tests/native/nomadnet_x86_flow/CMakeLists.txt", @@ -60,6 +60,9 @@ MANIFEST_FILES = ( "lib/tdeck_ui/UI/LXMF/NomadNetDocument.h", "lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.cpp", "lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.h", + "lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.cpp", + "lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.h", + "lib/tdeck_ui/UI/LXMF/NomadNetPartialHash.h", "lib/tdeck_ui/UI/LXMF/NomadNetGlyphs.cpp", "lib/tdeck_ui/UI/LXMF/NomadNetGlyphs.h", "lib/tdeck_ui/UI/LXMF/NomadNetForm.cpp", diff --git a/tests/native/test_nomadnet_partial_core.cpp b/tests/native/test_nomadnet_partial_core.cpp new file mode 100644 index 00000000..7c3ec4f1 --- /dev/null +++ b/tests/native/test_nomadnet_partial_core.cpp @@ -0,0 +1,314 @@ +#include +#include +#include +#include + +#include "NomadNetCompactPage.h" +#include "NomadNetDocument.h" +#include "NomadNetPartialScheduler.h" + +using UI::LXMF::NomadNet::BlockType; +using UI::LXMF::NomadNet::CompactPage; +using UI::LXMF::NomadNet::DocumentParser; +using UI::LXMF::NomadNet::PartialRequest; +using UI::LXMF::NomadNet::PartialScheduler; +using UI::LXMF::NomadNet::TruncationReason; + +int main() { + int failures = 0; + auto check = [&](const char* name, bool condition) { + if (!condition) { + std::cerr << "FAIL: " << name << '\n'; + ++failures; + } + }; + auto hex = [](const std::array& value) { + std::ostringstream output; + for (uint8_t byte : value) + output << std::hex << std::setw(2) << std::setfill('0') + << static_cast(byte); + return output.str(); + }; + + DocumentParser parser; + const auto document = parser.parse( + "before\n" + "`{f64a846313b874ee4a357040807f8c77:/page/hello.mu`10`pid=32|user_name}\n" + "after"); + + check("canonical partial descriptor is retained", + document.partials.size() == 1 && document.blocks.size() == 3 && + document.blocks[1].type == BlockType::PARTIAL && + document.blocks[1].partial_index == 0); + if (document.partials.size() == 1) { + const auto& partial = document.partials[0]; + check("partial URL is exact", + partial.url == "f64a846313b874ee4a357040807f8c77:/page/hello.mu"); + check("partial refresh is represented in milliseconds", + partial.refresh_interval_ms == 10000U); + check("partial ID and fields follow canonical components", + partial.id == "32" && partial.fields.size() == 2 && + partial.fields[0] == "pid=32" && partial.fields[1] == "user_name"); + check("partial selectors and canonical SHA-256 identity are retained", + partial.descriptor == + "f64a846313b874ee4a357040807f8c77:/page/hello.mu`10`pid=32|user_name" && + partial.selectors == "pid=32|user_name" && + hex(partial.descriptor_hash) == + "412d0e39b4703ab0f4b29f77158f252a66f560207ea5ea83882e270b8e68ea07"); + } + + const auto fractional = parser.parse( + "`{:relative.mu`.999}\n`{:timed.mu`1.25}"); + check("sub-second canonical refresh disables automatic scheduling", + fractional.partials.size() == 2 && + fractional.partials[0].refresh_interval_ms == 0U); + check("fractional canonical refresh retains millisecond precision", + fractional.partials.size() == 2 && + fractional.partials[1].refresh_interval_ms == 1250U); + check("absent selector component retains canonical empty selector", + fractional.partials[0].fields.size() == 1 && + fractional.partials[0].fields[0].empty() && + fractional.partials[1].fields.size() == 1 && + fractional.partials[1].fields[0].empty()); + + const auto escaped = parser.parse("\\`{:escaped.mu}"); + check("escaped canonical partial is still recognized", + escaped.partials.size() == 1 && escaped.partials[0].url == ":escaped.mu"); + const auto escaped_controls = parser.parse("\\-\n\\>heading\n\\`="); + check("escaped structural controls remain ordinary literal text", + escaped_controls.blocks.size() == 3 && + escaped_controls.blocks[0].type == BlockType::TEXT && + escaped_controls.blocks[0].runs[0].text == "-" && + escaped_controls.blocks[1].type == BlockType::TEXT && + escaped_controls.blocks[1].runs[0].text == ">heading" && + escaped_controls.blocks[2].type == BlockType::TEXT && + escaped_controls.blocks[2].runs[0].text == "`="); + + const auto sanitized_comment = parser.parse("># hidden `"); + check("heading field sanitization precedes comment classification", + sanitized_comment.blocks.empty() && sanitized_comment.fields.empty()); + const auto sanitized_table = parser.parse( + ">`t`\nA|B\nC|D\n`t"); + check("heading field sanitization precedes table command recognition", + sanitized_table.tables.size() == 1 && sanitized_table.fields.empty()); + const auto sanitized_reset = parser.parse( + ">Heading\n>"); + check("heading field sanitization precedes section reset", + sanitized_reset.blocks.size() == 2 && + sanitized_reset.blocks[1].depth == 0); + const auto sanitized_table_row = parser.parse( + "`t\n>Left|Right `\nBottom|Row\n`t"); + check("heading field sanitization occurs before table buffering", + !sanitized_table_row.table_runs.empty() && + sanitized_table_row.table_runs[0].text == "Left"); + const auto reset_comment = parser.parse("<# hidden"); + const auto reset_table = parser.parse("<`t\nA|B\nC|D\n`t"); + const auto reset_heading = parser.parse("<>Heading"); + const auto reset_divider = parser.parse("<-"); + check("section reset restarts canonical line classification", + reset_comment.blocks.empty() && + reset_table.tables.size() == 1 && + reset_heading.blocks.size() == 1 && + reset_heading.blocks[0].type == BlockType::HEADING && + reset_divider.blocks.size() == 1 && + reset_divider.blocks[0].type == BlockType::DIVIDER); + const auto reset_literal = parser.parse("<`=\n-\n`="); + check("section reset restarts literal-toggle classification", + reset_literal.blocks.size() == 1 && + reset_literal.blocks[0].type == BlockType::TEXT && + reset_literal.blocks[0].runs[0].text == "-" && + !reset_literal.malformed); + + const auto decimal_refresh = parser.parse( + "`{:space.mu`1.0 }\n`{:underscore.mu`1_0}\n`{:exponent.mu`1.e2}"); + check("canonical Python decimal refresh grammar is retained", + decimal_refresh.partials.size() == 3 && + decimal_refresh.partials[0].refresh_interval_ms == 1000U && + decimal_refresh.partials[1].refresh_interval_ms == 10000U && + decimal_refresh.partials[2].refresh_interval_ms == 100000U); + const auto unicode_refresh = parser.parse( + u8"`{:unicode.mu`\u00A0\u0661.\u0662\u00A0}"); + check("canonical Unicode decimal digits and outer whitespace are retained", + unicode_refresh.partials.size() == 1 && + unicode_refresh.partials[0].refresh_interval_ms == 1200U); + bool rejected_information_separators = true; + for (char control = 0x1c; control <= 0x1f; ++control) { + const std::string source = "`{:control.mu`" + std::string(1, control) + + "1.2" + std::string(1, control) + "}"; + const auto control_refresh = parser.parse(source); + rejected_information_separators = rejected_information_separators && + control_refresh.partials.empty() && control_refresh.malformed; + } + check("Python-float-rejected information separators remain invalid", + rejected_information_separators); + const auto hexadecimal_refresh = parser.parse("`{:hex.mu`0x1p1}"); + check("non-canonical C hexadecimal refresh is rejected", + hexadecimal_refresh.partials.empty() && hexadecimal_refresh.malformed); + + const auto canonical_vector = parser.parse("`{:/page/a.mu}TRAIL"); + check("first closing brace terminates the descriptor and canonical hash input", + canonical_vector.partials.size() == 1 && + canonical_vector.partials[0].descriptor == ":/page/a.mu" && + hex(canonical_vector.partials[0].descriptor_hash) == + "07216f28dfbd82d34e655ded06936a911a4ea2cc138b46c0dd5ccb12231f541c"); + const auto refresh_cap = parser.parse( + "`{:max.mu`604800}\n`{:too-long.mu`604800.001}\n`{:nan.mu`nan}"); + check("refresh interval is finite and bounded below INT32 wrap ambiguity", + refresh_cap.partials.size() == 1 && + refresh_cap.partials[0].refresh_interval_ms == + DocumentParser::MAX_PARTIAL_REFRESH_MS && refresh_cap.malformed); + + std::string too_many; + for (std::size_t index = 0; index <= DocumentParser::MAX_PARTIALS; ++index) + too_many += "`{:p" + std::to_string(index) + ".mu}\n"; + const auto capped = parser.parse(too_many); + check("peer-controlled partial count is capped", + capped.partials.size() == DocumentParser::MAX_PARTIALS && + capped.has_truncation(TruncationReason::PARTIALS)); + + const auto malformed = parser.parse("`{:broken.mu`not-a-number}"); + check("malformed refresh does not create a schedulable descriptor", + malformed.partials.empty() && malformed.malformed && + malformed.blocks.size() == 1 && + malformed.blocks[0].type == BlockType::UNSUPPORTED); + + const auto oversized_url = parser.parse( + "`{" + std::string(DocumentParser::MAX_PARTIAL_URL_BYTES + 1, 'u') + "}"); + check("partial URL bytes are capped", + oversized_url.partials.empty() && + oversized_url.has_truncation(TruncationReason::PARTIAL_DESCRIPTOR_BYTES)); + + std::string excessive_fields = "`{:fields.mu`1`"; + for (std::size_t index = 0; index <= DocumentParser::MAX_PARTIAL_FIELDS; ++index) { + if (index != 0) excessive_fields += '|'; + excessive_fields += "f" + std::to_string(index); + } + excessive_fields += '}'; + const auto fields_capped = parser.parse(excessive_fields); + check("partial field count is capped", + fields_capped.partials.empty() && + fields_capped.has_truncation(TruncationReason::PARTIAL_FIELDS)); + + const auto oversized_field = parser.parse( + "`{:field.mu`1`" + + std::string(DocumentParser::MAX_PARTIAL_FIELD_BYTES + 1, 'f') + "}"); + check("partial field bytes are capped", + oversized_field.partials.empty() && + oversized_field.has_truncation(TruncationReason::PARTIAL_FIELD_BYTES)); + + std::string total_metadata; + for (std::size_t index = 0; index < DocumentParser::MAX_PARTIALS; ++index) + total_metadata += "`{:" + std::string(256, static_cast('a' + index)) + "}\n"; + const auto metadata_capped = parser.parse(total_metadata); + check("aggregate partial metadata remains bounded", + metadata_capped.partials.size() < DocumentParser::MAX_PARTIALS && + metadata_capped.has_truncation(TruncationReason::PARTIAL_DESCRIPTOR_BYTES)); + + CompactPage compact; + check("partial descriptors survive compact-page assignment", compact.assign(document) && + compact.partials().size() == 1 && compact.blocks().size() == 3 && + compact.blocks()[1].partial_index == 0); + if (compact.partials().size() == 1) { + const auto& partial = compact.partials()[0]; + check("compact partial strings and fields remain exact", + compact.partial_url(partial) == + "f64a846313b874ee4a357040807f8c77:/page/hello.mu" && + compact.partial_selectors(partial) == "pid=32|user_name" && + hex(partial.descriptor_hash) == + "412d0e39b4703ab0f4b29f77158f252a66f560207ea5ea83882e270b8e68ea07" && + compact.partial_id(partial) == "32" && partial.field_count == 2 && + compact.partial_field(partial, 1) == "user_name"); + } + + const auto scheduled_document = parser.parse( + "`{:first.mu`10}\n`{:second.mu`20}"); + CompactPage scheduled_page; + check("scheduler fixture compacts", scheduled_page.assign(scheduled_document)); + PartialScheduler scheduler; + check("scheduler state has a fixed constrained-memory footprint", + sizeof(scheduler) <= 1024U); + scheduler.configure(scheduled_page, 7U, 1000U); + PartialRequest first{}; + PartialRequest blocked{}; + check("first partial is immediately due", + scheduler.poll(1000U, true, true, first) && first.partial_index == 0 && + first.page_generation == 7U && first.request_token != 0U); + check("only one browser request can be outstanding", + !scheduler.poll(1000U, true, true, blocked)); + check("matching completion is accepted", scheduler.complete(first, true, 1100U)); + PartialRequest second{}; + check("second initial partial is serialized after first", + scheduler.poll(1100U, true, true, second) && second.partial_index == 1); + check("second completion is accepted", scheduler.complete(second, true, 1200U)); + check("refresh is based on request-start time and canonical strict expiry", + !scheduler.poll(11000U, true, true, blocked) && + scheduler.poll(11001U, true, true, blocked) && blocked.partial_index == 0); + + PartialScheduler fair_scheduler; + const auto fair_page = parser.parse("`{:first.mu`1}\n`{:second.mu`1}"); + fair_scheduler.configure(fair_page, 77U, 0U); + PartialRequest fair_first; + PartialRequest fair_second; + check("overdue first partial cannot starve later due partials", [&] { + return fair_scheduler.poll(0U, true, true, fair_first) && + fair_first.partial_index == 0 && + fair_scheduler.complete(fair_first, true, 2000U) && + fair_scheduler.poll(2000U, true, true, fair_second) && + fair_second.partial_index == 1; + }()); + + scheduler.configure(scheduled_page, 8U, 20000U); + check("old-generation completion cannot mutate a new page", + !scheduler.complete(blocked, true, 20001U)); + check("hidden browser suppresses dispatch", + !scheduler.poll(20000U, false, true, blocked)); + check("busy browser owner suppresses dispatch", + !scheduler.poll(20000U, true, false, blocked)); + check("dispatch resumes once browser is active and idle", + scheduler.poll(20000U, true, true, blocked) && + blocked.page_generation == 8U); + scheduler.cancel(8U); + check("navigation cancellation revokes in-flight work", + !scheduler.complete(blocked, true, 20001U) && scheduler.empty()); + + const auto retry_document = parser.parse("`{:once.mu}"); + CompactPage retry_page; + check("retry fixture compacts", retry_page.assign(retry_document)); + scheduler.configure(retry_page, 9U, 0U); + PartialRequest retry{}; + check("one-shot partial starts immediately", scheduler.poll(0U, true, true, retry)); + const uint32_t first_partial_generation = retry.partial_generation; + check("failure is contained", scheduler.complete(retry, false, 1U)); + check("one-shot transfer failure has no automatic retry", + !scheduler.poll(0xffffffffU, true, true, retry)); + check("manual p-link style retry rearms the exact occurrence", + scheduler.request_now(0U, 9U, 5000U) && + scheduler.poll(5000U, true, true, retry) && + retry.partial_generation != first_partial_generation); + check("mismatched partial generation is rejected", [&] { + PartialRequest stale = retry; + --stale.partial_generation; + return !scheduler.complete(stale, true, 5001U); + }()); + check("mismatched descriptor identity is rejected", [&] { + PartialRequest stale = retry; + stale.descriptor_hash[0] ^= 0xffU; + return !scheduler.complete(stale, true, 5001U); + }()); + check("manual retry completion remains valid", scheduler.complete(retry, true, 5001U)); + + const auto wrap_document = parser.parse("`{:wrap.mu`5}"); + CompactPage wrap_page; + check("wrap fixture compacts", wrap_page.assign(wrap_document)); + scheduler.configure(wrap_page, 10U, 0xfffffff0U); + check("initial request remains due across clock wrap", + scheduler.poll(0xfffffff0U, true, true, retry)); + check("wrapped request failure is accepted", scheduler.complete(retry, false, 0xfffffff1U)); + check("wrap-safe deadline is not early", + !scheduler.poll(0x00001379U, true, true, retry)); + check("wrap-safe deadline becomes due exactly once", + scheduler.poll(0x0000137aU, true, true, retry)); + + if (failures == 0) std::cout << "partial core parser checks passed\n"; + return failures == 0 ? 0 : 1; +} diff --git a/tests/native/test_nomadnet_partial_core.py b/tests/native/test_nomadnet_partial_core.py new file mode 100644 index 00000000..5b113c05 --- /dev/null +++ b/tests/native/test_nomadnet_partial_core.py @@ -0,0 +1,58 @@ +import shutil +import subprocess +from pathlib import Path + +import pytest + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent.parent +INCLUDE = ROOT / "lib" / "tdeck_ui" / "UI" / "LXMF" +SOURCE = HERE / "test_nomadnet_partial_core.cpp" + + +def _cxx(): + for name in ("clang++", "g++"): + if shutil.which(name): + return name + pytest.skip("no C++ compiler found") + + +def test_nomadnet_partial_core_native(tmp_path): + binary = tmp_path / "test_nomadnet_partial_core" + command = [ + _cxx(), "-std=c++17", "-Wall", "-Wextra", "-Werror", + f"-I{INCLUDE}", str(SOURCE), + str(INCLUDE / "NomadNetDocument.cpp"), + str(INCLUDE / "NomadNetCompactPage.cpp"), + str(INCLUDE / "NomadNetGlyphs.cpp"), + str(INCLUDE / "NomadNetPartialScheduler.cpp"), + "-o", str(binary), + ] + compiled = subprocess.run(command, capture_output=True, text=True) + assert compiled.returncode == 0, compiled.stdout + compiled.stderr + result = subprocess.run([str(binary)], capture_output=True, text=True, timeout=30) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "partial core parser checks passed" + + +def test_partial_scheduler_is_generation_owned_but_transport_dispatch_is_deferred(): + manager_h = (INCLUDE / "UIManager.h").read_text() + manager_cpp = (INCLUDE / "UIManager.cpp").read_text() + assert "NomadNet::PartialScheduler _nomad_partial_scheduler;" in manager_h + advance = manager_cpp.split( + "uint32_t UIManager::nomad_advance_navigation_generation() {", 1 + )[1].split("\n}", 1)[0] + assert "_nomad_partial_scheduler.cancel(_nomad_navigation_generation);" in advance + publication = manager_cpp.split( + "NomadNet::PageApplyResult UIManager::nomad_apply_page_document(", 1 + )[1].split("void UIManager::nomad_update()", 1)[0] + assert publication.index("result != NomadNet::PageApplyResult::APPLIED") < publication.index( + "_nomad_partial_scheduler.configure(" + ) + # This increment owns only the bounded model and scheduler. Partial transport, + # response replacement, and form-map integration belong to the next increment. + assert "_nomad_partial_scheduler.poll(" not in manager_cpp + local_jump = manager_cpp.split( + "if (local_result == NomadNet::LocalNavigationResult::APPLIED)", 1 + )[1].split("return;", 1)[0] + assert "nomad_advance_navigation_generation()" not in local_jump