dandri/pyxis
1
0
Fork 0
mirror of https://github.com/torlando-tech/pyxis.git synced 2026-08-28 05:24:22 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat(nomadnet): add bounded partial core

Browse Source
This commit is contained in:
torlando-agent[bot]
2026-08-17 19:54:41 +00:00
parent 51e4b586c4
commit b6faec8104
13 changed files with 1205 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.cpp
+83
View File
@@ -43,6 +43,12 @@ bool CompactPage::assign(const Document& document) {
const std::size_t block_count = std::min(document.blocks.size(), block_limit);
const std::size_t link_count = std::min(document.links.size(), MAX_LINKS);
const std::size_t field_count = std::min(document.fields.size(), MAX_FIELDS);
const std::size_t partial_count = std::min(document.partials.size(), MAX_PARTIALS);
std::size_t partial_field_count = 0;
for (std::size_t i = 0; i < partial_count; ++i) {
partial_field_count += std::min(document.partials[i].fields.size(),
MAX_PARTIAL_FIELDS - std::min(partial_field_count, MAX_PARTIAL_FIELDS));
}
const std::size_t table_count = std::min(document.tables.size(), MAX_TABLES);
std::size_t table_cell_count = 0;
for (std::size_t i = 0; i < table_count; ++i) {
@@ -90,6 +96,14 @@ bool CompactPage::assign(const Document& document) {
if (bytes > MAX_ARENA_BYTES - std::min(arena_size, MAX_ARENA_BYTES)) return false;
arena_size += bytes;
}
for (std::size_t i = 0; i < partial_count; ++i) {
const auto& partial = document.partials[i];
std::size_t bytes = partial.descriptor.size() + partial.url.size() +
partial.selectors.size() + partial.id.size() + 4;
for (const auto& field : partial.fields) bytes += field.size() + 1;
if (bytes > MAX_ARENA_BYTES - std::min(arena_size, MAX_ARENA_BYTES)) return false;
arena_size += bytes;
}
std::size_t anchors_accounted = 0;
for (const auto& anchor : document.anchors) {
if (anchors_accounted >= anchor_count) break;
@@ -108,6 +122,8 @@ bool CompactPage::assign(const Document& document) {
_anchors.reserve(anchor_count);
_tables.reserve(table_count);
_table_cells.reserve(table_cell_count);
_partials.reserve(partial_count);
_partial_fields.reserve(partial_field_count);
for (std::size_t i = 0; i < link_count; ++i) {
LinkRecord link;
@@ -139,6 +155,33 @@ bool CompactPage::assign(const Document& document) {
_fields.push_back(field);
}
for (std::size_t i = 0; i < partial_count; ++i) {
const auto& source = document.partials[i];
PartialRecord partial;
partial.first_field = static_cast<uint32_t>(_partial_fields.size());
if (!append(source.descriptor, partial.descriptor_offset, partial.descriptor_length) ||
!append(source.url, partial.url_offset, partial.url_length) ||
!append(source.selectors, partial.selectors_offset, partial.selectors_length) ||
!append(source.id, partial.id_offset, partial.id_length)) {
clear();
return false;
}
partial.refresh_interval_ms = source.refresh_interval_ms;
partial.descriptor_hash = source.descriptor_hash;
for (const auto& source_field : source.fields) {
if (_partial_fields.size() >= MAX_PARTIAL_FIELDS ||
partial.field_count == std::numeric_limits<uint16_t>::max()) break;
PartialFieldRecord field;
if (!append(source_field, field.value_offset, field.value_length)) {
clear();
return false;
}
_partial_fields.push_back(field);
++partial.field_count;
}
_partials.push_back(partial);
}
for (const auto& source_anchor : document.anchors) {
if (_anchors.size() >= anchor_count) break;
if (source_anchor.block_index >= block_count ||
@@ -167,6 +210,9 @@ bool CompactPage::assign(const Document& document) {
block.table_index = source_block.table_index >= 0 &&
static_cast<std::size_t>(source_block.table_index) < table_count
? source_block.table_index : -1;
block.partial_index = source_block.partial_index >= 0 &&
static_cast<std::size_t>(source_block.partial_index) < partial_count
? source_block.partial_index : -1;
for (const auto& source_run : source_block.runs) {
if (_runs.size() >= run_limit || block.run_count == std::numeric_limits<uint16_t>::max()) {
_truncated = true;
@@ -264,6 +310,7 @@ bool CompactPage::assign(const Document& document) {
_truncated = _truncated || document.truncated || document.blocks.size() > block_count ||
document.links.size() > link_count || document.anchors.size() > anchor_count ||
document.fields.size() > field_count || document.tables.size() > table_count ||
document.partials.size() > partial_count ||
document.table_cells.size() > table_cell_count ||
document.table_runs.size() > table_run_count;
_unsupported = document.unsupported;
@@ -283,6 +330,8 @@ void CompactPage::clear() {
ExternalVector<TableRecord>().swap(_tables);
ExternalVector<TableCellRecord>().swap(_table_cells);
ExternalVector<FieldRecord>().swap(_fields);
ExternalVector<PartialRecord>().swap(_partials);
ExternalVector<PartialFieldRecord>().swap(_partial_fields);
_has_background = false;
_background = 0;
_has_foreground = false;
@@ -362,6 +411,40 @@ CompactPage::TextView CompactPage::field_label(std::size_t index) const {
return {_arena.data() + field.label_offset, field.label_length};
}
CompactPage::TextView CompactPage::partial_descriptor(const PartialRecord& partial) const {
if (partial.descriptor_offset > _arena.size() ||
partial.descriptor_length > _arena.size() - partial.descriptor_offset) return {};
return {_arena.data() + partial.descriptor_offset, partial.descriptor_length};
}
CompactPage::TextView CompactPage::partial_url(const PartialRecord& partial) const {
if (partial.url_offset > _arena.size() ||
partial.url_length > _arena.size() - partial.url_offset) return {};
return {_arena.data() + partial.url_offset, partial.url_length};
}
CompactPage::TextView CompactPage::partial_selectors(const PartialRecord& partial) const {
if (partial.selectors_offset > _arena.size() ||
partial.selectors_length > _arena.size() - partial.selectors_offset) return {};
return {_arena.data() + partial.selectors_offset, partial.selectors_length};
}
CompactPage::TextView CompactPage::partial_id(const PartialRecord& partial) const {
if (partial.id_offset > _arena.size() ||
partial.id_length > _arena.size() - partial.id_offset) return {};
return {_arena.data() + partial.id_offset, partial.id_length};
}
CompactPage::TextView CompactPage::partial_field(const PartialRecord& partial,
std::size_t index) const {
if (index >= partial.field_count || partial.first_field > _partial_fields.size() ||
index >= _partial_fields.size() - partial.first_field) return {};
const auto& field = _partial_fields[partial.first_field + index];
if (field.value_offset > _arena.size() ||
field.value_length > _arena.size() - field.value_offset) return {};
return {_arena.data() + field.value_offset, field.value_length};
}
bool CompactPage::find_anchor(const std::string& name, uint16_t& block_index) const {
if (name.size() > DocumentParser::MAX_ANCHOR_NAME_BYTES) return false;
for (const auto& anchor : _anchors) {
lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.h
+33
View File
@@ -98,6 +98,9 @@ public:
static constexpr std::size_t MAX_TABLES = DocumentParser::MAX_TABLES;
static constexpr std::size_t MAX_TABLE_CELLS = DocumentParser::MAX_TOTAL_TABLE_CELLS;
static constexpr std::size_t MAX_FIELDS = DocumentParser::MAX_FIELDS;
static constexpr std::size_t MAX_PARTIALS = DocumentParser::MAX_PARTIALS;
static constexpr std::size_t MAX_PARTIAL_FIELDS =
MAX_PARTIALS * DocumentParser::MAX_PARTIAL_FIELDS;
// Text runs, link targets, and anchor names originate in the bounded source.
// Link targets also remain visible in runs, while anchor declarations are
// zero-width. Account for both bounded copies and one terminator per record.
@@ -106,6 +109,7 @@ public:
DocumentParser::MAX_DOCUMENT_BYTES * 2 +
DocumentParser::MAX_FORM_BYTES + MAX_FIELDS * 3 +
MAX_ANCHORS * (DocumentParser::MAX_ANCHOR_NAME_BYTES + 1) +
DocumentParser::MAX_PARTIAL_BYTES +
MAX_RUNS + MAX_LINKS + MAX_NOTICE_BYTES + 1;
enum Style : uint8_t {
@@ -124,6 +128,7 @@ public:
Alignment alignment = Alignment::LEFT;
uint32_t divider_codepoint = 0x2500;
int16_t table_index = -1;
int16_t partial_index = -1;
};
struct RunRecord {
@@ -174,6 +179,26 @@ public:
bool masked = false;
};
struct PartialRecord {
uint32_t descriptor_offset = 0;
uint32_t url_offset = 0;
uint32_t selectors_offset = 0;
uint32_t id_offset = 0;
uint32_t first_field = 0;
uint16_t descriptor_length = 0;
uint16_t url_length = 0;
uint16_t selectors_length = 0;
uint16_t id_length = 0;
uint16_t field_count = 0;
uint32_t refresh_interval_ms = 0;
std::array<uint8_t, 32> descriptor_hash{};
};
struct PartialFieldRecord {
uint32_t value_offset = 0;
uint16_t value_length = 0;
};
struct TextView {
const char* value = nullptr;
std::size_t length = 0;
@@ -202,11 +227,17 @@ public:
const ExternalVector<TableRecord>& tables() const { return _tables; }
const ExternalVector<TableCellRecord>& table_cells() const { return _table_cells; }
const ExternalVector<FieldRecord>& fields() const { return _fields; }
const ExternalVector<PartialRecord>& partials() const { return _partials; }
TextView text(const RunRecord& run) const;
TextView target(std::size_t index) const;
TextView field_name(std::size_t index) const;
TextView field_value(std::size_t index) const;
TextView field_label(std::size_t index) const;
TextView partial_descriptor(const PartialRecord& partial) const;
TextView partial_url(const PartialRecord& partial) const;
TextView partial_selectors(const PartialRecord& partial) const;
TextView partial_id(const PartialRecord& partial) const;
TextView partial_field(const PartialRecord& partial, std::size_t index) const;
bool find_anchor(const std::string& name, uint16_t& block_index) const;
bool has_background() const { return _has_background; }
@@ -229,6 +260,8 @@ private:
ExternalVector<TableRecord> _tables;
ExternalVector<TableCellRecord> _table_cells;
ExternalVector<FieldRecord> _fields;
ExternalVector<PartialRecord> _partials;
ExternalVector<PartialFieldRecord> _partial_fields;
bool _has_background = false;
uint32_t _background = 0;
bool _has_foreground = false;
lib/tdeck_ui/UI/LXMF/NomadNetDocument.cpp
+359 -24
View File
@@ -1,8 +1,10 @@
#include "NomadNetDocument.h"
#include "NomadNetPartialHash.h"
#include <algorithm>
#include <array>
#include <cctype>
#include <cmath>
#include <cstdlib>
namespace UI::LXMF::NomadNet {
@@ -642,6 +644,259 @@ bool append_table(Document& doc, const std::vector<std::string>& lines,
return true;
}
struct DecimalDigitRange {
uint32_t first;
uint32_t last;
};
constexpr DecimalDigitRange PYTHON_DECIMAL_DIGIT_RANGES[] = {
{0x30U, 0x39U}, {0x660U, 0x669U}, {0x6F0U, 0x6F9U},
{0x7C0U, 0x7C9U}, {0x966U, 0x96FU}, {0x9E6U, 0x9EFU},
{0xA66U, 0xA6FU}, {0xAE6U, 0xAEFU}, {0xB66U, 0xB6FU},
{0xBE6U, 0xBEFU}, {0xC66U, 0xC6FU}, {0xCE6U, 0xCEFU},
{0xD66U, 0xD6FU}, {0xDE6U, 0xDEFU}, {0xE50U, 0xE59U},
{0xED0U, 0xED9U}, {0xF20U, 0xF29U}, {0x1040U, 0x1049U},
{0x1090U, 0x1099U}, {0x17E0U, 0x17E9U}, {0x1810U, 0x1819U},
{0x1946U, 0x194FU}, {0x19D0U, 0x19D9U}, {0x1A80U, 0x1A89U},
{0x1A90U, 0x1A99U}, {0x1B50U, 0x1B59U}, {0x1BB0U, 0x1BB9U},
{0x1C40U, 0x1C49U}, {0x1C50U, 0x1C59U}, {0xA620U, 0xA629U},
{0xA8D0U, 0xA8D9U}, {0xA900U, 0xA909U}, {0xA9D0U, 0xA9D9U},
{0xA9F0U, 0xA9F9U}, {0xAA50U, 0xAA59U}, {0xABF0U, 0xABF9U},
{0xFF10U, 0xFF19U}, {0x104A0U, 0x104A9U}, {0x10D30U, 0x10D39U},
{0x11066U, 0x1106FU}, {0x110F0U, 0x110F9U}, {0x11136U, 0x1113FU},
{0x111D0U, 0x111D9U}, {0x112F0U, 0x112F9U}, {0x11450U, 0x11459U},
{0x114D0U, 0x114D9U}, {0x11650U, 0x11659U}, {0x116C0U, 0x116C9U},
{0x11730U, 0x11739U}, {0x118E0U, 0x118E9U}, {0x11950U, 0x11959U},
{0x11C50U, 0x11C59U}, {0x11D50U, 0x11D59U}, {0x11DA0U, 0x11DA9U},
{0x11F50U, 0x11F59U}, {0x16A60U, 0x16A69U}, {0x16AC0U, 0x16AC9U},
{0x16B50U, 0x16B59U}, {0x1D7CEU, 0x1D7FFU}, {0x1E140U, 0x1E149U},
{0x1E2F0U, 0x1E2F9U}, {0x1E4F0U, 0x1E4F9U}, {0x1E950U, 0x1E959U},
{0x1FBF0U, 0x1FBF9U},
};
bool decode_utf8_codepoint(const std::string& input, std::size_t& position,
std::size_t end, uint32_t& codepoint) {
if (position >= end) return false;
const unsigned char lead = static_cast<unsigned char>(input[position]);
std::size_t length = 0;
if (lead < 0x80U) {
codepoint = lead;
length = 1;
} else if (lead >= 0xC2U && lead <= 0xDFU) {
codepoint = lead & 0x1FU;
length = 2;
} else if (lead >= 0xE0U && lead <= 0xEFU) {
codepoint = lead & 0x0FU;
length = 3;
} else if (lead >= 0xF0U && lead <= 0xF4U) {
codepoint = lead & 0x07U;
length = 4;
} else {
return false;
}
if (length > end - position) return false;
for (std::size_t i = 1; i < length; ++i) {
const unsigned char continuation = static_cast<unsigned char>(input[position + i]);
if ((continuation & 0xC0U) != 0x80U) return false;
codepoint = (codepoint << 6U) | (continuation & 0x3FU);
}
if ((length == 3 && codepoint < 0x800U) ||
(length == 4 && codepoint < 0x10000U) ||
(codepoint >= 0xD800U && codepoint <= 0xDFFFU) || codepoint > 0x10FFFFU)
return false;
position += length;
return true;
}
bool decode_python_decimal_digit(const std::string& input, std::size_t& position,
std::size_t end, uint8_t& digit) {
std::size_t after_codepoint = position;
uint32_t codepoint = 0;
if (!decode_utf8_codepoint(input, after_codepoint, end, codepoint)) return false;
for (const auto& range : PYTHON_DECIMAL_DIGIT_RANGES) {
if (codepoint >= range.first && codepoint <= range.last) {
digit = static_cast<uint8_t>((codepoint - range.first) % 10U);
position = after_codepoint;
return true;
}
}
return false;
}
bool python_float_whitespace(uint32_t codepoint) {
return (codepoint >= 0x09U && codepoint <= 0x0DU) ||
codepoint == 0x20U || codepoint == 0x85U || codepoint == 0xA0U ||
codepoint == 0x1680U ||
(codepoint >= 0x2000U && codepoint <= 0x200AU) ||
codepoint == 0x2028U || codepoint == 0x2029U || codepoint == 0x202FU ||
codepoint == 0x205FU || codepoint == 0x3000U;
}
bool parse_python_decimal(const std::string& input, double& value) {
std::string canonical;
canonical.reserve(input.size());
for (std::size_t position = 0; position < input.size();) {
std::size_t after_digit = position;
uint8_t digit = 0;
if (decode_python_decimal_digit(input, after_digit, input.size(), digit)) {
canonical.push_back(static_cast<char>('0' + digit));
position = after_digit;
continue;
}
uint32_t codepoint = 0;
std::size_t after_codepoint = position;
if (!decode_utf8_codepoint(input, after_codepoint, input.size(), codepoint)) return false;
if (python_float_whitespace(codepoint)) canonical.push_back(' ');
else if (codepoint <= 0x7FU) canonical.push_back(static_cast<char>(codepoint));
else return false;
position = after_codepoint;
}
std::size_t begin = 0;
std::size_t end = canonical.size();
while (begin < end && canonical[begin] == ' ') ++begin;
while (end > begin && canonical[end - 1] == ' ') --end;
if (begin == end) return false;
std::size_t cursor = begin;
if (canonical[cursor] == '+' || canonical[cursor] == '-') ++cursor;
if (cursor == end) return false;
auto consume_digits = [&](std::size_t& position) {
bool any = false;
bool previous_digit = false;
while (position < end) {
const unsigned char current = static_cast<unsigned char>(canonical[position]);
if (std::isdigit(current)) {
any = true;
previous_digit = true;
++position;
} else if (canonical[position] == '_' && previous_digit &&
position + 1 < end) {
if (!std::isdigit(static_cast<unsigned char>(canonical[position + 1]))) break;
previous_digit = false;
++position;
} else {
break;
}
}
return any && previous_digit;
};
const bool integer_digits = consume_digits(cursor);
bool fraction_digits = false;
if (cursor < end && canonical[cursor] == '.') {
++cursor;
fraction_digits = consume_digits(cursor);
}
if (!integer_digits && !fraction_digits) return false;
if (cursor < end && (canonical[cursor] == 'e' || canonical[cursor] == 'E')) {
++cursor;
if (cursor < end && (canonical[cursor] == '+' || canonical[cursor] == '-')) ++cursor;
if (!consume_digits(cursor)) return false;
}
if (cursor != end) return false;
std::string normalized;
normalized.reserve(end - begin);
for (std::size_t i = begin; i < end; ++i) {
if (canonical[i] != '_') normalized.push_back(canonical[i]);
}
char* parse_end = nullptr;
value = std::strtod(normalized.c_str(), &parse_end);
return parse_end && parse_end != normalized.c_str() && *parse_end == '\0';
}
bool parse_partial_descriptor(const std::string& line, Partial& partial,
TruncationReason& limit_reason) {
limit_reason = TruncationReason::NONE;
const std::size_t close = line.find('}', 2);
if (line.rfind("`{", 0) != 0 || close == std::string::npos) return false;
const std::size_t data_size = close - 2;
if (data_size > DocumentParser::MAX_PARTIAL_DESCRIPTOR_BYTES) {
limit_reason = TruncationReason::PARTIAL_DESCRIPTOR_BYTES;
return false;
}
const std::string data = line.substr(2, data_size);
const std::size_t first_separator = data.find('`');
const std::size_t second_separator = first_separator == std::string::npos
? std::string::npos : data.find('`', first_separator + 1);
if (second_separator != std::string::npos &&
data.find('`', second_separator + 1) != std::string::npos) return false;
const std::size_t url_end = first_separator == std::string::npos
? data.size() : first_separator;
if (url_end == 0) return false;
if (url_end > DocumentParser::MAX_PARTIAL_URL_BYTES) {
limit_reason = TruncationReason::PARTIAL_DESCRIPTOR_BYTES;
return false;
}
partial.url = data.substr(0, url_end);
partial.descriptor = data;
std::string hash_input = partial.url;
if (first_separator != std::string::npos) {
const std::size_t refresh_start = first_separator + 1;
const std::size_t refresh_size = (second_separator == std::string::npos
? data.size() : second_separator) - refresh_start;
const std::string refresh = data.substr(refresh_start, refresh_size);
double seconds = 0.0;
if (!parse_python_decimal(refresh, seconds) || !std::isfinite(seconds)) return false;
if (seconds >= 1.0) {
const double milliseconds = seconds * 1000.0;
if (milliseconds > static_cast<double>(DocumentParser::MAX_PARTIAL_REFRESH_MS))
return false;
partial.refresh_interval_ms = static_cast<uint32_t>(milliseconds);
}
hash_input += "|" + refresh;
}
if (second_separator == std::string::npos) {
// Canonical split(\"|\") semantics retain one empty selector.
partial.fields.emplace_back();
} else {
const std::size_t selectors_start = second_separator + 1;
const std::size_t selectors_size = data.size() - selectors_start;
if (selectors_size > DocumentParser::MAX_PARTIAL_FIELD_BYTES) {
limit_reason = TruncationReason::PARTIAL_FIELD_BYTES;
return false;
}
partial.selectors = data.substr(selectors_start, selectors_size);
hash_input += "|" + partial.selectors;
std::size_t field_start = 0;
while (field_start <= partial.selectors.size()) {
if (partial.fields.size() >= DocumentParser::MAX_PARTIAL_FIELDS) {
limit_reason = TruncationReason::PARTIAL_FIELDS;
return false;
}
const std::size_t field_end = partial.selectors.find('|', field_start);
const std::size_t field_size = (field_end == std::string::npos
? partial.selectors.size() : field_end) - field_start;
if (field_size > DocumentParser::MAX_PARTIAL_FIELD_BYTES) {
limit_reason = TruncationReason::PARTIAL_FIELD_BYTES;
return false;
}
const std::string field = partial.selectors.substr(field_start, field_size);
partial.fields.push_back(field);
if (field.rfind("pid=", 0) == 0) {
const std::size_t value_end = field.find('=', 4);
const std::size_t id_size = (value_end == std::string::npos
? field.size() : value_end) - 4;
if (id_size > DocumentParser::MAX_PARTIAL_ID_BYTES) {
limit_reason = TruncationReason::PARTIAL_FIELD_BYTES;
return false;
}
partial.id = field.substr(4, id_size);
}
if (field_end == std::string::npos) break;
field_start = field_end + 1;
}
}
partial.descriptor_hash = partial_descriptor_sha256(
hash_input.data(), hash_input.size());
return true;
}
} // namespace
Document DocumentParser::parse(const std::string& source) const {
@@ -691,6 +946,7 @@ Document DocumentParser::parse(const char* source, std::size_t size) const {
Alignment table_alignment = Alignment::LEFT;
uint16_t table_width = DEFAULT_TABLE_WIDTH;
std::size_t table_bytes = 0;
std::size_t partial_bytes = 0;
std::vector<std::string> table_lines;
uint8_t section_depth = 0;
std::size_t total_runs = 0;
@@ -765,13 +1021,36 @@ Document DocumentParser::parse(const char* source, std::size_t size) const {
continue;
}
if (line.empty()) continue;
bool pre_escaped_table_line = false;
if (!literal && line[0] == '\\' &&
(table_mode || line.rfind("\\`t", 0) == 0)) {
line.erase(0, 1);
pre_escaped_table_line = true;
char classification_first_char = 0;
while (!line.empty()) {
if (!literal && line == "`=") {
literal = true;
line.clear();
break;
}
classification_first_char = line[0];
if (!literal && classification_first_char == '>' &&
line.find("`<") != std::string::npos) {
const auto first_non_heading = line.find_first_not_of('>');
line.erase(0, first_non_heading == std::string::npos
? line.size() : first_non_heading);
if (line.empty()) break;
classification_first_char = line[0];
}
if (!literal && !table_mode && classification_first_char == '<') {
section_depth = 0;
line.erase(0, 1);
continue;
}
break;
}
if (!literal && !pre_escaped_table_line && !line.empty() && line[0] == '#') continue;
if (line.empty()) continue;
bool pre_escaped = false;
if (!literal && line[0] == '\\') {
line.erase(0, 1);
pre_escaped = true;
}
if (!literal && classification_first_char == '#') continue;
if (!literal && line.rfind("`t", 0) == 0) {
if (table_mode) {
const bool canonical_table_rendered = table_lines.size() >= 2;
@@ -824,17 +1103,6 @@ Document DocumentParser::parse(const char* source, std::size_t size) const {
}
continue;
}
while (!line.empty() && line[0] == '<') {
section_depth = 0;
line.erase(0, 1);
}
if (line.empty()) continue;
if (!literal && line[0] == '>' && line.find("`<") != std::string::npos) {
const auto first_non_heading = line.find_first_not_of('>');
line.erase(0, first_non_heading == std::string::npos
? line.size() : first_non_heading);
if (line.empty()) continue;
}
if (doc.blocks.size() >= MAX_BLOCKS) {
doc.mark_truncated(TruncationReason::BLOCKS);
break;
@@ -847,7 +1115,7 @@ Document DocumentParser::parse(const char* source, std::size_t size) const {
Run run;
run.text = line;
block.runs.push_back(std::move(run));
} else if (line[0] == '>') {
} else if (classification_first_char == '>') {
block.type = BlockType::HEADING;
std::size_t depth = 0;
while (depth < line.size() && line[depth] == '>') ++depth;
@@ -856,7 +1124,7 @@ Document DocumentParser::parse(const char* source, std::size_t size) const {
const std::string heading = line.substr(depth);
parse_inline(doc, block, heading, style);
add_anchor(doc, heading_slug(heading), doc.blocks.size());
} else if (line[0] == '-') {
} else if (classification_first_char == '-') {
block.type = BlockType::DIVIDER;
uint32_t codepoint = 0;
std::size_t codepoint_bytes = 0;
@@ -865,12 +1133,70 @@ Document DocumentParser::parse(const char* source, std::size_t size) const {
block.divider_codepoint = codepoint;
}
} else if (line.rfind("`{", 0) == 0) {
block.type = BlockType::UNSUPPORTED;
Run run;
run.text = "[Unsupported Micron content]";
block.runs.push_back(std::move(run));
doc.unsupported = true;
Partial partial;
TruncationReason partial_limit = TruncationReason::NONE;
if (parse_partial_descriptor(line, partial, partial_limit)) {
bool admitted = true;
if (doc.partials.size() >= MAX_PARTIALS) {
doc.mark_truncated(TruncationReason::PARTIALS);
admitted = false;
}
if (partial.descriptor.size() > MAX_PARTIAL_DESCRIPTOR_BYTES ||
partial.url.size() > MAX_PARTIAL_URL_BYTES) {
doc.mark_truncated(TruncationReason::PARTIAL_DESCRIPTOR_BYTES);
admitted = false;
}
if (partial.fields.size() > MAX_PARTIAL_FIELDS) {
doc.mark_truncated(TruncationReason::PARTIAL_FIELDS);
admitted = false;
}
if (partial.selectors.size() > MAX_PARTIAL_FIELD_BYTES ||
partial.id.size() > MAX_PARTIAL_ID_BYTES) {
doc.mark_truncated(TruncationReason::PARTIAL_FIELD_BYTES);
admitted = false;
}
for (const auto& field : partial.fields) {
if (field.size() > MAX_PARTIAL_FIELD_BYTES) {
doc.mark_truncated(TruncationReason::PARTIAL_FIELD_BYTES);
admitted = false;
break;
}
}
std::size_t bytes = partial.descriptor.size() + partial.url.size() +
partial.selectors.size() + partial.id.size() + partial.fields.size() + 4;
for (const auto& field : partial.fields) bytes += field.size();
if (bytes > MAX_PARTIAL_BYTES - std::min(partial_bytes, MAX_PARTIAL_BYTES)) {
doc.mark_truncated(TruncationReason::PARTIAL_DESCRIPTOR_BYTES);
admitted = false;
}
if (admitted) {
partial_bytes += bytes;
block.type = BlockType::PARTIAL;
doc.partials.push_back(std::move(partial));
block.partial_index = static_cast<int16_t>(doc.partials.size() - 1);
Run run;
run.text = "[Dynamic content loading]";
block.runs.push_back(std::move(run));
} else {
block.type = BlockType::UNSUPPORTED;
Run run;
run.text = "[Partial omitted: limits exceeded]";
block.runs.push_back(std::move(run));
}
} else {
block.type = BlockType::UNSUPPORTED;
Run run;
if (partial_limit != TruncationReason::NONE) {
doc.mark_truncated(partial_limit);
run.text = "[Partial omitted: limits exceeded]";
} else {
run.text = "[Invalid Micron partial]";
doc.malformed = true;
}
block.runs.push_back(std::move(run));
}
} else {
if (pre_escaped) line.insert(line.begin(), '\\');
parse_inline(doc, block, line, style);
}
const bool total_runs_exceeded =
@@ -938,6 +1264,15 @@ std::string truncation_notice(const Document& document) {
if (document.has_truncation(TruncationReason::TABLES))
return "[Page truncated: more than " +
std::to_string(DocumentParser::MAX_TABLES) + " tables]";
if (document.has_truncation(TruncationReason::PARTIALS))
return "[Page truncated: more than " +
std::to_string(DocumentParser::MAX_PARTIALS) + " dynamic partials]";
if (document.has_truncation(TruncationReason::PARTIAL_DESCRIPTOR_BYTES))
return "[Page truncated: dynamic partial metadata exceeds limits]";
if (document.has_truncation(TruncationReason::PARTIAL_FIELDS))
return "[Page truncated: too many dynamic partial fields]";
if (document.has_truncation(TruncationReason::PARTIAL_FIELD_BYTES))
return "[Page truncated: dynamic partial field exceeds limits]";
if (document.has_truncation(TruncationReason::FORM_NAME_BYTES))
return "[Page truncated: form field name exceeds " +
std::to_string(DocumentParser::MAX_FIELD_NAME_BYTES) + " bytes]";
lib/tdeck_ui/UI/LXMF/NomadNetDocument.h
+27 -1
View File
@@ -1,5 +1,6 @@
#pragma once
#include <array>
#include <cstddef>
#include <cstdint>
#include <string>
@@ -7,12 +8,13 @@
namespace UI::LXMF::NomadNet {
enum class BlockType { TEXT, HEADING, DIVIDER, TABLE, UNSUPPORTED };
enum class BlockType { TEXT, HEADING, DIVIDER, TABLE, PARTIAL, UNSUPPORTED };
enum class Alignment { LEFT, CENTER, RIGHT };
enum class FormFieldType : uint8_t { TEXT, PASSWORD, CHECKBOX, RADIO };
enum class ParseStatus : uint8_t { OK, INVALID_INPUT, ALLOCATION_FAILED };
enum class TruncationReason : uint32_t {
NONE = 0,
DOCUMENT_BYTES = 1 << 0,
SOURCE_LINES = 1 << 1,
SOURCE_LINE_BYTES = 1 << 2,
@@ -34,6 +36,10 @@ enum class TruncationReason : uint32_t {
FORM_VALUE_BYTES = 1u << 18,
FORM_LABEL_BYTES = 1u << 19,
FORM_BYTES = 1u << 20,
PARTIALS = 1u << 21,
PARTIAL_DESCRIPTOR_BYTES = 1u << 22,
PARTIAL_FIELDS = 1u << 23,
PARTIAL_FIELD_BYTES = 1u << 24,
};
struct Run {
@@ -74,12 +80,23 @@ struct Table {
uint16_t max_width = 100;
};
struct Partial {
std::string descriptor;
std::string url;
std::string selectors;
std::string id;
std::vector<std::string> fields;
std::array<uint8_t, 32> descriptor_hash{};
uint32_t refresh_interval_ms = 0;
};
struct Block {
BlockType type = BlockType::TEXT;
uint8_t depth = 0;
Alignment alignment = Alignment::LEFT;
uint32_t divider_codepoint = 0x2500;
int16_t table_index = -1;
int16_t partial_index = -1;
std::vector<Run> runs;
};
@@ -112,6 +129,7 @@ struct Document {
std::vector<TableCell> table_cells;
std::vector<Run> table_runs;
std::vector<FormField> fields;
std::vector<Partial> partials;
uint32_t cache_seconds = 12U * 60U * 60U;
bool has_cache_directive = false;
bool cache_directive_valid = true;
@@ -161,6 +179,14 @@ public:
static constexpr std::size_t MAX_FIELD_VALUE_BYTES = 512;
static constexpr std::size_t MAX_FIELD_LABEL_BYTES = 256;
static constexpr std::size_t MAX_FORM_BYTES = 16 * 1024;
static constexpr std::size_t MAX_PARTIALS = 16;
static constexpr std::size_t MAX_PARTIAL_DESCRIPTOR_BYTES = 1024;
static constexpr std::size_t MAX_PARTIAL_URL_BYTES = 512;
static constexpr std::size_t MAX_PARTIAL_FIELDS = 64;
static constexpr std::size_t MAX_PARTIAL_FIELD_BYTES = 511;
static constexpr std::size_t MAX_PARTIAL_ID_BYTES = 64;
static constexpr uint32_t MAX_PARTIAL_REFRESH_MS = 604800000U;
static constexpr std::size_t MAX_PARTIAL_BYTES = 8 * 1024;
static constexpr uint16_t DEFAULT_FIELD_WIDTH = 24;
static constexpr uint16_t MAX_FIELD_WIDTH = 256;
static constexpr uint16_t DEFAULT_TABLE_WIDTH = 100;
lib/tdeck_ui/UI/LXMF/NomadNetPartialHash.h
+113
View File
@@ -0,0 +1,113 @@
#pragma once
#include <array>
#include <cstddef>
#include <cstdint>
#include <cstring>
namespace UI::LXMF::NomadNet {
namespace PartialHashDetail {
constexpr std::array<uint32_t, 64> ROUND_CONSTANTS{{
0x428a2f98U, 0x71374491U, 0xb5c0fbcfU, 0xe9b5dba5U,
0x3956c25bU, 0x59f111f1U, 0x923f82a4U, 0xab1c5ed5U,
0xd807aa98U, 0x12835b01U, 0x243185beU, 0x550c7dc3U,
0x72be5d74U, 0x80deb1feU, 0x9bdc06a7U, 0xc19bf174U,
0xe49b69c1U, 0xefbe4786U, 0x0fc19dc6U, 0x240ca1ccU,
0x2de92c6fU, 0x4a7484aaU, 0x5cb0a9dcU, 0x76f988daU,
0x983e5152U, 0xa831c66dU, 0xb00327c8U, 0xbf597fc7U,
0xc6e00bf3U, 0xd5a79147U, 0x06ca6351U, 0x14292967U,
0x27b70a85U, 0x2e1b2138U, 0x4d2c6dfcU, 0x53380d13U,
0x650a7354U, 0x766a0abbU, 0x81c2c92eU, 0x92722c85U,
0xa2bfe8a1U, 0xa81a664bU, 0xc24b8b70U, 0xc76c51a3U,
0xd192e819U, 0xd6990624U, 0xf40e3585U, 0x106aa070U,
0x19a4c116U, 0x1e376c08U, 0x2748774cU, 0x34b0bcb5U,
0x391c0cb3U, 0x4ed8aa4aU, 0x5b9cca4fU, 0x682e6ff3U,
0x748f82eeU, 0x78a5636fU, 0x84c87814U, 0x8cc70208U,
0x90befffaU, 0xa4506cebU, 0xbef9a3f7U, 0xc67178f2U,
}};
inline uint32_t rotate_right(uint32_t value, uint8_t bits) noexcept {
return (value >> bits) | (value << (32U - bits));
}
inline void transform(const uint8_t block[64],
std::array<uint32_t, 8>& state) noexcept {
uint32_t words[64]{};
for (std::size_t i = 0; i < 16; ++i) {
words[i] = (static_cast<uint32_t>(block[i * 4]) << 24) |
(static_cast<uint32_t>(block[i * 4 + 1]) << 16) |
(static_cast<uint32_t>(block[i * 4 + 2]) << 8) |
static_cast<uint32_t>(block[i * 4 + 3]);
}
for (std::size_t i = 16; i < 64; ++i) {
const uint32_t s0 = rotate_right(words[i - 15], 7) ^
rotate_right(words[i - 15], 18) ^ (words[i - 15] >> 3);
const uint32_t s1 = rotate_right(words[i - 2], 17) ^
rotate_right(words[i - 2], 19) ^ (words[i - 2] >> 10);
words[i] = words[i - 16] + s0 + words[i - 7] + s1;
}
uint32_t a = state[0], b = state[1], c = state[2], d = state[3];
uint32_t e = state[4], f = state[5], g = state[6], h = state[7];
for (std::size_t i = 0; i < 64; ++i) {
const uint32_t s1 = rotate_right(e, 6) ^ rotate_right(e, 11) ^
rotate_right(e, 25);
const uint32_t choice = (e & f) ^ ((~e) & g);
const uint32_t temp1 = h + s1 + choice + ROUND_CONSTANTS[i] + words[i];
const uint32_t s0 = rotate_right(a, 2) ^ rotate_right(a, 13) ^
rotate_right(a, 22);
const uint32_t majority = (a & b) ^ (a & c) ^ (b & c);
const uint32_t temp2 = s0 + majority;
h = g;
g = f;
f = e;
e = d + temp1;
d = c;
c = b;
b = a;
a = temp1 + temp2;
}
state[0] += a; state[1] += b; state[2] += c; state[3] += d;
state[4] += e; state[5] += f; state[6] += g; state[7] += h;
}
} // namespace PartialHashDetail
// Standalone so native parser harnesses and ESP32 builds use identical bytes
// without depending on a platform-specific crypto provider.
inline std::array<uint8_t, 32> partial_descriptor_sha256(
const char* data, std::size_t size) noexcept {
using namespace PartialHashDetail;
std::array<uint32_t, 8> state{{
0x6a09e667U, 0xbb67ae85U, 0x3c6ef372U, 0xa54ff53aU,
0x510e527fU, 0x9b05688cU, 0x1f83d9abU, 0x5be0cd19U,
}};
std::size_t offset = 0;
while (size - offset >= 64) {
transform(reinterpret_cast<const uint8_t*>(data + offset), state);
offset += 64;
}
uint8_t final_blocks[128]{};
const std::size_t remainder = size - offset;
if (remainder != 0) std::memcpy(final_blocks, data + offset, remainder);
final_blocks[remainder] = 0x80;
const std::size_t final_size = remainder < 56 ? 64 : 128;
const uint64_t bit_length = static_cast<uint64_t>(size) * 8U;
for (std::size_t i = 0; i < 8; ++i)
final_blocks[final_size - 1 - i] = static_cast<uint8_t>(bit_length >> (i * 8));
transform(final_blocks, state);
if (final_size == 128) transform(final_blocks + 64, state);
std::array<uint8_t, 32> digest{};
for (std::size_t i = 0; i < state.size(); ++i) {
digest[i * 4] = static_cast<uint8_t>(state[i] >> 24);
digest[i * 4 + 1] = static_cast<uint8_t>(state[i] >> 16);
digest[i * 4 + 2] = static_cast<uint8_t>(state[i] >> 8);
digest[i * 4 + 3] = static_cast<uint8_t>(state[i]);
}
return digest;
}
} // namespace UI::LXMF::NomadNet
lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.cpp
+134
View File
@@ -0,0 +1,134 @@
#include "NomadNetPartialScheduler.h"
#include <algorithm>
namespace UI::LXMF::NomadNet {
bool PartialScheduler::due(uint32_t now_ms, uint32_t due_at_ms) noexcept {
return static_cast<int32_t>(now_ms - due_at_ms) >= 0;
}
uint32_t PartialScheduler::next_token() noexcept {
++_token_sequence;
if (_token_sequence == 0) ++_token_sequence;
return _token_sequence;
}
void PartialScheduler::configure(const CompactPage& page,
uint32_t page_generation,
uint32_t now_ms) noexcept {
_entries = {};
_count = page_generation == 0 ? 0 :
std::min(page.partials().size(), _entries.size());
_page_generation = page_generation;
_next_scan_index = 0;
_in_flight_token = 0;
_in_flight_index = 0;
for (std::size_t i = 0; i < _count; ++i) {
_entries[i].refresh_interval_ms = page.partials()[i].refresh_interval_ms;
_entries[i].descriptor_hash = page.partials()[i].descriptor_hash;
_entries[i].due_at_ms = now_ms;
_entries[i].pending = true;
}
}
void PartialScheduler::configure(const Document& document,
uint32_t page_generation,
uint32_t now_ms) noexcept {
_entries = {};
_count = page_generation == 0 ? 0 :
std::min(document.partials.size(), _entries.size());
_page_generation = page_generation;
_next_scan_index = 0;
_in_flight_token = 0;
_in_flight_index = 0;
for (std::size_t i = 0; i < _count; ++i) {
_entries[i].refresh_interval_ms = document.partials[i].refresh_interval_ms;
_entries[i].descriptor_hash = document.partials[i].descriptor_hash;
_entries[i].due_at_ms = now_ms;
_entries[i].pending = true;
}
}
void PartialScheduler::cancel(uint32_t page_generation) noexcept {
if (_page_generation != page_generation) return;
_entries = {};
_count = 0;
_page_generation = 0;
_next_scan_index = 0;
_in_flight_token = 0;
_in_flight_index = 0;
}
bool PartialScheduler::poll(uint32_t now_ms, bool browser_active,
bool owner_available,
PartialRequest& request) noexcept {
if (!browser_active || !owner_available || _page_generation == 0 ||
_in_flight_token != 0) return false;
for (std::size_t offset = 0; offset < _count; ++offset) {
const std::size_t i = (_next_scan_index + offset) % _count;
Entry& entry = _entries[i];
if (!entry.pending || entry.in_flight || !due(now_ms, entry.due_at_ms)) continue;
entry.pending = false;
entry.in_flight = true;
entry.started_at_ms = now_ms;
entry.request_token = next_token();
++entry.partial_generation;
if (entry.partial_generation == 0) ++entry.partial_generation;
_in_flight_token = entry.request_token;
_in_flight_index = static_cast<uint16_t>(i);
request.partial_index = static_cast<uint16_t>(i);
request.page_generation = _page_generation;
request.partial_generation = entry.partial_generation;
request.request_token = entry.request_token;
request.descriptor_hash = entry.descriptor_hash;
_next_scan_index = static_cast<uint8_t>((i + 1U) % _count);
return true;
}
return false;
}
bool PartialScheduler::complete(const PartialRequest& request, bool success,
uint32_t now_ms) noexcept {
if (_page_generation == 0 || request.page_generation != _page_generation ||
request.partial_index >= _count || request.request_token == 0 ||
request.request_token != _in_flight_token ||
request.partial_index != _in_flight_index) return false;
Entry& entry = _entries[request.partial_index];
if (!entry.in_flight || entry.request_token != request.request_token ||
entry.partial_generation != request.partial_generation ||
entry.descriptor_hash != request.descriptor_hash) return false;
entry.in_flight = false;
entry.request_token = 0;
_in_flight_token = 0;
_in_flight_index = 0;
if (success) {
if (entry.refresh_interval_ms != 0) {
entry.due_at_ms = entry.started_at_ms + entry.refresh_interval_ms + 1U;
entry.pending = true;
}
return true;
}
if (entry.refresh_interval_ms == 0) {
entry.pending = false;
return true;
}
entry.due_at_ms = now_ms + entry.refresh_interval_ms + 1U;
entry.pending = true;
return true;
}
bool PartialScheduler::request_now(std::size_t partial_index,
uint32_t page_generation,
uint32_t now_ms) noexcept {
if (page_generation != _page_generation || partial_index >= _count ||
_entries[partial_index].in_flight) return false;
Entry& entry = _entries[partial_index];
entry.due_at_ms = now_ms;
entry.pending = true;
return true;
}
} // namespace UI::LXMF::NomadNet
lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.h
+67
View File
@@ -0,0 +1,67 @@
#pragma once
#include "NomadNetCompactPage.h"
#include <array>
#include <cstddef>
#include <cstdint>
namespace UI::LXMF::NomadNet {
struct PartialRequest {
uint16_t partial_index = 0;
uint32_t page_generation = 0;
uint32_t partial_generation = 0;
uint32_t request_token = 0;
std::array<uint8_t, 32> descriptor_hash{};
};
// Allocation-free, single-owner scheduler for bounded dynamic partial work.
// It never retains CompactPage pointers or response bytes. The caller owns
// transport and must validate completion through the returned generation/token.
class PartialScheduler {
public:
void configure(const CompactPage& page, uint32_t page_generation,
uint32_t now_ms) noexcept;
void configure(const Document& document, uint32_t page_generation,
uint32_t now_ms) noexcept;
void cancel(uint32_t page_generation) noexcept;
bool poll(uint32_t now_ms, bool browser_active, bool owner_available,
PartialRequest& request) noexcept;
bool complete(const PartialRequest& request, bool success,
uint32_t now_ms) noexcept;
bool request_now(std::size_t partial_index, uint32_t page_generation,
uint32_t now_ms) noexcept;
bool empty() const noexcept { return _page_generation == 0 || _count == 0; }
bool in_flight() const noexcept { return _in_flight_token != 0; }
std::size_t size() const noexcept { return _count; }
uint32_t page_generation() const noexcept { return _page_generation; }
private:
struct Entry {
uint32_t refresh_interval_ms = 0;
uint32_t due_at_ms = 0;
uint32_t started_at_ms = 0;
uint32_t request_token = 0;
uint32_t partial_generation = 0;
std::array<uint8_t, 32> descriptor_hash{};
bool pending = false;
bool in_flight = false;
};
static bool due(uint32_t now_ms, uint32_t due_at_ms) noexcept;
uint32_t next_token() noexcept;
std::array<Entry, CompactPage::MAX_PARTIALS> _entries{};
std::size_t _count = 0;
uint32_t _page_generation = 0;
uint32_t _token_sequence = 0;
uint8_t _next_scan_index = 0;
uint32_t _in_flight_token = 0;
uint16_t _in_flight_index = 0;
};
} // namespace UI::LXMF::NomadNet
lib/tdeck_ui/UI/LXMF/UIManager.cpp
+4 -1
View File
@@ -2167,6 +2167,7 @@ bool UIManager::nomad_refresh_path_after_link_failure() {
}
uint32_t UIManager::nomad_advance_navigation_generation() {
_nomad_partial_scheduler.cancel(_nomad_navigation_generation);
++_nomad_navigation_generation;
if (_nomad_navigation_generation == 0) ++_nomad_navigation_generation;
return _nomad_navigation_generation;
@@ -2271,7 +2272,7 @@ void UIManager::nomad_open(const std::string& address, bool add_history,
if (local_result == NomadNet::LocalNavigationResult::APPLIED) {
_nomad_url = std::move(next_url);
_nomad_pending_scroll = -1;
nomad_advance_navigation_generation();
// A fragment jump keeps the same published page and partials.
return;
}
} catch (const std::bad_alloc&) {
@@ -2553,6 +2554,8 @@ NomadNet::PageApplyResult UIManager::nomad_apply_page_document(
_nomad_pending_history.clear();
return result;
}
_nomad_partial_scheduler.configure(
document, _nomad_navigation_generation, millis());
if (library_changed) _nomad_library_dirty = true;
_nomad_pending_scroll = -1;
return result;
lib/tdeck_ui/UI/LXMF/UIManager.h
+2
View File
@@ -15,6 +15,7 @@
#include "NavigationStack.h"
#include "NomadNetUrl.h"
#include "NomadNetDocument.h"
#include "NomadNetPartialScheduler.h"
#include "NomadNetProtocol.h"
#include "NomadNetHistory.h"
#include "NomadNetMailbox.h"
@@ -411,6 +412,7 @@ private:
NomadNet::Url _nomad_url;
NomadNet::DocumentParser _nomad_parser;
NomadNet::PartialScheduler _nomad_partial_scheduler;
NomadNet::ResponseBuffer _nomad_response;
NomadNet::PageHistory _nomad_history;
NomadNet::PageHistory::PendingOpen _nomad_pending_history;
tests/native/nomadnet_x86_flow/CMakeLists.txt
+6 -2
View File
@@ -27,8 +27,8 @@ target_compile_definitions(udp_interface PRIVATE
set(PYXIS_NOMADNET_DIR "${CMAKE_CURRENT_LIST_DIR}/../../../lib/tdeck_ui/UI/LXMF")
set(PYXIS_ROOT "${CMAKE_CURRENT_LIST_DIR}/../../..")
set(PYXIS_MANIFEST_BASE "3658c54cc76bdc16e7a514ff008f2cb70c69e192")
set(PYXIS_MANIFEST_BRANCH "feat/nomadnet-limits-observability")
set(PYXIS_MANIFEST_BASE "51e4b586c4c3867ae399f573557edda6f3b48a44")
set(PYXIS_MANIFEST_BRANCH "feat/nomadnet-partials-core")
set(PYXIS_MANIFEST_MICRORETICULUM "cd0338e7fc07d3a7785a450656ba766491cbf6e8")
execute_process(COMMAND git rev-parse HEAD WORKING_DIRECTORY "${RNS_SOURCE}"
OUTPUT_VARIABLE _rns_commit OUTPUT_STRIP_TRAILING_WHITESPACE
@@ -50,6 +50,9 @@ set(PYXIS_MANIFEST_FILES
"lib/tdeck_ui/UI/LXMF/NomadNetDocument.h"
"lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.cpp"
"lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.h"
"lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.cpp"
"lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.h"
"lib/tdeck_ui/UI/LXMF/NomadNetPartialHash.h"
"lib/tdeck_ui/UI/LXMF/NomadNetGlyphs.cpp"
"lib/tdeck_ui/UI/LXMF/NomadNetGlyphs.h"
"lib/tdeck_ui/UI/LXMF/NomadNetForm.cpp"
@@ -73,6 +76,7 @@ add_executable(pyxis_nomadnet_x86_flow
"${PYXIS_ROOT}/src/TCPClientInterface.cpp"
"${PYXIS_NOMADNET_DIR}/NomadNetDocument.cpp"
"${PYXIS_ROOT}/lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.cpp"
"${PYXIS_ROOT}/lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.cpp"
"${PYXIS_ROOT}/lib/tdeck_ui/UI/LXMF/NomadNetForm.cpp"
"${PYXIS_ROOT}/lib/tdeck_ui/UI/LXMF/NomadNetGlyphs.cpp"
"${PYXIS_NOMADNET_DIR}/NomadNetLibrary.cpp"
tests/native/nomadnet_x86_flow/run_flow.py
+5 -2
View File
@@ -43,8 +43,8 @@ if os.environ.get("PYXIS_FLOW_SCENARIOS"):
raise SystemExit("invalid PYXIS_FLOW_SCENARIOS")
SCENARIOS = requested
MANIFEST_BASE = "3658c54cc76bdc16e7a514ff008f2cb70c69e192"
MANIFEST_BRANCH = "feat/nomadnet-limits-observability"
MANIFEST_BASE = "51e4b586c4c3867ae399f573557edda6f3b48a44"
MANIFEST_BRANCH = "feat/nomadnet-partials-core"
MANIFEST_MICRORETICULUM = "cd0338e7fc07d3a7785a450656ba766491cbf6e8"
MANIFEST_FILES = (
"tests/native/nomadnet_x86_flow/CMakeLists.txt",
@@ -60,6 +60,9 @@ MANIFEST_FILES = (
"lib/tdeck_ui/UI/LXMF/NomadNetDocument.h",
"lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.cpp",
"lib/tdeck_ui/UI/LXMF/NomadNetCompactPage.h",
"lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.cpp",
"lib/tdeck_ui/UI/LXMF/NomadNetPartialScheduler.h",
"lib/tdeck_ui/UI/LXMF/NomadNetPartialHash.h",
"lib/tdeck_ui/UI/LXMF/NomadNetGlyphs.cpp",
"lib/tdeck_ui/UI/LXMF/NomadNetGlyphs.h",
"lib/tdeck_ui/UI/LXMF/NomadNetForm.cpp",
tests/native/test_nomadnet_partial_core.cpp
+314
View File
@@ -0,0 +1,314 @@
#include <iostream>
#include <iomanip>
#include <sstream>
#include <string>
#include "NomadNetCompactPage.h"
#include "NomadNetDocument.h"
#include "NomadNetPartialScheduler.h"
using UI::LXMF::NomadNet::BlockType;
using UI::LXMF::NomadNet::CompactPage;
using UI::LXMF::NomadNet::DocumentParser;
using UI::LXMF::NomadNet::PartialRequest;
using UI::LXMF::NomadNet::PartialScheduler;
using UI::LXMF::NomadNet::TruncationReason;
int main() {
int failures = 0;
auto check = [&](const char* name, bool condition) {
if (!condition) {
std::cerr << "FAIL: " << name << '\n';
++failures;
}
};
auto hex = [](const std::array<uint8_t, 32>& value) {
std::ostringstream output;
for (uint8_t byte : value)
output << std::hex << std::setw(2) << std::setfill('0')
<< static_cast<unsigned>(byte);
return output.str();
};
DocumentParser parser;
const auto document = parser.parse(
"before\n"
"`{f64a846313b874ee4a357040807f8c77:/page/hello.mu`10`pid=32|user_name}\n"
"after");
check("canonical partial descriptor is retained",
document.partials.size() == 1 && document.blocks.size() == 3 &&
document.blocks[1].type == BlockType::PARTIAL &&
document.blocks[1].partial_index == 0);
if (document.partials.size() == 1) {
const auto& partial = document.partials[0];
check("partial URL is exact",
partial.url == "f64a846313b874ee4a357040807f8c77:/page/hello.mu");
check("partial refresh is represented in milliseconds",
partial.refresh_interval_ms == 10000U);
check("partial ID and fields follow canonical components",
partial.id == "32" && partial.fields.size() == 2 &&
partial.fields[0] == "pid=32" && partial.fields[1] == "user_name");
check("partial selectors and canonical SHA-256 identity are retained",
partial.descriptor ==
"f64a846313b874ee4a357040807f8c77:/page/hello.mu`10`pid=32|user_name" &&
partial.selectors == "pid=32|user_name" &&
hex(partial.descriptor_hash) ==
"412d0e39b4703ab0f4b29f77158f252a66f560207ea5ea83882e270b8e68ea07");
}
const auto fractional = parser.parse(
"`{:relative.mu`.999}\n`{:timed.mu`1.25}");
check("sub-second canonical refresh disables automatic scheduling",
fractional.partials.size() == 2 &&
fractional.partials[0].refresh_interval_ms == 0U);
check("fractional canonical refresh retains millisecond precision",
fractional.partials.size() == 2 &&
fractional.partials[1].refresh_interval_ms == 1250U);
check("absent selector component retains canonical empty selector",
fractional.partials[0].fields.size() == 1 &&
fractional.partials[0].fields[0].empty() &&
fractional.partials[1].fields.size() == 1 &&
fractional.partials[1].fields[0].empty());
const auto escaped = parser.parse("\\`{:escaped.mu}");
check("escaped canonical partial is still recognized",
escaped.partials.size() == 1 && escaped.partials[0].url == ":escaped.mu");
const auto escaped_controls = parser.parse("\\-\n\\>heading\n\\`=");
check("escaped structural controls remain ordinary literal text",
escaped_controls.blocks.size() == 3 &&
escaped_controls.blocks[0].type == BlockType::TEXT &&
escaped_controls.blocks[0].runs[0].text == "-" &&
escaped_controls.blocks[1].type == BlockType::TEXT &&
escaped_controls.blocks[1].runs[0].text == ">heading" &&
escaped_controls.blocks[2].type == BlockType::TEXT &&
escaped_controls.blocks[2].runs[0].text == "`=");
const auto sanitized_comment = parser.parse("># hidden `<name`value>");
check("heading field sanitization precedes comment classification",
sanitized_comment.blocks.empty() && sanitized_comment.fields.empty());
const auto sanitized_table = parser.parse(
">`t`<name`value>\nA|B\nC|D\n`t");
check("heading field sanitization precedes table command recognition",
sanitized_table.tables.size() == 1 && sanitized_table.fields.empty());
const auto sanitized_reset = parser.parse(
">Heading\n><body `<name`value>");
check("heading field sanitization precedes section reset",
sanitized_reset.blocks.size() == 2 &&
sanitized_reset.blocks[1].depth == 0);
const auto sanitized_table_row = parser.parse(
"`t\n>Left|Right `<name`value>\nBottom|Row\n`t");
check("heading field sanitization occurs before table buffering",
!sanitized_table_row.table_runs.empty() &&
sanitized_table_row.table_runs[0].text == "Left");
const auto reset_comment = parser.parse("<# hidden");
const auto reset_table = parser.parse("<`t\nA|B\nC|D\n`t");
const auto reset_heading = parser.parse("<>Heading");
const auto reset_divider = parser.parse("<-");
check("section reset restarts canonical line classification",
reset_comment.blocks.empty() &&
reset_table.tables.size() == 1 &&
reset_heading.blocks.size() == 1 &&
reset_heading.blocks[0].type == BlockType::HEADING &&
reset_divider.blocks.size() == 1 &&
reset_divider.blocks[0].type == BlockType::DIVIDER);
const auto reset_literal = parser.parse("<`=\n-\n`=");
check("section reset restarts literal-toggle classification",
reset_literal.blocks.size() == 1 &&
reset_literal.blocks[0].type == BlockType::TEXT &&
reset_literal.blocks[0].runs[0].text == "-" &&
!reset_literal.malformed);
const auto decimal_refresh = parser.parse(
"`{:space.mu`1.0 }\n`{:underscore.mu`1_0}\n`{:exponent.mu`1.e2}");
check("canonical Python decimal refresh grammar is retained",
decimal_refresh.partials.size() == 3 &&
decimal_refresh.partials[0].refresh_interval_ms == 1000U &&
decimal_refresh.partials[1].refresh_interval_ms == 10000U &&
decimal_refresh.partials[2].refresh_interval_ms == 100000U);
const auto unicode_refresh = parser.parse(
u8"`{:unicode.mu`\u00A0\u0661.\u0662\u00A0}");
check("canonical Unicode decimal digits and outer whitespace are retained",
unicode_refresh.partials.size() == 1 &&
unicode_refresh.partials[0].refresh_interval_ms == 1200U);
bool rejected_information_separators = true;
for (char control = 0x1c; control <= 0x1f; ++control) {
const std::string source = "`{:control.mu`" + std::string(1, control) +
"1.2" + std::string(1, control) + "}";
const auto control_refresh = parser.parse(source);
rejected_information_separators = rejected_information_separators &&
control_refresh.partials.empty() && control_refresh.malformed;
}
check("Python-float-rejected information separators remain invalid",
rejected_information_separators);
const auto hexadecimal_refresh = parser.parse("`{:hex.mu`0x1p1}");
check("non-canonical C hexadecimal refresh is rejected",
hexadecimal_refresh.partials.empty() && hexadecimal_refresh.malformed);
const auto canonical_vector = parser.parse("`{:/page/a.mu}TRAIL");
check("first closing brace terminates the descriptor and canonical hash input",
canonical_vector.partials.size() == 1 &&
canonical_vector.partials[0].descriptor == ":/page/a.mu" &&
hex(canonical_vector.partials[0].descriptor_hash) ==
"07216f28dfbd82d34e655ded06936a911a4ea2cc138b46c0dd5ccb12231f541c");
const auto refresh_cap = parser.parse(
"`{:max.mu`604800}\n`{:too-long.mu`604800.001}\n`{:nan.mu`nan}");
check("refresh interval is finite and bounded below INT32 wrap ambiguity",
refresh_cap.partials.size() == 1 &&
refresh_cap.partials[0].refresh_interval_ms ==
DocumentParser::MAX_PARTIAL_REFRESH_MS && refresh_cap.malformed);
std::string too_many;
for (std::size_t index = 0; index <= DocumentParser::MAX_PARTIALS; ++index)
too_many += "`{:p" + std::to_string(index) + ".mu}\n";
const auto capped = parser.parse(too_many);
check("peer-controlled partial count is capped",
capped.partials.size() == DocumentParser::MAX_PARTIALS &&
capped.has_truncation(TruncationReason::PARTIALS));
const auto malformed = parser.parse("`{:broken.mu`not-a-number}");
check("malformed refresh does not create a schedulable descriptor",
malformed.partials.empty() && malformed.malformed &&
malformed.blocks.size() == 1 &&
malformed.blocks[0].type == BlockType::UNSUPPORTED);
const auto oversized_url = parser.parse(
"`{" + std::string(DocumentParser::MAX_PARTIAL_URL_BYTES + 1, 'u') + "}");
check("partial URL bytes are capped",
oversized_url.partials.empty() &&
oversized_url.has_truncation(TruncationReason::PARTIAL_DESCRIPTOR_BYTES));
std::string excessive_fields = "`{:fields.mu`1`";
for (std::size_t index = 0; index <= DocumentParser::MAX_PARTIAL_FIELDS; ++index) {
if (index != 0) excessive_fields += '|';
excessive_fields += "f" + std::to_string(index);
}
excessive_fields += '}';
const auto fields_capped = parser.parse(excessive_fields);
check("partial field count is capped",
fields_capped.partials.empty() &&
fields_capped.has_truncation(TruncationReason::PARTIAL_FIELDS));
const auto oversized_field = parser.parse(
"`{:field.mu`1`" +
std::string(DocumentParser::MAX_PARTIAL_FIELD_BYTES + 1, 'f') + "}");
check("partial field bytes are capped",
oversized_field.partials.empty() &&
oversized_field.has_truncation(TruncationReason::PARTIAL_FIELD_BYTES));
std::string total_metadata;
for (std::size_t index = 0; index < DocumentParser::MAX_PARTIALS; ++index)
total_metadata += "`{:" + std::string(256, static_cast<char>('a' + index)) + "}\n";
const auto metadata_capped = parser.parse(total_metadata);
check("aggregate partial metadata remains bounded",
metadata_capped.partials.size() < DocumentParser::MAX_PARTIALS &&
metadata_capped.has_truncation(TruncationReason::PARTIAL_DESCRIPTOR_BYTES));
CompactPage compact;
check("partial descriptors survive compact-page assignment", compact.assign(document) &&
compact.partials().size() == 1 && compact.blocks().size() == 3 &&
compact.blocks()[1].partial_index == 0);
if (compact.partials().size() == 1) {
const auto& partial = compact.partials()[0];
check("compact partial strings and fields remain exact",
compact.partial_url(partial) ==
"f64a846313b874ee4a357040807f8c77:/page/hello.mu" &&
compact.partial_selectors(partial) == "pid=32|user_name" &&
hex(partial.descriptor_hash) ==
"412d0e39b4703ab0f4b29f77158f252a66f560207ea5ea83882e270b8e68ea07" &&
compact.partial_id(partial) == "32" && partial.field_count == 2 &&
compact.partial_field(partial, 1) == "user_name");
}
const auto scheduled_document = parser.parse(
"`{:first.mu`10}\n`{:second.mu`20}");
CompactPage scheduled_page;
check("scheduler fixture compacts", scheduled_page.assign(scheduled_document));
PartialScheduler scheduler;
check("scheduler state has a fixed constrained-memory footprint",
sizeof(scheduler) <= 1024U);
scheduler.configure(scheduled_page, 7U, 1000U);
PartialRequest first{};
PartialRequest blocked{};
check("first partial is immediately due",
scheduler.poll(1000U, true, true, first) && first.partial_index == 0 &&
first.page_generation == 7U && first.request_token != 0U);
check("only one browser request can be outstanding",
!scheduler.poll(1000U, true, true, blocked));
check("matching completion is accepted", scheduler.complete(first, true, 1100U));
PartialRequest second{};
check("second initial partial is serialized after first",
scheduler.poll(1100U, true, true, second) && second.partial_index == 1);
check("second completion is accepted", scheduler.complete(second, true, 1200U));
check("refresh is based on request-start time and canonical strict expiry",
!scheduler.poll(11000U, true, true, blocked) &&
scheduler.poll(11001U, true, true, blocked) && blocked.partial_index == 0);
PartialScheduler fair_scheduler;
const auto fair_page = parser.parse("`{:first.mu`1}\n`{:second.mu`1}");
fair_scheduler.configure(fair_page, 77U, 0U);
PartialRequest fair_first;
PartialRequest fair_second;
check("overdue first partial cannot starve later due partials", [&] {
return fair_scheduler.poll(0U, true, true, fair_first) &&
fair_first.partial_index == 0 &&
fair_scheduler.complete(fair_first, true, 2000U) &&
fair_scheduler.poll(2000U, true, true, fair_second) &&
fair_second.partial_index == 1;
}());
scheduler.configure(scheduled_page, 8U, 20000U);
check("old-generation completion cannot mutate a new page",
!scheduler.complete(blocked, true, 20001U));
check("hidden browser suppresses dispatch",
!scheduler.poll(20000U, false, true, blocked));
check("busy browser owner suppresses dispatch",
!scheduler.poll(20000U, true, false, blocked));
check("dispatch resumes once browser is active and idle",
scheduler.poll(20000U, true, true, blocked) &&
blocked.page_generation == 8U);
scheduler.cancel(8U);
check("navigation cancellation revokes in-flight work",
!scheduler.complete(blocked, true, 20001U) && scheduler.empty());
const auto retry_document = parser.parse("`{:once.mu}");
CompactPage retry_page;
check("retry fixture compacts", retry_page.assign(retry_document));
scheduler.configure(retry_page, 9U, 0U);
PartialRequest retry{};
check("one-shot partial starts immediately", scheduler.poll(0U, true, true, retry));
const uint32_t first_partial_generation = retry.partial_generation;
check("failure is contained", scheduler.complete(retry, false, 1U));
check("one-shot transfer failure has no automatic retry",
!scheduler.poll(0xffffffffU, true, true, retry));
check("manual p-link style retry rearms the exact occurrence",
scheduler.request_now(0U, 9U, 5000U) &&
scheduler.poll(5000U, true, true, retry) &&
retry.partial_generation != first_partial_generation);
check("mismatched partial generation is rejected", [&] {
PartialRequest stale = retry;
--stale.partial_generation;
return !scheduler.complete(stale, true, 5001U);
}());
check("mismatched descriptor identity is rejected", [&] {
PartialRequest stale = retry;
stale.descriptor_hash[0] ^= 0xffU;
return !scheduler.complete(stale, true, 5001U);
}());
check("manual retry completion remains valid", scheduler.complete(retry, true, 5001U));
const auto wrap_document = parser.parse("`{:wrap.mu`5}");
CompactPage wrap_page;
check("wrap fixture compacts", wrap_page.assign(wrap_document));
scheduler.configure(wrap_page, 10U, 0xfffffff0U);
check("initial request remains due across clock wrap",
scheduler.poll(0xfffffff0U, true, true, retry));
check("wrapped request failure is accepted", scheduler.complete(retry, false, 0xfffffff1U));
check("wrap-safe deadline is not early",
!scheduler.poll(0x00001379U, true, true, retry));
check("wrap-safe deadline becomes due exactly once",
scheduler.poll(0x0000137aU, true, true, retry));
if (failures == 0) std::cout << "partial core parser checks passed\n";
return failures == 0 ? 0 : 1;
}
tests/native/test_nomadnet_partial_core.py
+58
View File
@@ -0,0 +1,58 @@
import shutil
import subprocess
from pathlib import Path
import pytest
HERE = Path(__file__).resolve().parent
ROOT = HERE.parent.parent
INCLUDE = ROOT / "lib" / "tdeck_ui" / "UI" / "LXMF"
SOURCE = HERE / "test_nomadnet_partial_core.cpp"
def _cxx():
for name in ("clang++", "g++"):
if shutil.which(name):
return name
pytest.skip("no C++ compiler found")
def test_nomadnet_partial_core_native(tmp_path):
binary = tmp_path / "test_nomadnet_partial_core"
command = [
_cxx(), "-std=c++17", "-Wall", "-Wextra", "-Werror",
f"-I{INCLUDE}", str(SOURCE),
str(INCLUDE / "NomadNetDocument.cpp"),
str(INCLUDE / "NomadNetCompactPage.cpp"),
str(INCLUDE / "NomadNetGlyphs.cpp"),
str(INCLUDE / "NomadNetPartialScheduler.cpp"),
"-o", str(binary),
]
compiled = subprocess.run(command, capture_output=True, text=True)
assert compiled.returncode == 0, compiled.stdout + compiled.stderr
result = subprocess.run([str(binary)], capture_output=True, text=True, timeout=30)
assert result.returncode == 0, result.stdout + result.stderr
assert result.stdout.strip() == "partial core parser checks passed"
def test_partial_scheduler_is_generation_owned_but_transport_dispatch_is_deferred():
manager_h = (INCLUDE / "UIManager.h").read_text()
manager_cpp = (INCLUDE / "UIManager.cpp").read_text()
assert "NomadNet::PartialScheduler _nomad_partial_scheduler;" in manager_h
advance = manager_cpp.split(
"uint32_t UIManager::nomad_advance_navigation_generation() {", 1
)[1].split("\n}", 1)[0]
assert "_nomad_partial_scheduler.cancel(_nomad_navigation_generation);" in advance
publication = manager_cpp.split(
"NomadNet::PageApplyResult UIManager::nomad_apply_page_document(", 1
)[1].split("void UIManager::nomad_update()", 1)[0]
assert publication.index("result != NomadNet::PageApplyResult::APPLIED") < publication.index(
"_nomad_partial_scheduler.configure("
)
# This increment owns only the bounded model and scheduler. Partial transport,
# response replacement, and form-map integration belong to the next increment.
assert "_nomad_partial_scheduler.poll(" not in manager_cpp
local_jump = manager_cpp.split(
"if (local_result == NomadNet::LocalNavigationResult::APPLIED)", 1
)[1].split("return;", 1)[0]
assert "nomad_advance_navigation_generation()" not in local_jump