From ff0db0dcb81c8a3659fe552294cebbd6647857c0 Mon Sep 17 00:00:00 2001 From: liquidraver <504870+liquidraver@users.noreply.github.com> Date: Thu, 13 Aug 2026 21:38:09 +0200 Subject: [PATCH] lr2021 debug v2 --- zephcore/adapters/radio/LoRaRadioBase.cpp | 8 ++- zephcore/boards/common/debug.conf | 30 +++++++-- zephcore/boards/common/hangdebug.conf | 64 +++++++++++++++++++ .../drivers/lora/lr20xx/lr20xx_lora.c | 46 ++++++++++++- 4 files changed, 138 insertions(+), 10 deletions(-) create mode 100644 zephcore/boards/common/hangdebug.conf diff --git a/zephcore/adapters/radio/LoRaRadioBase.cpp b/zephcore/adapters/radio/LoRaRadioBase.cpp index fdd16eb..2fb7329 100644 --- a/zephcore/adapters/radio/LoRaRadioBase.cpp +++ b/zephcore/adapters/radio/LoRaRadioBase.cpp @@ -1091,7 +1091,13 @@ int8_t LoRaRadioBase::cadLevelMinEff() uint8_t base = hwCadBasePeak(); uint8_t pmin = hwCadPeakMin(); - if (base == 0 || pmin == 0 || pmin <= base - CAD_LEVEL_MIN) { + /* The clamp only binds when the lowest peak the static window can reach, + * base + CAD_LEVEL_MIN, would land below it. Note the sign: this was + * written `base - CAD_LEVEL_MIN` once, which with CAD_LEVEL_MIN negative + * evaluates to base + 8 — always above pmin, so the narrowing never + * happened and the whole function was inert. */ + if (base == 0 || pmin == 0 || + (int)base + CAD_LEVEL_MIN >= (int)pmin) { return CAD_LEVEL_MIN; } return (int8_t)((int)pmin - (int)base); diff --git a/zephcore/boards/common/debug.conf b/zephcore/boards/common/debug.conf index 3eab5a0..ea5569c 100644 --- a/zephcore/boards/common/debug.conf +++ b/zephcore/boards/common/debug.conf @@ -12,10 +12,18 @@ # ========== Core debug features ========== CONFIG_LOG=y CONFIG_ASSERT=y -CONFIG_USE_SEGGER_RTT=y -CONFIG_SEGGER_RTT_BUFFER_SIZE_UP=4096 CONFIG_THREAD_NAME=y +# SEGGER RTT is OFF by default: it costs 4,280 bytes of RAM (a 4 KB up-buffer +# plus control block) and is unusable without a debug probe, which most people +# do not have. Logs reach a plain serial terminal over USB CDC regardless — +# see the backend note below. Measured on meshtracker_x1: 1.8 % of total RAM +# on a build that sits near 92 %. +# +# With a J-Link attached, turn it back on for that build only: +# -DCONFIG_USE_SEGGER_RTT=y -DCONFIG_LOG_BACKEND_RTT=y \ +# -DCONFIG_SEGGER_RTT_BUFFER_SIZE_UP=4096 + # Halt-and-inspect on fatal faults instead of the production reboot-on-fatal # (prj.conf sets CONFIG_ZEPHCORE_RESET_ON_FATAL_ERROR=y). A debugger needs the # core stopped at the fault with a stack dump, not rebooted out from under it. @@ -30,11 +38,19 @@ CONFIG_LOG_BUFFER_SIZE=8192 CONFIG_LOG_PROCESS_TRIGGER_THRESHOLD=1 CONFIG_LOG_PROCESS_THREAD_STACK_SIZE=2048 -# RTT backend only — no UART backend on nRF52. -# UART backend stalls on uart_poll_out() when the USB host disconnects; -# RTT DROP mode discards messages without blocking. -CONFIG_LOG_BACKEND_RTT=y -CONFIG_LOG_BACKEND_RTT_MODE_DROP=y +# No backend is selected here, deliberately. Zephyr's Kconfig.uart is +# "default y if !SHELL_BACKEND_SERIAL" and ZephCore never enables the serial +# shell backend, so CONFIG_LOG_BACKEND_UART is already on in every build — and +# on the boards whose chosen console is a cdc_acm_uart (nRF52 with native USB, +# e.g. meshtracker_x1, promicro_lr2021, t1000_e) that means logs come out of the +# USB CDC port with no J-Link and no extra flags. +# +# The comment that used to sit here claimed "RTT backend only — no UART backend +# on nRF52", which was never true: this file has never set +# CONFIG_LOG_BACKEND_UART=n, so it described an intent it did not implement. +# The stated worry — uart_poll_out() stalling when the USB host disconnects — +# does not apply in deferred mode, where the log thread does the blocking, not +# the mesh or radio threads. # Silence noisy subsystems CONFIG_CFB_LOG_LEVEL_ERR=y diff --git a/zephcore/boards/common/hangdebug.conf b/zephcore/boards/common/hangdebug.conf new file mode 100644 index 0000000..17cdbfb --- /dev/null +++ b/zephcore/boards/common/hangdebug.conf @@ -0,0 +1,64 @@ +# Hang / freeze diagnosis overlay. +# +# Build (stacks on top of debug.conf — order matters, this one must come last): +# west build -b meshtracker_x1 zephcore --pristine -- \ +# -DEXTRA_CONF_FILE="boards/common/debug.conf;boards/common/hangdebug.conf" +# +# Exists to answer one question before any others: when the node goes silent, +# is the MCU dead, or is the radio wedged with the MCU still running? Those +# have completely different fixes and the ordinary debug build cannot tell them +# apart, because it produces no output either way. + +# ========== Logging mode: deliberately left DEFERRED ========== +# CONFIG_LOG_MODE_IMMEDIATE was tried here and BRICKS THE BOOT on any board +# whose console is a USB CDC ACM (meshtracker_x1, promicro_lr2021, t1000_e...). +# Immediate logging writes synchronously from the calling context, and before +# the USB host has enumerated, uart_poll_out() on a CDC ACM with no host blocks +# forever — so the node hangs during early init and never comes up. This is the +# stall debug.conf's own backend comment has always warned about; it is harmless +# in deferred mode because the log thread absorbs it, and fatal in immediate +# mode because the booting thread does. Do not re-add it. +# +# It is also unnecessary for catching a crash: Zephyr's fatal handler calls +# LOG_PANIC(), which switches logging to synchronous and flushes whatever is +# queued. A genuine fault therefore prints its dump even in deferred mode — +# meaning a freeze that produces NO output is evidence of a hard hang or a +# supply collapse rather than of a lost message. + +# ========== Claw back the RAM immediate mode would have saved ========== +# debug.conf sizes the deferred log buffer at 8 KB for heavy DBG output. This +# overlay is not trying to capture a firehose — it wants the thread table and a +# fault dump — so a quarter of that is plenty and returns 6 KB to a build that +# otherwise sits above 91 % on the X1. Headroom is the point: a production +# build survives what a debug build does not, so the debug build's own footprint +# is a suspect in its own right. +CONFIG_LOG_BUFFER_SIZE=2048 + +# ========== Proof of life + stack headroom ========== +# Periodic per-thread stack usage, printed from a thread of its own. Two jobs: +# +# 1. Proof of life. If these lines keep appearing after the radio goes quiet, +# the MCU is healthy and the radio is wedged — look at the driver. If they +# stop when the radio does, the hang is MCU-side (fault, deadlock, or a +# starved scheduler) and the radio is a red herring. +# +# 2. Stack headroom. The X1 debug build sits near 92% RAM, so a stack overflow +# is a live hypothesis rather than a formality. This rules it in or out by +# showing each thread's high-water mark. +CONFIG_THREAD_ANALYZER=y +CONFIG_THREAD_ANALYZER_AUTO=y +CONFIG_THREAD_ANALYZER_AUTO_INTERVAL=10 +CONFIG_THREAD_ANALYZER_RUN_UNLOCKED=y +CONFIG_THREAD_NAME=y + +# ========== Turn silent corruption into a loud fault ========== +# Without this an overflowing stack quietly scribbles on whatever is below it and +# the failure surfaces somewhere unrelated. With it, the MPU traps the overrun +# at the moment it happens and — thanks to IMMEDIATE logging above — the dump +# actually reaches the console. +CONFIG_HW_STACK_PROTECTION=y + +# Keep the fatal handler halting rather than rebooting (debug.conf already sets +# this). With synchronous logging the dump is out before the halt, and halting +# preserves the state for a debugger if one is ever attached. +CONFIG_ZEPHCORE_RESET_ON_FATAL_ERROR=n diff --git a/zephcore/patches/zephyr-new/drivers/lora/lr20xx/lr20xx_lora.c b/zephcore/patches/zephyr-new/drivers/lora/lr20xx/lr20xx_lora.c index fe35234..f4f54c3 100644 --- a/zephcore/patches/zephyr-new/drivers/lora/lr20xx/lr20xx_lora.c +++ b/zephcore/patches/zephyr-new/drivers/lora/lr20xx/lr20xx_lora.c @@ -37,8 +37,22 @@ LOG_MODULE_REGISTER(lr20xx_lora, CONFIG_LORA_LOG_LEVEL); /* Dedicated DIO1 work queue — keeps LoRa interrupt processing off the - * system work queue so USB/BLE/timer work items cannot delay packet RX. */ -#define LR20XX_DIO1_WQ_STACK_SIZE 2560 + * system work queue so USB/BLE/timer work items cannot delay packet RX. + * + * 4 KB, where the SX126x and LR11xx both use 2560 and are stable on that. This + * is NOT "the others got it wrong" — it is one path those two do not have: + * lr20xx_hardware_reset() runs *from this work handler*, and since the PRAM + * landed it carries lr20xx_load_pram() → lr20xx_patch_load_pram() → 18 block + * writes through the SDK's regmem layer. Neither sibling has a firmware patch + * to load, so neither reaches that depth from work-queue context. The ordinary + * path (apply_modem_config / restart_rx, plus a LOG_DBG and a CHECK_CMD + * get_status per command in CONFIG_LOG builds) is comparable across all three. + * + * Cheap insurance rather than a diagnosed fix: 1,536 bytes, funded several + * times over by dropping SEGGER RTT from debug.conf. If the thread analyzer + * shows this queue's high-water mark nowhere near 2560, put it back — the + * number should come from a measurement, not from this comment. */ +#define LR20XX_DIO1_WQ_STACK_SIZE 4096 K_THREAD_STACK_DEFINE(lr20xx_dio1_wq_stack, LR20XX_DIO1_WQ_STACK_SIZE); /* Hardware limit on concurrent LoRa side detectors (ConfigureSideDetectors @@ -1471,6 +1485,34 @@ static void lr20xx_dio1_work_handler(struct k_work *work) lr20xx_system_clear_errors(ctx); } + /* Supply faults, observed passively. + * + * Bit 10 LOW_BATTERY ("power supply level dropped below the threshold") + * and bit 11 PA_OVP_OCP ("power amplifier over-current protection has + * triggered") are deliberately NOT in the DIO1 mask set by + * apply_modem_config: routing them would make them assert the pin and + * land in this handler with no branch to handle them, i.e. straight into + * the "no IRQ handled" safety restart — a behaviour change nobody asked + * for. But they latch in the status register regardless of routing, and + * get_and_clear_irq_status above already read the whole word, so noticing + * them here is free and changes nothing. + * + * Worth noticing because a rail that sags under PA load is invisible + * otherwise, and the chip's own GetVbat is only sampled once at init with + * the radio idle. Coverage is partial by nature — this only fires when + * some other IRQ brings us into the handler — so absence is not proof. */ + if (irq & (LR20XX_SYSTEM_IRQ_LOW_BATTERY | + LR20XX_SYSTEM_IRQ_PA_OVP_OCP)) { + uint16_t vbat_mv = 0; + + lr20xx_system_get_vbat(ctx, LR20XX_SYSTEM_VALUE_FORMAT_UNIT, + LR20XX_SYSTEM_MEAS_RES_12_BITS, &vbat_mv); + LOG_ERR("SUPPLY FAULT: %s%s (chip VBAT now %u mV)", + (irq & LR20XX_SYSTEM_IRQ_LOW_BATTERY) ? "LOW_BATTERY " : "", + (irq & LR20XX_SYSTEM_IRQ_PA_OVP_OCP) ? "PA_OVP/OCP" : "", + vbat_mv); + } + /* Error-only IRQs are not progress. Resetting the counter on them let * an error that re-fires on every RX restart spin forever, never * reaching the stuck-DIO1 escape hatch below. */