From 5cae929f0cefa932821bbaba588e972c2bfe3ba5 Mon Sep 17 00:00:00 2001 From: spacedandy-agents-simplex <342056592+spacedandy-agents-simplex@users.noreply.github.com> Date: Thu, 15 Oct 2026 11:34:49 +0000 Subject: [PATCH] ci: find text by exact phrase first, report boxes and avatar colours in the desktop ui helper A phrase that is exactly the text now wins over one that only contains it (the alert's Search button under "Search in Directory?", the Directory header next to the Search in Directory row), and words are compared by their joined characters, so OCR splitting "#ci_group" still matches. New commands: ui find prints a text's centre and box; ui avatar measures the profile image left of a list row's name, saves that area, and with a colour checks that it dominates. --- scripts/ci/desktop-linux/screen-colour.py | 45 +++++++++++++++++ scripts/ci/desktop-linux/screen-text.py | 59 +++++++++++++++-------- scripts/ci/desktop-linux/ui | 57 +++++++++++++++++++--- 3 files changed, 132 insertions(+), 29 deletions(-) create mode 100755 scripts/ci/desktop-linux/screen-colour.py diff --git a/scripts/ci/desktop-linux/screen-colour.py b/scripts/ci/desktop-linux/screen-colour.py new file mode 100755 index 0000000000..5135f33afe --- /dev/null +++ b/scripts/ci/desktop-linux/screen-colour.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +# Prints how many pixels of an image stand out from its background (its most common colour), and their mean colour. +# With a colour, fails unless enough pixels stand out and that channel of their mean exceeds the other two by a margin. +# Usage: screen-colour.py [red|green|blue] + +import re +import sys +from collections import Counter + +# A pixel stands out when one of its channels differs from the background's by more than this. +DIFFERENCE = 48 +# A 42 px profile image has about 1200 pixels; fewer than this standing out is no image. +MIN_PIXELS = 300 +DOMINANCE = 50 +CHANNELS = {"red": 0, "green": 1, "blue": 2} + + +def read_ppm(path): + with open(path, "rb") as f: + data = f.read() + header = re.match(rb"P6\s+(\d+)\s+(\d+)\s+(\d+)\s", data) + if int(header[3]) != 255: + sys.exit(f"{path}: expected 8-bit samples, got maximum {int(header[3])}") + samples = data[header.end():] + return [tuple(samples[i:i + 3]) for i in range(0, len(samples) - 2, 3)] + + +def main(): + pixels = read_ppm(sys.argv[1]) + colour = sys.argv[2] if len(sys.argv) > 2 else None + if colour is not None and colour not in CHANNELS: + sys.exit(f"unknown colour: {colour}") + background = Counter(pixels).most_common(1)[0][0] + standing_out = [p for p in pixels if max(abs(a - b) for a, b in zip(p, background)) > DIFFERENCE] + n = len(standing_out) + mean = [round(sum(p[c] for p in standing_out) / n) if n else 0 for c in range(3)] + print(f"{n} px, mean {mean[0]},{mean[1]},{mean[2]}") + if colour is not None: + c = CHANNELS[colour] + if n < MIN_PIXELS or any(mean[c] - mean[o] < DOMINANCE for o in range(3) if o != c): + sys.exit(1) + + +main() diff --git a/scripts/ci/desktop-linux/screen-text.py b/scripts/ci/desktop-linux/screen-text.py index a6bfbdec79..1f8c81fbd1 100755 --- a/scripts/ci/desktop-linux/screen-text.py +++ b/scripts/ci/desktop-linux/screen-text.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 -# Reads the words tesseract found on a screenshot and prints the phrases on screen, or the centre of one phrase. +# Reads the words tesseract found on a screenshot and prints the phrases on screen, or where one text is. # # screen-text.py lines one phrase per line: centre x,y and its words -# screen-text.py find "x y" of the first phrase containing the text; fails if none does +# screen-text.py find "x y left top right bottom": centre and box of the text, from the phrase +# that is exactly the text, else the first one containing it; fails if none does # # has tesseract's TSV of the dark-text and the light-text pass (dark.tsv, light.tsv), in pixels of the # screenshot enlarged by , and gray.pgm, the screenshot in 8-bit grayscale. Printed coordinates are -# screen pixels. Words are compared ignoring case and punctuation. +# screen pixels. Text is compared ignoring case, spaces and punctuation. import csv import os @@ -53,9 +54,13 @@ def background(gray, w): return statistics.median(luminance[y * width + x] for y in rows if 0 <= y < height for x in range(left, right + 1)) +def box(words): + return min(w["l"] for w in words), min(w["t"] for w in words), max(w["r"] for w in words), max(w["b"] for w in words) + + def centre(words): - return ((min(w["l"] for w in words) + max(w["r"] for w in words)) / 2, - (min(w["t"] for w in words) + max(w["b"] for w in words)) / 2) + left, top, right, bottom = box(words) + return (left + right) / 2, (top + bottom) / 2 # Each pass also reads noise where its polarity doesn't apply (the gaps between dark letters look like light text), @@ -89,23 +94,35 @@ def phrases(words): return sorted(result, key=lambda p: (min(w["t"] for w in p), p[0]["l"])) -# OCR can glue an icon to the first word, or punctuation or the text cursor to the last, -# so long edge words match by suffix or prefix. -def word_matches(word, k, first, last): - if word == k: - return True - if len(k) < 4: - return False - return (first and word.endswith(k)) or (last and word.startswith(k)) +# OCR can split or join words ("#ci_group" read as "#ci_ group"), so words are compared by their joined characters. +# It can also glue an icon to the first word, or punctuation or the text cursor to the last, so the text may start +# inside the first word and end inside the last when at least 4 of its characters are in that word. +def span_matches(words, target): + keys = [w["key"] for w in words] + joined = "".join(keys) + start = joined.find(target) + while start >= 0: + end = start + len(target) + first_ok = start == 0 or len(keys[0]) - start >= 4 + last_ok = end == len(joined) or len(keys[-1]) - (len(joined) - end) >= 4 + if start < len(keys[0]) and end > len(joined) - len(keys[-1]) and first_ok and last_ok: + return True + start = joined.find(target, start + 1) + return False def find(all_phrases, text): - keys = [k for k in map(key, text.split()) if k] - n = len(keys) + target = key(text) + if not target: + return None for phrase in all_phrases: - for i in range(len(phrase) - n + 1): - if all(word_matches(phrase[i + j]["key"], k, j == 0, j == n - 1) for j, k in enumerate(keys)): - return centre(phrase[i:i + n]) + if "".join(w["key"] for w in phrase) == target: + return phrase + for phrase in all_phrases: + for i in range(len(phrase)): + for j in range(i + 1, len(phrase) + 1): + if span_matches(phrase[i:j], target): + return phrase[i:j] return None @@ -117,10 +134,10 @@ def main(): x, y = centre(phrase) print(f"{round(x)},{round(y)} {' '.join(w['text'] for w in phrase)}") elif command == "find": - found = find(on_screen, args[0]) - if not found: + words = find(on_screen, args[0]) + if not words: sys.exit(1) - print(f"{round(found[0])} {round(found[1])}") + print(" ".join(str(round(v)) for v in centre(words) + box(words))) else: sys.exit(f"unknown command: {command}") diff --git a/scripts/ci/desktop-linux/ui b/scripts/ci/desktop-linux/ui index 42d267ba57..006d20aa65 100755 --- a/scripts/ci/desktop-linux/ui +++ b/scripts/ci/desktop-linux/ui @@ -8,12 +8,18 @@ # ui start start Xvfb, openbox and /bin/simplex, and wait for its window # ui shot save UI_OUT/NN-.png and NN-.txt (OCR's input and output in UI_OUT/ocr), and print the text on screen # ui text print the text on screen, one phrase per line after its centre x,y -# ui wait [secs] wait up to secs (30) until the text is on screen (case and punctuation ignored) +# ui find print "x y left top right bottom", the text's centre and box; fail if it is not on screen +# ui wait [secs] wait up to secs (30) until the text is on screen # ui click click at x,y # ui click-text [secs] wait up to secs (20) for the text and click its centre +# ui avatar [colour] print how many pixels of the profile image left of a list row's name (the text) stand out +# from the background, and their mean colour, saving the area as UI_OUT/NN-avatar-.png; +# with red, green or blue, fail unless that colour dominates # ui type type into the focused field # ui key ... press keys named as in xdotool (Return, Escape, ctrl+a) # ui stop stop the app, openbox and Xvfb +# +# Text is compared ignoring case, spaces and punctuation; a phrase that is exactly the text wins over one that contains it. set -euo pipefail @@ -102,7 +108,7 @@ screen_ocr() { ocr "$state/ocr/screen.png" "$@" } -# prints "x y" of the text once it is on screen +# prints "x y left top right bottom" of the text once it is on screen wait_for() { local text=$1 deadline=$((SECONDS + $2)) until screen_ocr find "$text"; do @@ -111,10 +117,16 @@ wait_for() { done } -shot() { - local n f +# prints UI_OUT/NN-, numbered after the files already saved +next_file() { + local n n=$(find "$UI_OUT" -maxdepth 1 -name '[0-9][0-9]-*.png' | wc -l) - f=$(printf '%s/%02d-%s' "$UI_OUT" $((n + 1)) "$1") + printf '%s/%02d-%s' "$UI_OUT" $((n + 1)) "$1" +} + +shot() { + local f + f=$(next_file "$1") import -window root "$f.png" ocr "$f.png" lines > "$f.txt" mkdir -p "$UI_OUT/ocr" @@ -133,6 +145,25 @@ click() { sleep 1 } +# A list row's profile image is 42 px square, 8 px left of the row's name, and starts a little below the name's top: +# the area allows for OCR's box being a few pixels off, and leaves out the rows above and below. +avatar() { + local text=$1 colour=${2:-} pos x y l t r b f stats + if ! pos=$(screen_ocr find "$text"); then + log "avatar \"$text\": not on screen" + return 1 + fi + read -r x y l t r b <<< "$pos" + f=$(next_file "avatar-$(tr -cd 'a-zA-Z0-9_' <<< "$text")") + convert "$state/ocr/screen.png" -crop "52x64+$((l > 56 ? l - 56 : 0))+$((t > 6 ? t - 6 : 0))" +repage "$f.png" + convert "$f.png" -depth 8 "$state/ocr/avatar.ppm" + if ! stats=$(python3 "$here/screen-colour.py" "$state/ocr/avatar.ppm" ${colour:+"$colour"}); then + log "avatar \"$text\": $stats, not $colour" + return 1 + fi + log "avatar \"$text\": $stats${colour:+, $colour}" +} + case "${1:-}" in start) log "start $2" @@ -140,13 +171,22 @@ case "${1:-}" in ;; shot) shot "$2" ;; text) screen_ocr lines ;; + find) + if ! pos=$(screen_ocr find "$2"); then + log "find \"$2\": not on screen" >&2 + exit 1 + fi + log "find \"$2\": $pos" > /dev/null + echo "$pos" + ;; wait) secs=${3:-30} if ! pos=$(wait_for "$2" "$secs"); then log "wait \"$2\": not on screen after $secs s" exit 1 fi - log "wait \"$2\": at ${pos/ /,}" + read -r x y _ <<< "$pos" + log "wait \"$2\": at $x,$y" ;; click) click "$2" "$3" @@ -158,10 +198,11 @@ case "${1:-}" in log "click-text \"$2\": not on screen after $secs s" exit 1 fi - read -r x y <<< "$pos" + read -r x y _ <<< "$pos" click "$x" "$y" log "click-text \"$2\" at $x,$y" ;; + avatar) avatar "$2" "${3:-}" ;; type) xdotool type --delay 50 -- "$2" log "type \"$2\"" @@ -178,7 +219,7 @@ case "${1:-}" in log "stop" ;; *) - sed -n '3,16s/^# \{0,1\}//p' "$0" + sed -n '3,/^$/s/^# \{0,1\}//p' "$0" exit 1 ;; esac