diff --git a/scripts/ci/desktop-linux/screen-colour.py b/scripts/ci/desktop-linux/screen-colour.py new file mode 100755 index 0000000000..5135f33afe --- /dev/null +++ b/scripts/ci/desktop-linux/screen-colour.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +# Prints how many pixels of an image stand out from its background (its most common colour), and their mean colour. +# With a colour, fails unless enough pixels stand out and that channel of their mean exceeds the other two by a margin. +# Usage: screen-colour.py [red|green|blue] + +import re +import sys +from collections import Counter + +# A pixel stands out when one of its channels differs from the background's by more than this. +DIFFERENCE = 48 +# A 42 px profile image has about 1200 pixels; fewer than this standing out is no image. +MIN_PIXELS = 300 +DOMINANCE = 50 +CHANNELS = {"red": 0, "green": 1, "blue": 2} + + +def read_ppm(path): + with open(path, "rb") as f: + data = f.read() + header = re.match(rb"P6\s+(\d+)\s+(\d+)\s+(\d+)\s", data) + if int(header[3]) != 255: + sys.exit(f"{path}: expected 8-bit samples, got maximum {int(header[3])}") + samples = data[header.end():] + return [tuple(samples[i:i + 3]) for i in range(0, len(samples) - 2, 3)] + + +def main(): + pixels = read_ppm(sys.argv[1]) + colour = sys.argv[2] if len(sys.argv) > 2 else None + if colour is not None and colour not in CHANNELS: + sys.exit(f"unknown colour: {colour}") + background = Counter(pixels).most_common(1)[0][0] + standing_out = [p for p in pixels if max(abs(a - b) for a, b in zip(p, background)) > DIFFERENCE] + n = len(standing_out) + mean = [round(sum(p[c] for p in standing_out) / n) if n else 0 for c in range(3)] + print(f"{n} px, mean {mean[0]},{mean[1]},{mean[2]}") + if colour is not None: + c = CHANNELS[colour] + if n < MIN_PIXELS or any(mean[c] - mean[o] < DOMINANCE for o in range(3) if o != c): + sys.exit(1) + + +main() diff --git a/scripts/ci/desktop-linux/screen-text.py b/scripts/ci/desktop-linux/screen-text.py index a6bfbdec79..1f8c81fbd1 100755 --- a/scripts/ci/desktop-linux/screen-text.py +++ b/scripts/ci/desktop-linux/screen-text.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 -# Reads the words tesseract found on a screenshot and prints the phrases on screen, or the centre of one phrase. +# Reads the words tesseract found on a screenshot and prints the phrases on screen, or where one text is. # # screen-text.py lines one phrase per line: centre x,y and its words -# screen-text.py find "x y" of the first phrase containing the text; fails if none does +# screen-text.py find "x y left top right bottom": centre and box of the text, from the phrase +# that is exactly the text, else the first one containing it; fails if none does # # has tesseract's TSV of the dark-text and the light-text pass (dark.tsv, light.tsv), in pixels of the # screenshot enlarged by , and gray.pgm, the screenshot in 8-bit grayscale. Printed coordinates are -# screen pixels. Words are compared ignoring case and punctuation. +# screen pixels. Text is compared ignoring case, spaces and punctuation. import csv import os @@ -53,9 +54,13 @@ def background(gray, w): return statistics.median(luminance[y * width + x] for y in rows if 0 <= y < height for x in range(left, right + 1)) +def box(words): + return min(w["l"] for w in words), min(w["t"] for w in words), max(w["r"] for w in words), max(w["b"] for w in words) + + def centre(words): - return ((min(w["l"] for w in words) + max(w["r"] for w in words)) / 2, - (min(w["t"] for w in words) + max(w["b"] for w in words)) / 2) + left, top, right, bottom = box(words) + return (left + right) / 2, (top + bottom) / 2 # Each pass also reads noise where its polarity doesn't apply (the gaps between dark letters look like light text), @@ -89,23 +94,35 @@ def phrases(words): return sorted(result, key=lambda p: (min(w["t"] for w in p), p[0]["l"])) -# OCR can glue an icon to the first word, or punctuation or the text cursor to the last, -# so long edge words match by suffix or prefix. -def word_matches(word, k, first, last): - if word == k: - return True - if len(k) < 4: - return False - return (first and word.endswith(k)) or (last and word.startswith(k)) +# OCR can split or join words ("#ci_group" read as "#ci_ group"), so words are compared by their joined characters. +# It can also glue an icon to the first word, or punctuation or the text cursor to the last, so the text may start +# inside the first word and end inside the last when at least 4 of its characters are in that word. +def span_matches(words, target): + keys = [w["key"] for w in words] + joined = "".join(keys) + start = joined.find(target) + while start >= 0: + end = start + len(target) + first_ok = start == 0 or len(keys[0]) - start >= 4 + last_ok = end == len(joined) or len(keys[-1]) - (len(joined) - end) >= 4 + if start < len(keys[0]) and end > len(joined) - len(keys[-1]) and first_ok and last_ok: + return True + start = joined.find(target, start + 1) + return False def find(all_phrases, text): - keys = [k for k in map(key, text.split()) if k] - n = len(keys) + target = key(text) + if not target: + return None for phrase in all_phrases: - for i in range(len(phrase) - n + 1): - if all(word_matches(phrase[i + j]["key"], k, j == 0, j == n - 1) for j, k in enumerate(keys)): - return centre(phrase[i:i + n]) + if "".join(w["key"] for w in phrase) == target: + return phrase + for phrase in all_phrases: + for i in range(len(phrase)): + for j in range(i + 1, len(phrase) + 1): + if span_matches(phrase[i:j], target): + return phrase[i:j] return None @@ -117,10 +134,10 @@ def main(): x, y = centre(phrase) print(f"{round(x)},{round(y)} {' '.join(w['text'] for w in phrase)}") elif command == "find": - found = find(on_screen, args[0]) - if not found: + words = find(on_screen, args[0]) + if not words: sys.exit(1) - print(f"{round(found[0])} {round(found[1])}") + print(" ".join(str(round(v)) for v in centre(words) + box(words))) else: sys.exit(f"unknown command: {command}") diff --git a/scripts/ci/desktop-linux/ui b/scripts/ci/desktop-linux/ui index 42d267ba57..006d20aa65 100755 --- a/scripts/ci/desktop-linux/ui +++ b/scripts/ci/desktop-linux/ui @@ -8,12 +8,18 @@ # ui start start Xvfb, openbox and /bin/simplex, and wait for its window # ui shot save UI_OUT/NN-.png and NN-.txt (OCR's input and output in UI_OUT/ocr), and print the text on screen # ui text print the text on screen, one phrase per line after its centre x,y -# ui wait [secs] wait up to secs (30) until the text is on screen (case and punctuation ignored) +# ui find print "x y left top right bottom", the text's centre and box; fail if it is not on screen +# ui wait [secs] wait up to secs (30) until the text is on screen # ui click click at x,y # ui click-text [secs] wait up to secs (20) for the text and click its centre +# ui avatar [colour] print how many pixels of the profile image left of a list row's name (the text) stand out +# from the background, and their mean colour, saving the area as UI_OUT/NN-avatar-.png; +# with red, green or blue, fail unless that colour dominates # ui type type into the focused field # ui key ... press keys named as in xdotool (Return, Escape, ctrl+a) # ui stop stop the app, openbox and Xvfb +# +# Text is compared ignoring case, spaces and punctuation; a phrase that is exactly the text wins over one that contains it. set -euo pipefail @@ -102,7 +108,7 @@ screen_ocr() { ocr "$state/ocr/screen.png" "$@" } -# prints "x y" of the text once it is on screen +# prints "x y left top right bottom" of the text once it is on screen wait_for() { local text=$1 deadline=$((SECONDS + $2)) until screen_ocr find "$text"; do @@ -111,10 +117,16 @@ wait_for() { done } -shot() { - local n f +# prints UI_OUT/NN-, numbered after the files already saved +next_file() { + local n n=$(find "$UI_OUT" -maxdepth 1 -name '[0-9][0-9]-*.png' | wc -l) - f=$(printf '%s/%02d-%s' "$UI_OUT" $((n + 1)) "$1") + printf '%s/%02d-%s' "$UI_OUT" $((n + 1)) "$1" +} + +shot() { + local f + f=$(next_file "$1") import -window root "$f.png" ocr "$f.png" lines > "$f.txt" mkdir -p "$UI_OUT/ocr" @@ -133,6 +145,25 @@ click() { sleep 1 } +# A list row's profile image is 42 px square, 8 px left of the row's name, and starts a little below the name's top: +# the area allows for OCR's box being a few pixels off, and leaves out the rows above and below. +avatar() { + local text=$1 colour=${2:-} pos x y l t r b f stats + if ! pos=$(screen_ocr find "$text"); then + log "avatar \"$text\": not on screen" + return 1 + fi + read -r x y l t r b <<< "$pos" + f=$(next_file "avatar-$(tr -cd 'a-zA-Z0-9_' <<< "$text")") + convert "$state/ocr/screen.png" -crop "52x64+$((l > 56 ? l - 56 : 0))+$((t > 6 ? t - 6 : 0))" +repage "$f.png" + convert "$f.png" -depth 8 "$state/ocr/avatar.ppm" + if ! stats=$(python3 "$here/screen-colour.py" "$state/ocr/avatar.ppm" ${colour:+"$colour"}); then + log "avatar \"$text\": $stats, not $colour" + return 1 + fi + log "avatar \"$text\": $stats${colour:+, $colour}" +} + case "${1:-}" in start) log "start $2" @@ -140,13 +171,22 @@ case "${1:-}" in ;; shot) shot "$2" ;; text) screen_ocr lines ;; + find) + if ! pos=$(screen_ocr find "$2"); then + log "find \"$2\": not on screen" >&2 + exit 1 + fi + log "find \"$2\": $pos" > /dev/null + echo "$pos" + ;; wait) secs=${3:-30} if ! pos=$(wait_for "$2" "$secs"); then log "wait \"$2\": not on screen after $secs s" exit 1 fi - log "wait \"$2\": at ${pos/ /,}" + read -r x y _ <<< "$pos" + log "wait \"$2\": at $x,$y" ;; click) click "$2" "$3" @@ -158,10 +198,11 @@ case "${1:-}" in log "click-text \"$2\": not on screen after $secs s" exit 1 fi - read -r x y <<< "$pos" + read -r x y _ <<< "$pos" click "$x" "$y" log "click-text \"$2\" at $x,$y" ;; + avatar) avatar "$2" "${3:-}" ;; type) xdotool type --delay 50 -- "$2" log "type \"$2\"" @@ -178,7 +219,7 @@ case "${1:-}" in log "stop" ;; *) - sed -n '3,16s/^# \{0,1\}//p' "$0" + sed -n '3,/^$/s/^# \{0,1\}//p' "$0" exit 1 ;; esac