diff --git a/scripts/ci/desktop-linux/screen-text.py b/scripts/ci/desktop-linux/screen-text.py index 41964ecbb9..26cae42156 100755 --- a/scripts/ci/desktop-linux/screen-text.py +++ b/scripts/ci/desktop-linux/screen-text.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 -# Reads the words tesseract found on a screenshot (its TSV output) and prints the phrases on screen, or the centre of one phrase. +# Reads the words tesseract found on a screenshot (TSV output, one file per OCR pass) and +# prints the phrases on screen, or the centre of one phrase. # -# screen-text.py lines one phrase per line: centre x,y and its words -# screen-text.py find "x y" of the first phrase containing the text; fails if none does +# screen-text.py lines ... one phrase per line: centre x,y and its words +# screen-text.py find ... "x y" of the first phrase containing the text; fails if none does # # is how much the screenshot was enlarged for OCR; printed coordinates are screen pixels. # Words are compared ignoring case and punctuation. @@ -27,10 +28,30 @@ def read_words(path, scale): if row["level"] != "5" or not key(text) or float(row["conf"]) < MIN_CONFIDENCE: continue left, top, width, height = (int(row[k]) / scale for k in ("left", "top", "width", "height")) - words.append({"text": text, "key": key(text), "l": left, "t": top, "r": left + width, "b": top + height}) + words.append({"text": text, "key": key(text), "conf": float(row["conf"]), + "l": left, "t": top, "r": left + width, "b": top + height}) return words +def area(w): + return (w["r"] - w["l"]) * (w["b"] - w["t"]) + + +def same_place(a, b): + dx = min(a["r"], b["r"]) - max(a["l"], b["l"]) + dy = min(a["b"], b["b"]) - max(a["t"], b["t"]) + return dx > 0 and dy > 0 and dx * dy >= 0.5 * min(area(a), area(b)) + + +# The passes read the same place differently; keep the more confident reading of each place. +def merge(passes): + kept = [] + for w in sorted((w for words in passes for w in words), key=lambda w: -w["conf"]): + if not any(same_place(w, k) for k in kept): + kept.append(w) + return kept + + def centre(words): return ((min(w["l"] for w in words) + max(w["r"] for w in words)) / 2, (min(w["t"] for w in words) + max(w["b"] for w in words)) / 2) @@ -80,11 +101,11 @@ def find(all_phrases, text): def main(): scale, command, args = float(sys.argv[1]), sys.argv[2], sys.argv[3:] if command == "lines": - for phrase in phrases(read_words(args[0], scale)): + for phrase in phrases(merge(read_words(p, scale) for p in args)): x, y = centre(phrase) print(f"{round(x)},{round(y)} {' '.join(w['text'] for w in phrase)}") elif command == "find": - found = find(phrases(read_words(args[1], scale)), args[0]) + found = find(phrases(merge(read_words(p, scale) for p in args[1:])), args[0]) if not found: sys.exit(1) print(f"{round(found[0])} {round(found[1])}") diff --git a/scripts/ci/desktop-linux/ui b/scripts/ci/desktop-linux/ui index c83aac7daf..49eb1afd44 100755 --- a/scripts/ci/desktop-linux/ui +++ b/scripts/ci/desktop-linux/ui @@ -84,17 +84,17 @@ start() { } # ocr lines | ocr find -# Text of either polarity becomes dark on white: the difference from a morphological close (which removes dark strokes) and -# from an open (which removes light ones). A mean-based local threshold rings coloured buttons, and tesseract skips text in rings. +# Dark text is cut out by a local threshold, light text (labels on coloured buttons) by the difference from a morphological open: +# a local threshold of the negative rings the buttons, and tesseract reads nothing inside a ring. ocr() { local png=$1 o=$state/ocr shift convert "$png" -colorspace Gray "$o/gray.png" - convert "$o/gray.png" \( +clone -morphology Dilate Rectangle:9x1 -morphology Dilate Rectangle:1x9 -morphology Erode Rectangle:9x1 -morphology Erode Rectangle:1x9 \) -compose Difference -composite "$o/dark.png" - convert "$o/gray.png" \( +clone -morphology Erode Rectangle:9x1 -morphology Erode Rectangle:1x9 -morphology Dilate Rectangle:9x1 -morphology Dilate Rectangle:1x9 \) -compose Difference -composite "$o/light.png" - convert "$o/dark.png" "$o/light.png" -compose Lighten -composite -negate -resize 200% "$o/text.png" - tesseract "$o/text.png" "$o/text" --psm 11 tsv > /dev/null 2>&1 - python3 "$here/screen-text.py" 2 "$@" "$o/text.tsv" + convert "$o/gray.png" -resize 200% -lat 60x60-5% "$o/dark.png" + convert "$o/gray.png" \( +clone -morphology Erode Rectangle:9x1 -morphology Erode Rectangle:1x9 -morphology Dilate Rectangle:9x1 -morphology Dilate Rectangle:1x9 \) -compose Difference -composite -negate -resize 200% "$o/light.png" + tesseract "$o/dark.png" "$o/dark" --psm 11 tsv > /dev/null 2>&1 + tesseract "$o/light.png" "$o/light" --psm 11 tsv > /dev/null 2>&1 + python3 "$here/screen-text.py" 2 "$@" "$o/dark.tsv" "$o/light.tsv" } screen_ocr() { @@ -118,8 +118,10 @@ shot() { import -window root "$f.png" ocr "$f.png" lines > "$f.txt" mkdir -p "$UI_OUT/ocr" - cp "$state/ocr/text.png" "$UI_OUT/ocr/$(basename "$f").png" - cp "$state/ocr/text.tsv" "$UI_OUT/ocr/$(basename "$f").tsv" + for pass in dark light; do + cp "$state/ocr/$pass.png" "$UI_OUT/ocr/$(basename "$f")-$pass.png" + cp "$state/ocr/$pass.tsv" "$UI_OUT/ocr/$(basename "$f")-$pass.tsv" + done echo "::group::text on screen: $(basename "$f")" cat "$f.txt" echo "::endgroup::"