ci: OCR dark and light text in separate passes

Run 38712060819 showed that one image for both polarities smears page text: the morphological open treats the white gaps between dark letters as light text. Dark text now comes from the local threshold of run 38707461053, which read the grey and dark onboarding text; light text, such as labels on coloured buttons, from the difference from an open, which leaves no ring at the button edge. The words of both passes are merged by position, keeping the more confident reading.

scripts/ci/desktop-linux/screen-text.py

@@ -1,9 +1,10 @@
 #!/usr/bin/env python3
 
-# Reads the words tesseract found on a screenshot (its TSV output) and prints the phrases on screen, or the centre of one phrase.
+# Reads the words tesseract found on a screenshot (TSV output, one file per OCR pass) and
+# prints the phrases on screen, or the centre of one phrase.
 #
-#   screen-text.py <scale> lines <tsv>        one phrase per line: centre x,y and its words
-#   screen-text.py <scale> find <text> <tsv>  "x y" of the first phrase containing the text; fails if none does
+#   screen-text.py <scale> lines <tsv>...        one phrase per line: centre x,y and its words
+#   screen-text.py <scale> find <text> <tsv>...  "x y" of the first phrase containing the text; fails if none does
 #
 # <scale> is how much the screenshot was enlarged for OCR; printed coordinates are screen pixels.
 # Words are compared ignoring case and punctuation.
@@ -27,10 +28,30 @@ def read_words(path, scale):
             if row["level"] != "5" or not key(text) or float(row["conf"]) < MIN_CONFIDENCE:
                 continue
             left, top, width, height = (int(row[k]) / scale for k in ("left", "top", "width", "height"))
-            words.append({"text": text, "key": key(text), "l": left, "t": top, "r": left + width, "b": top + height})
+            words.append({"text": text, "key": key(text), "conf": float(row["conf"]),
+                          "l": left, "t": top, "r": left + width, "b": top + height})
     return words
 
 
+def area(w):
+    return (w["r"] - w["l"]) * (w["b"] - w["t"])
+
+
+def same_place(a, b):
+    dx = min(a["r"], b["r"]) - max(a["l"], b["l"])
+    dy = min(a["b"], b["b"]) - max(a["t"], b["t"])
+    return dx > 0 and dy > 0 and dx * dy >= 0.5 * min(area(a), area(b))
+
+
+# The passes read the same place differently; keep the more confident reading of each place.
+def merge(passes):
+    kept = []
+    for w in sorted((w for words in passes for w in words), key=lambda w: -w["conf"]):
+        if not any(same_place(w, k) for k in kept):
+            kept.append(w)
+    return kept
+
+
 def centre(words):
     return ((min(w["l"] for w in words) + max(w["r"] for w in words)) / 2,
             (min(w["t"] for w in words) + max(w["b"] for w in words)) / 2)
@@ -80,11 +101,11 @@ def find(all_phrases, text):
 def main():
     scale, command, args = float(sys.argv[1]), sys.argv[2], sys.argv[3:]
     if command == "lines":
-        for phrase in phrases(read_words(args[0], scale)):
+        for phrase in phrases(merge(read_words(p, scale) for p in args)):
             x, y = centre(phrase)
             print(f"{round(x)},{round(y)} {' '.join(w['text'] for w in phrase)}")
     elif command == "find":
-        found = find(phrases(read_words(args[1], scale)), args[0])
+        found = find(phrases(merge(read_words(p, scale) for p in args[1:])), args[0])
         if not found:
             sys.exit(1)
         print(f"{round(found[0])} {round(found[1])}")

scripts/ci/desktop-linux/ui

@@ -84,17 +84,17 @@ start() {
 }
 
 # ocr <png> lines | ocr <png> find <text>
-# Text of either polarity becomes dark on white: the difference from a morphological close (which removes dark strokes) and
-# from an open (which removes light ones). A mean-based local threshold rings coloured buttons, and tesseract skips text in rings.
+# Dark text is cut out by a local threshold, light text (labels on coloured buttons) by the difference from a morphological open:
+# a local threshold of the negative rings the buttons, and tesseract reads nothing inside a ring.
 ocr() {
   local png=$1 o=$state/ocr
   shift
   convert "$png" -colorspace Gray "$o/gray.png"
-  convert "$o/gray.png" \( +clone -morphology Dilate Rectangle:9x1 -morphology Dilate Rectangle:1x9 -morphology Erode Rectangle:9x1 -morphology Erode Rectangle:1x9 \) -compose Difference -composite "$o/dark.png"
-  convert "$o/gray.png" \( +clone -morphology Erode Rectangle:9x1 -morphology Erode Rectangle:1x9 -morphology Dilate Rectangle:9x1 -morphology Dilate Rectangle:1x9 \) -compose Difference -composite "$o/light.png"
-  convert "$o/dark.png" "$o/light.png" -compose Lighten -composite -negate -resize 200% "$o/text.png"
-  tesseract "$o/text.png" "$o/text" --psm 11 tsv > /dev/null 2>&1
-  python3 "$here/screen-text.py" 2 "$@" "$o/text.tsv"
+  convert "$o/gray.png" -resize 200% -lat 60x60-5% "$o/dark.png"
+  convert "$o/gray.png" \( +clone -morphology Erode Rectangle:9x1 -morphology Erode Rectangle:1x9 -morphology Dilate Rectangle:9x1 -morphology Dilate Rectangle:1x9 \) -compose Difference -composite -negate -resize 200% "$o/light.png"
+  tesseract "$o/dark.png" "$o/dark" --psm 11 tsv > /dev/null 2>&1
+  tesseract "$o/light.png" "$o/light" --psm 11 tsv > /dev/null 2>&1
+  python3 "$here/screen-text.py" 2 "$@" "$o/dark.tsv" "$o/light.tsv"
 }
 
 screen_ocr() {
@@ -118,8 +118,10 @@ shot() {
   import -window root "$f.png"
   ocr "$f.png" lines > "$f.txt"
   mkdir -p "$UI_OUT/ocr"
-  cp "$state/ocr/text.png" "$UI_OUT/ocr/$(basename "$f").png"
-  cp "$state/ocr/text.tsv" "$UI_OUT/ocr/$(basename "$f").tsv"
+  for pass in dark light; do
+    cp "$state/ocr/$pass.png" "$UI_OUT/ocr/$(basename "$f")-$pass.png"
+    cp "$state/ocr/$pass.tsv" "$UI_OUT/ocr/$(basename "$f")-$pass.tsv"
+  done
   echo "::group::text on screen: $(basename "$f")"
   cat "$f.txt"
   echo "::endgroup::"