ci: judge each OCR word's background from the rows around it

Run 38715611090 still stopped at the profile screen: averaging 4x4 blocks and taking a 5x5-block median leaves large bold text in the estimate (the title read as a dark background) and lets the white labels lift a grey disabled button over the threshold. A map coarse enough for titles would swallow 55-pixel buttons, so each word's background is now the median of the rows 2 to 4 pixels above and below its box, read from the 8-bit grayscale screenshot, which also feeds both OCR passes. Checked offline on the screenshots and TSVs of runs 38712854507 and 38714550750: the first screen, the profile screen with its title and grey button, and the branch chat list with the typed text and the directory row all read correctly.

scripts/ci/desktop-linux/screen-text.py

@@ -6,16 +6,18 @@
 #   screen-text.py <scale> <ocr dir> find <text>   "x y" of the first phrase containing the text; fails if none does
 #
 # <ocr dir> has tesseract's TSV of the dark-text and the light-text pass (dark.tsv, light.tsv), in pixels of the
-# screenshot enlarged by <scale>, and background.pgm, the background luminance at screen size. Printed coordinates are
+# screenshot enlarged by <scale>, and gray.pgm, the screenshot in 8-bit grayscale. Printed coordinates are
 # screen pixels. Words are compared ignoring case and punctuation.
 
 import csv
 import os
 import re
+import statistics
 import sys
 
-# Luminance (0-255) below which a background is dark enough for light text: the blue buttons are about 115.
-LIGHT_BACKGROUND = 160
+# Luminance (0-255) below which a background is dark enough for light text: blue buttons are about 115, disabled grey
+# ones about 137, light ones (window, fields, cards) over 230.
+LIGHT_BACKGROUND = 190
 
 
 def key(text):
@@ -34,7 +36,7 @@ def read_words(path, scale):
     return words
 
 
-def read_background(path):
+def read_gray(path):
     with open(path, "rb") as f:
         data = f.read()
     header = re.match(rb"P5\s+(\d+)\s+(\d+)\s+(\d+)\s", data)
@@ -43,6 +45,14 @@ def read_background(path):
     return int(header[1]), int(header[2]), data[header.end():]
 
 
+# The median of the rows 2 to 4 pixels above and below the word: its background, free of its own letters.
+def background(gray, w):
+    width, height, luminance = gray
+    left, right = max(int(w["l"]), 0), min(int(w["r"]), width - 1)
+    rows = [int(w["t"]) - d for d in (2, 3, 4)] + [int(w["b"]) + d for d in (2, 3, 4)]
+    return statistics.median(luminance[y * width + x] for y in rows if 0 <= y < height for x in range(left, right + 1))
+
+
 def centre(words):
     return ((min(w["l"] for w in words) + max(w["r"] for w in words)) / 2,
             (min(w["t"] for w in words) + max(w["b"] for w in words)) / 2)
@@ -51,11 +61,10 @@ def centre(words):
 # Each pass also reads noise where its polarity doesn't apply (the gaps between dark letters look like light text),
 # so a word counts only from the pass that matches the background under it.
 def screen_words(ocr_dir, scale):
-    width, height, luminance = read_background(os.path.join(ocr_dir, "background.pgm"))
+    gray = read_gray(os.path.join(ocr_dir, "gray.pgm"))
 
     def on_light_background(w):
-        x, y = centre([w])
-        return luminance[min(int(y), height - 1) * width + min(int(x), width - 1)] >= LIGHT_BACKGROUND
+        return background(gray, w) >= LIGHT_BACKGROUND
 
     dark = read_words(os.path.join(ocr_dir, "dark.tsv"), scale)
     light = read_words(os.path.join(ocr_dir, "light.tsv"), scale)

scripts/ci/desktop-linux/ui

@@ -89,10 +89,9 @@ start() {
 ocr() {
   local png=$1 o=$state/ocr
   shift
-  convert "$png" -colorspace Gray "$o/gray.png"
-  convert "$o/gray.png" -scale 25% -statistic Median 5x5 -scale 400% -depth 8 "$o/background.pgm"
-  convert "$o/gray.png" -resize 200% -lat 60x60-5% "$o/dark.png"
-  convert "$o/gray.png" \( +clone -morphology Erode Rectangle:9x1 -morphology Erode Rectangle:1x9 -morphology Dilate Rectangle:9x1 -morphology Dilate Rectangle:1x9 \) -compose Difference -composite -negate -resize 200% "$o/light.png"
+  convert "$png" -colorspace Gray -depth 8 "$o/gray.pgm"
+  convert "$o/gray.pgm" -resize 200% -lat 60x60-5% "$o/dark.png"
+  convert "$o/gray.pgm" \( +clone -morphology Erode Rectangle:9x1 -morphology Erode Rectangle:1x9 -morphology Dilate Rectangle:9x1 -morphology Dilate Rectangle:1x9 \) -compose Difference -composite -negate -resize 200% "$o/light.png"
   tesseract "$o/dark.png" "$o/dark" --psm 11 tsv > /dev/null 2>&1
   tesseract "$o/light.png" "$o/light" --psm 11 tsv > /dev/null 2>&1
   python3 "$here/screen-text.py" 2 "$o" "$@"