mirror of
https://github.com/simplex-chat/simplex-chat.git
synced 2026-11-05 04:07:44 +00:00
spacedandy-agents-simplex
4fa605d056
Run 38707461053 read the grey onboarding text after the local threshold, but not the white labels on the blue buttons: in the negative image the threshold drew a thick ring around each button, and tesseract output nothing inside it. The text image is now the difference of the screenshot from its morphological close and open, so dark and light text both become dark on white and step edges between regions cancel; one tesseract pass instead of two. Each shot keeps that image and tesseract's TSV in UI_OUT/ocr.
96 lines
3.4 KiB
Python
Executable File
96 lines
3.4 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
|
|
# Reads the words tesseract found on a screenshot (its TSV output) and prints the phrases on screen, or the centre of one phrase.
|
|
#
|
|
# screen-text.py <scale> lines <tsv> one phrase per line: centre x,y and its words
|
|
# screen-text.py <scale> find <text> <tsv> "x y" of the first phrase containing the text; fails if none does
|
|
#
|
|
# <scale> is how much the screenshot was enlarged for OCR; printed coordinates are screen pixels.
|
|
# Words are compared ignoring case and punctuation.
|
|
|
|
import csv
|
|
import re
|
|
import sys
|
|
|
|
MIN_CONFIDENCE = 10
|
|
|
|
|
|
def key(text):
|
|
return re.sub(r"[^0-9a-z]", "", text.lower())
|
|
|
|
|
|
def read_words(path, scale):
|
|
words = []
|
|
with open(path, newline="") as f:
|
|
for row in csv.DictReader(f, delimiter="\t", quoting=csv.QUOTE_NONE):
|
|
text = (row["text"] or "").strip()
|
|
if row["level"] != "5" or not key(text) or float(row["conf"]) < MIN_CONFIDENCE:
|
|
continue
|
|
left, top, width, height = (int(row[k]) / scale for k in ("left", "top", "width", "height"))
|
|
words.append({"text": text, "key": key(text), "l": left, "t": top, "r": left + width, "b": top + height})
|
|
return words
|
|
|
|
|
|
def centre(words):
|
|
return ((min(w["l"] for w in words) + max(w["r"] for w in words)) / 2,
|
|
(min(w["t"] for w in words) + max(w["b"] for w in words)) / 2)
|
|
|
|
|
|
# Words on one row that are closer than 1.5 of their height form a phrase; wider gaps separate UI elements.
|
|
def phrases(words):
|
|
rows = []
|
|
for w in sorted(words, key=lambda w: (w["t"] + w["b"]) / 2):
|
|
cy, h = (w["t"] + w["b"]) / 2, w["b"] - w["t"]
|
|
row = next((r for r in rows if abs(r["cy"] - cy) <= 0.5 * min(r["h"], h)), None)
|
|
if row:
|
|
row["words"].append(w)
|
|
else:
|
|
rows.append({"cy": cy, "h": h, "words": [w]})
|
|
result = []
|
|
for row in rows:
|
|
phrase = []
|
|
for w in sorted(row["words"], key=lambda w: w["l"]):
|
|
if phrase and w["l"] - phrase[-1]["r"] > 1.5 * (w["b"] - w["t"]):
|
|
result.append(phrase)
|
|
phrase = []
|
|
phrase.append(w)
|
|
result.append(phrase)
|
|
return sorted(result, key=lambda p: (min(w["t"] for w in p), p[0]["l"]))
|
|
|
|
|
|
# OCR can glue an icon to the first word or punctuation to the last, so long edge words match by suffix or prefix.
|
|
def word_matches(word, k, first, last):
|
|
if word == k:
|
|
return True
|
|
if len(k) < 4:
|
|
return False
|
|
return (first and word.endswith(k)) or (last and word.startswith(k))
|
|
|
|
|
|
def find(all_phrases, text):
|
|
keys = [k for k in map(key, text.split()) if k]
|
|
n = len(keys)
|
|
for phrase in all_phrases:
|
|
for i in range(len(phrase) - n + 1):
|
|
if all(word_matches(phrase[i + j]["key"], k, j == 0, j == n - 1) for j, k in enumerate(keys)):
|
|
return centre(phrase[i:i + n])
|
|
return None
|
|
|
|
|
|
def main():
|
|
scale, command, args = float(sys.argv[1]), sys.argv[2], sys.argv[3:]
|
|
if command == "lines":
|
|
for phrase in phrases(read_words(args[0], scale)):
|
|
x, y = centre(phrase)
|
|
print(f"{round(x)},{round(y)} {' '.join(w['text'] for w in phrase)}")
|
|
elif command == "find":
|
|
found = find(phrases(read_words(args[1], scale)), args[0])
|
|
if not found:
|
|
sys.exit(1)
|
|
print(f"{round(found[0])} {round(found[1])}")
|
|
else:
|
|
sys.exit(f"unknown command: {command}")
|
|
|
|
|
|
main()
|