mirror of
https://github.com/simplex-chat/simplex-chat.git
synced 2026-11-11 11:47:44 +00:00
ci: find text by exact phrase first, report boxes and avatar colours in the desktop ui helper
A phrase that is exactly the text now wins over one that only contains it (the alert's Search button under "Search in Directory?", the Directory header next to the Search in Directory row), and words are compared by their joined characters, so OCR splitting "#ci_group" still matches. New commands: ui find prints a text's centre and box; ui avatar measures the profile image left of a list row's name, saves that area, and with a colour checks that it dominates.
This commit is contained in:
spacedandy-agents-simplex committed
1 parent
65759c6233
commit
5cae929f0c
3 files changed
+132
-29
No files matched your search
Executable
+45
@@ -0,0 +1,45 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Prints how many pixels of an image stand out from its background (its most common colour), and their mean colour.
|
||||
# With a colour, fails unless enough pixels stand out and that channel of their mean exceeds the other two by a margin.
|
||||
# Usage: screen-colour.py <image.ppm> [red|green|blue]
|
||||
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
|
||||
# A pixel stands out when one of its channels differs from the background's by more than this.
|
||||
DIFFERENCE = 48
|
||||
# A 42 px profile image has about 1200 pixels; fewer than this standing out is no image.
|
||||
MIN_PIXELS = 300
|
||||
DOMINANCE = 50
|
||||
CHANNELS = {"red": 0, "green": 1, "blue": 2}
|
||||
|
||||
|
||||
def read_ppm(path):
|
||||
with open(path, "rb") as f:
|
||||
data = f.read()
|
||||
header = re.match(rb"P6\s+(\d+)\s+(\d+)\s+(\d+)\s", data)
|
||||
if int(header[3]) != 255:
|
||||
sys.exit(f"{path}: expected 8-bit samples, got maximum {int(header[3])}")
|
||||
samples = data[header.end():]
|
||||
return [tuple(samples[i:i + 3]) for i in range(0, len(samples) - 2, 3)]
|
||||
|
||||
|
||||
def main():
|
||||
pixels = read_ppm(sys.argv[1])
|
||||
colour = sys.argv[2] if len(sys.argv) > 2 else None
|
||||
if colour is not None and colour not in CHANNELS:
|
||||
sys.exit(f"unknown colour: {colour}")
|
||||
background = Counter(pixels).most_common(1)[0][0]
|
||||
standing_out = [p for p in pixels if max(abs(a - b) for a, b in zip(p, background)) > DIFFERENCE]
|
||||
n = len(standing_out)
|
||||
mean = [round(sum(p[c] for p in standing_out) / n) if n else 0 for c in range(3)]
|
||||
print(f"{n} px, mean {mean[0]},{mean[1]},{mean[2]}")
|
||||
if colour is not None:
|
||||
c = CHANNELS[colour]
|
||||
if n < MIN_PIXELS or any(mean[c] - mean[o] < DOMINANCE for o in range(3) if o != c):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
main()
|
||||
@@ -1,13 +1,14 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
# Reads the words tesseract found on a screenshot and prints the phrases on screen, or the centre of one phrase.
|
||||
# Reads the words tesseract found on a screenshot and prints the phrases on screen, or where one text is.
|
||||
#
|
||||
# screen-text.py <scale> <ocr dir> lines one phrase per line: centre x,y and its words
|
||||
# screen-text.py <scale> <ocr dir> find <text> "x y" of the first phrase containing the text; fails if none does
|
||||
# screen-text.py <scale> <ocr dir> find <text> "x y left top right bottom": centre and box of the text, from the phrase
|
||||
# that is exactly the text, else the first one containing it; fails if none does
|
||||
#
|
||||
# <ocr dir> has tesseract's TSV of the dark-text and the light-text pass (dark.tsv, light.tsv), in pixels of the
|
||||
# screenshot enlarged by <scale>, and gray.pgm, the screenshot in 8-bit grayscale. Printed coordinates are
|
||||
# screen pixels. Words are compared ignoring case and punctuation.
|
||||
# screen pixels. Text is compared ignoring case, spaces and punctuation.
|
||||
|
||||
import csv
|
||||
import os
|
||||
@@ -53,9 +54,13 @@ def background(gray, w):
|
||||
return statistics.median(luminance[y * width + x] for y in rows if 0 <= y < height for x in range(left, right + 1))
|
||||
|
||||
|
||||
def box(words):
|
||||
return min(w["l"] for w in words), min(w["t"] for w in words), max(w["r"] for w in words), max(w["b"] for w in words)
|
||||
|
||||
|
||||
def centre(words):
|
||||
return ((min(w["l"] for w in words) + max(w["r"] for w in words)) / 2,
|
||||
(min(w["t"] for w in words) + max(w["b"] for w in words)) / 2)
|
||||
left, top, right, bottom = box(words)
|
||||
return (left + right) / 2, (top + bottom) / 2
|
||||
|
||||
|
||||
# Each pass also reads noise where its polarity doesn't apply (the gaps between dark letters look like light text),
|
||||
@@ -89,23 +94,35 @@ def phrases(words):
|
||||
return sorted(result, key=lambda p: (min(w["t"] for w in p), p[0]["l"]))
|
||||
|
||||
|
||||
# OCR can glue an icon to the first word, or punctuation or the text cursor to the last,
|
||||
# so long edge words match by suffix or prefix.
|
||||
def word_matches(word, k, first, last):
|
||||
if word == k:
|
||||
return True
|
||||
if len(k) < 4:
|
||||
return False
|
||||
return (first and word.endswith(k)) or (last and word.startswith(k))
|
||||
# OCR can split or join words ("#ci_group" read as "#ci_ group"), so words are compared by their joined characters.
|
||||
# It can also glue an icon to the first word, or punctuation or the text cursor to the last, so the text may start
|
||||
# inside the first word and end inside the last when at least 4 of its characters are in that word.
|
||||
def span_matches(words, target):
|
||||
keys = [w["key"] for w in words]
|
||||
joined = "".join(keys)
|
||||
start = joined.find(target)
|
||||
while start >= 0:
|
||||
end = start + len(target)
|
||||
first_ok = start == 0 or len(keys[0]) - start >= 4
|
||||
last_ok = end == len(joined) or len(keys[-1]) - (len(joined) - end) >= 4
|
||||
if start < len(keys[0]) and end > len(joined) - len(keys[-1]) and first_ok and last_ok:
|
||||
return True
|
||||
start = joined.find(target, start + 1)
|
||||
return False
|
||||
|
||||
|
||||
def find(all_phrases, text):
|
||||
keys = [k for k in map(key, text.split()) if k]
|
||||
n = len(keys)
|
||||
target = key(text)
|
||||
if not target:
|
||||
return None
|
||||
for phrase in all_phrases:
|
||||
for i in range(len(phrase) - n + 1):
|
||||
if all(word_matches(phrase[i + j]["key"], k, j == 0, j == n - 1) for j, k in enumerate(keys)):
|
||||
return centre(phrase[i:i + n])
|
||||
if "".join(w["key"] for w in phrase) == target:
|
||||
return phrase
|
||||
for phrase in all_phrases:
|
||||
for i in range(len(phrase)):
|
||||
for j in range(i + 1, len(phrase) + 1):
|
||||
if span_matches(phrase[i:j], target):
|
||||
return phrase[i:j]
|
||||
return None
|
||||
|
||||
|
||||
@@ -117,10 +134,10 @@ def main():
|
||||
x, y = centre(phrase)
|
||||
print(f"{round(x)},{round(y)} {' '.join(w['text'] for w in phrase)}")
|
||||
elif command == "find":
|
||||
found = find(on_screen, args[0])
|
||||
if not found:
|
||||
words = find(on_screen, args[0])
|
||||
if not words:
|
||||
sys.exit(1)
|
||||
print(f"{round(found[0])} {round(found[1])}")
|
||||
print(" ".join(str(round(v)) for v in centre(words) + box(words)))
|
||||
else:
|
||||
sys.exit(f"unknown command: {command}")
|
||||
|
||||
|
||||
@@ -8,12 +8,18 @@
|
||||
# ui start <app dir> start Xvfb, openbox and <app dir>/bin/simplex, and wait for its window
|
||||
# ui shot <name> save UI_OUT/NN-<name>.png and NN-<name>.txt (OCR's input and output in UI_OUT/ocr), and print the text on screen
|
||||
# ui text print the text on screen, one phrase per line after its centre x,y
|
||||
# ui wait <text> [secs] wait up to secs (30) until the text is on screen (case and punctuation ignored)
|
||||
# ui find <text> print "x y left top right bottom", the text's centre and box; fail if it is not on screen
|
||||
# ui wait <text> [secs] wait up to secs (30) until the text is on screen
|
||||
# ui click <x> <y> click at x,y
|
||||
# ui click-text <text> [secs] wait up to secs (20) for the text and click its centre
|
||||
# ui avatar <text> [colour] print how many pixels of the profile image left of a list row's name (the text) stand out
|
||||
# from the background, and their mean colour, saving the area as UI_OUT/NN-avatar-<text>.png;
|
||||
# with red, green or blue, fail unless that colour dominates
|
||||
# ui type <text> type into the focused field
|
||||
# ui key <key>... press keys named as in xdotool (Return, Escape, ctrl+a)
|
||||
# ui stop stop the app, openbox and Xvfb
|
||||
#
|
||||
# Text is compared ignoring case, spaces and punctuation; a phrase that is exactly the text wins over one that contains it.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -102,7 +108,7 @@ screen_ocr() {
|
||||
ocr "$state/ocr/screen.png" "$@"
|
||||
}
|
||||
|
||||
# prints "x y" of the text once it is on screen
|
||||
# prints "x y left top right bottom" of the text once it is on screen
|
||||
wait_for() {
|
||||
local text=$1 deadline=$((SECONDS + $2))
|
||||
until screen_ocr find "$text"; do
|
||||
@@ -111,10 +117,16 @@ wait_for() {
|
||||
done
|
||||
}
|
||||
|
||||
shot() {
|
||||
local n f
|
||||
# prints UI_OUT/NN-<name>, numbered after the files already saved
|
||||
next_file() {
|
||||
local n
|
||||
n=$(find "$UI_OUT" -maxdepth 1 -name '[0-9][0-9]-*.png' | wc -l)
|
||||
f=$(printf '%s/%02d-%s' "$UI_OUT" $((n + 1)) "$1")
|
||||
printf '%s/%02d-%s' "$UI_OUT" $((n + 1)) "$1"
|
||||
}
|
||||
|
||||
shot() {
|
||||
local f
|
||||
f=$(next_file "$1")
|
||||
import -window root "$f.png"
|
||||
ocr "$f.png" lines > "$f.txt"
|
||||
mkdir -p "$UI_OUT/ocr"
|
||||
@@ -133,6 +145,25 @@ click() {
|
||||
sleep 1
|
||||
}
|
||||
|
||||
# A list row's profile image is 42 px square, 8 px left of the row's name, and starts a little below the name's top:
|
||||
# the area allows for OCR's box being a few pixels off, and leaves out the rows above and below.
|
||||
avatar() {
|
||||
local text=$1 colour=${2:-} pos x y l t r b f stats
|
||||
if ! pos=$(screen_ocr find "$text"); then
|
||||
log "avatar \"$text\": not on screen"
|
||||
return 1
|
||||
fi
|
||||
read -r x y l t r b <<< "$pos"
|
||||
f=$(next_file "avatar-$(tr -cd 'a-zA-Z0-9_' <<< "$text")")
|
||||
convert "$state/ocr/screen.png" -crop "52x64+$((l > 56 ? l - 56 : 0))+$((t > 6 ? t - 6 : 0))" +repage "$f.png"
|
||||
convert "$f.png" -depth 8 "$state/ocr/avatar.ppm"
|
||||
if ! stats=$(python3 "$here/screen-colour.py" "$state/ocr/avatar.ppm" ${colour:+"$colour"}); then
|
||||
log "avatar \"$text\": $stats, not $colour"
|
||||
return 1
|
||||
fi
|
||||
log "avatar \"$text\": $stats${colour:+, $colour}"
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
start)
|
||||
log "start $2"
|
||||
@@ -140,13 +171,22 @@ case "${1:-}" in
|
||||
;;
|
||||
shot) shot "$2" ;;
|
||||
text) screen_ocr lines ;;
|
||||
find)
|
||||
if ! pos=$(screen_ocr find "$2"); then
|
||||
log "find \"$2\": not on screen" >&2
|
||||
exit 1
|
||||
fi
|
||||
log "find \"$2\": $pos" > /dev/null
|
||||
echo "$pos"
|
||||
;;
|
||||
wait)
|
||||
secs=${3:-30}
|
||||
if ! pos=$(wait_for "$2" "$secs"); then
|
||||
log "wait \"$2\": not on screen after $secs s"
|
||||
exit 1
|
||||
fi
|
||||
log "wait \"$2\": at ${pos/ /,}"
|
||||
read -r x y _ <<< "$pos"
|
||||
log "wait \"$2\": at $x,$y"
|
||||
;;
|
||||
click)
|
||||
click "$2" "$3"
|
||||
@@ -158,10 +198,11 @@ case "${1:-}" in
|
||||
log "click-text \"$2\": not on screen after $secs s"
|
||||
exit 1
|
||||
fi
|
||||
read -r x y <<< "$pos"
|
||||
read -r x y _ <<< "$pos"
|
||||
click "$x" "$y"
|
||||
log "click-text \"$2\" at $x,$y"
|
||||
;;
|
||||
avatar) avatar "$2" "${3:-}" ;;
|
||||
type)
|
||||
xdotool type --delay 50 -- "$2"
|
||||
log "type \"$2\""
|
||||
@@ -178,7 +219,7 @@ case "${1:-}" in
|
||||
log "stop"
|
||||
;;
|
||||
*)
|
||||
sed -n '3,16s/^# \{0,1\}//p' "$0"
|
||||
sed -n '3,/^$/s/^# \{0,1\}//p' "$0"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
Reference in new issue
Block a user