Skip to content

Instantly share code, notes, and snippets.

@hugs

hugs/1_valet-demo.py

Created August 30, 2024 17:41
Show Gist options
  • Save hugs/9a5b0643a675bfa791c7ef21be44f5d7 to your computer and use it in GitHub Desktop.
Save hugs/9a5b0643a675bfa791c7ef21be44f5d7 to your computer and use it in GitHub Desktop.
Download ZIP
Tapster Valet - Calculator Demo
Raw
1_valet-demo.py
# To run script and start an interactive session (REPL):
# $ python -i demo.py --host=http://localhost:5000
import cv2
import os
import sys
import shutil
import time
import json
import sys
import numpy as np
from datetime import datetime
import pytesseract
import requests
sys.path.append('/home/tapster/Projects/checkbox-client-python')
from checkbox import Ping, Config, Keyboard, Mouse, MouseKeys, Screenshot, Video
import argparse
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument('--host', help="host of the Checkbox API server\n" + \
"default host is http://checkbox.local:5000\n" + \
"example:\n python demo.py --host=http://example.local:5000")
args = parser.parse_args()
if args.host:
host = args.host
else:
host = "http://localhost:5000"
print("Using API host: %s" % host)
# Class init
c = Config(host)
k = Keyboard(host)
p = Ping(host)
m = Mouse(host)
mk = MouseKeys(host)
s = Screenshot(host)
v = Video(host)
# Shortcuts
ping = p.ping
Shift = 0x02
Tab = 0x2b
Escape = 0x29
Enter = 0x28
Backspace = 0x2a
Space = KEY_SPACE = 0x2c # Keyboard Spacebar
KEY_SEARCH = 0x54
KEY_A = 0x04
KEY_F = 0x09
KEY_N = 0x11
KEY_R = 0x15
KEY_J = 0x0d
MOD_LEFT_CONTROL = 0x01
MOD_LEFT_SHIFT = 0x02
MOD_LEFT_ALT = 0x04
MOD_LEFT_GUI = 0x08
KEY_F3 = 0x3c
KEY_HOME = 0x4a # Keyboard Home
KEY_PAGEUP = 0x4b # Keyboard Page Up
KEY_PAGEDOWN = 0x4e # Keyboard Page Down
KEY_DOWN = 0x51 # Keyboard Down Arrow
KEY_UP = 0x52 # Keyboard Up Arrow
image_dir = "images"
template_dir = os.path.join(image_dir, "templates")
image_counter = 1
network_connect_counter = 0
def unlock_phone():
m.home()
m.raw.move_by(400,1500)
time.sleep(.5)
m.raw.swipe_up()
m.home()
def wake():
k.press([Shift])
def home():
k.press([MOD_LEFT_GUI], Enter)
def mouse_home_bottom_left():
m.home()
m.raw.move_by(2000,3000)
m.raw.move_by(2000,3000)
m.raw.move_by(0,-100)
m.raw.move_by(-100,0)
def recent_apps():
k.press([MOD_LEFT_ALT, MOD_LEFT_CONTROL], KEY_R)
def notifications():
k.press([MOD_LEFT_GUI], KEY_N)
def previous_app():
k.press([MOD_LEFT_ALT, MOD_LEFT_SHIFT], Tab)
def show_apps():
k.press([MOD_LEFT_ALT], KEY_A)
def show_open_apps():
k.press([MOD_LEFT_GUI], Tab)
def search():
k.press([MOD_LEFT_GUI], KEY_F)
def get_screenshot():
global image_counter
k.press([Shift])
time.sleep(1)
filepath = image_dir + "/image-%004d.png" % image_counter
s.get_screenshot(filepath, format="gray")
image_counter += 1
return filepath
def look_for2(template_filename = "", description = "pattern", timeout=5, threshold=15, maskPoints=None):
global image_counter
print(" looking for %s..." % description, end="")
sys.stdout.flush()
MIN_MATCH_COUNT = threshold
for i in range(timeout):
screenshot_path = get_screenshot()
screenshot = cv2.imread(screenshot_path)
gray = cv2.cvtColor(screenshot, cv2.COLOR_BGR2GRAY)
blur = cv2.GaussianBlur(gray, (3, 3), 3)
thresh = cv2.adaptiveThreshold(blur, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV, 7, -2)
if maskPoints:
print("\n using mask to find image...")
# Create a mask
mask = np.zeros(thresh.shape[:2], np.uint8)
# Get points
(x1,y1), (x2,y2) = maskPoints
mask[y1:y2, x1:x2] = 255
# Compute the bitwise AND using the mask
thresh = cv2.bitwise_and(thresh, thresh, mask = mask)
template_path = os.path.join(template_dir, template_filename)
template = cv2.imread(template_path)
gray_template = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)
# Initiate SIFT detector
sift = cv2.SIFT_create()
# find the keypoints and descriptors with SIFT
kp1, des1 = sift.detectAndCompute(gray_template,None)
kp2, des2 = sift.detectAndCompute(thresh,None)
FLANN_INDEX_KDTREE = 1
index_params = dict(algorithm = FLANN_INDEX_KDTREE, trees = 5)
search_params = dict(checks = 50)
flann = cv2.FlannBasedMatcher(index_params, search_params)
#print()
#print("des1 is None: ", (type(des1)== type(None)))
#print("des2 is None: ", (type(des2)== type(None)))
if ( (type(des1)== type(None)) or (type(des2)== type(None))):
print()
print(" invalid screenshot")
print(" still looking for %s" % description)
print(".", end="")
sys.stdout.flush()
time.sleep(1)
continue
matches = flann.knnMatch(des1, des2, k=2)
# store all the good matches as per Lowe's ratio test.
good = []
for m,n in matches:
if m.distance < 0.7*n.distance:
good.append(m)
print("\n good matches found: ", len(good))
sys.stdout.flush()
if len(good) >= MIN_MATCH_COUNT:
src_pts = np.float32([ kp1[m.queryIdx].pt for m in good ]).reshape(-1,1,2)
dst_pts = np.float32([ kp2[m.trainIdx].pt for m in good ]).reshape(-1,1,2)
M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC,5.0)
matchesMask = mask.ravel().tolist()
h,w = gray_template.shape
pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
#if type(M) != type(None):
# dst = cv2.perspectiveTransform(pts,M)
#thresh = cv2.polylines(thresh, [np.int32(dst)], True, 255, 3, cv2.LINE_AA)
#thresh = cv2.polylines(thresh, [np.int32(dst)], True, (0,0,0), 3, cv2.LINE_AA)
print(" found %s..." % description)
sys.stdout.flush()
draw_params = dict(matchColor = (0,255,0), # draw matches in green color
singlePointColor = None,
matchesMask = matchesMask, # draw only inliers
flags = 2)
img3 = cv2.drawMatches(gray_template,kp1,thresh,kp2,good,None,**draw_params)
#img4 = cv2.drawMatches(gray_template,kp1,screenshot,kp2,good,None,**draw_params)
#if type(M) != type(None):
# # Draw bounding box in Red
# if len(good) > MIN_MATCH_COUNT:
# dst += (w, 0) # adding offset
# img3 = cv2.polylines(img3, [np.int32(dst)], True, (0,0,255), 3, cv2.LINE_AA)
# #img4 = cv2.polylines(img4, [np.int32(dst)], True, (0,0,255), 3, cv2.LINE_AA)
img4 = cv2.hconcat([img3, screenshot])
cv2.imwrite(screenshot_path, img4)
#cv2.imwrite(image_dir + ("/image-%004d-screenshot.png" % (image_counter-1)), img4)
print(" image_counter: %s" % (image_counter-1))
return True
else:
print(" still looking for %s" % description)
print(" not enough matches are found - {}/{}".format(len(good), MIN_MATCH_COUNT) )
matchesMask = None
print(".", end="")
sys.stdout.flush()
time.sleep(1)
continue
print("")
return False
def look_for_text(template_filename = "", text = "pattern", invert=False, timeout=5, threshold=15, maskPoints=None):
global image_counter
print(' looking for "%s"...' % text, end="")
sys.stdout.flush()
MIN_MATCH_COUNT = threshold
for i in range(timeout):
screenshot_path = get_screenshot()
screenshot = cv2.imread(screenshot_path)
gray = cv2.cvtColor(screenshot, cv2.COLOR_BGR2GRAY)
# OCR works best when the text is black on a white background.
# Sometimes you need to invert the image...
if invert == True:
gray = cv2.bitwise_not(gray)
#blur = cv2.GaussianBlur(gray, (3, 3), 3)
#thresh = cv2.adaptiveThreshold(blur, 255, cv2.ADAPTIVE_THRESH_GAUSSIAN_C, cv2.THRESH_BINARY_INV, 7, -2)
if maskPoints:
print("\n using mask to find image...")
# Create a mask
mask = np.zeros(gray.shape[:2], np.uint8)
# Get points
(x1,y1), (x2,y2) = maskPoints
mask[y1:y2, x1:x2] = 255
# Compute the bitwise AND using the mask
gray = cv2.bitwise_and(gray, gray, mask = mask)
#template_path = os.path.join(template_dir, template_filename)
#template = cv2.imread(template_path)
#gray_template = cv2.cvtColor(template, cv2.COLOR_BGR2GRAY)
# Initiate OCR search
found_text = pytesseract.image_to_string(gray).strip()
print('\n OCR text found: "%s"' % found_text)
sys.stdout.flush()
img3 = cv2.rectangle(gray, maskPoints[0], maskPoints[1], (255,255,255), 3)
img4 = cv2.hconcat([img3, cv2.cvtColor(screenshot, cv2.COLOR_BGR2GRAY)])
cv2.imwrite(image_dir + ("/image-%004d.png" % (image_counter-1)), img4)
if text in found_text:
#if True in [text in item for item in found_text]:
#src_pts = np.float32([ kp1[m.queryIdx].pt for m in good ]).reshape(-1,1,2)
#dst_pts = np.float32([ kp2[m.trainIdx].pt for m in good ]).reshape(-1,1,2)
#M, mask = cv2.findHomography(src_pts, dst_pts, cv2.RANSAC,5.0)
#matchesMask = mask.ravel().tolist()
#h,w = gray_template.shape
#pts = np.float32([ [0,0],[0,h-1],[w-1,h-1],[w-1,0] ]).reshape(-1,1,2)
#dst = cv2.perspectiveTransform(pts,M)
#print("dts: ", dst)
#thresh = cv2.polylines(thresh, [np.int32(dst)], True, 255, 3, cv2.LINE_AA)
#thresh = cv2.polylines(thresh, [np.int32(dst)], True, (0,0,0), 3, cv2.LINE_AA)
print(" found %s..." % text)
sys.stdout.flush()
#draw_params = dict(matchColor = (0,255,0), # draw matches in green color
# singlePointColor = None,
# matchesMask = matchesMask, # draw only inliers
# flags = 2)
#img3 = cv2.drawMatches(gray_template,kp1,thresh,kp2,good,None,**draw_params)
#img4 = cv2.drawMatches(gray_template,kp1,screenshot,kp2,good,None,**draw_params)
# Draw bounding box in Red
#if len(good) > MIN_MATCH_COUNT:
# dst += (w, 0) # adding offset
# img3 = cv2.polylines(img3, [np.int32(dst)], True, (0,0,255), 3, cv2.LINE_AA)
#img4 = cv2.polylines(img4, [np.int32(dst)], True, (0,0,255), 3, cv2.LINE_AA)
#img4 = cv2.hconcat([img3, screenshot])
#cv2.imwrite(screenshot_path, img4)
#cv2.imwrite(image_dir + ("/image-%004d-text-found.png" % (image_counter-1)), screenshot)
print(" image_counter: %s" % (image_counter-1))
return True
else:
print(" still looking for %s" % text)
print(".", end="")
sys.stdout.flush()
time.sleep(1)
continue
print("")
return False
######################################
def find_1():
print('Look for "1" button...')
result = look_for2("1.png", '"1"', timeout=3, threshold=20)
if result == False:
raise Exception("Image not found")
def find_1_mask():
print('Look for "1" button using a mask...')
result = look_for2("1.png", '"1"', timeout=3, threshold=20, maskPoints= ((5, 462), (78, 527)))
if result == False:
raise Exception("Image not found")
######################################
def find_plus():
print('Look for "+" button...')
result = look_for2("+.png", '"+"', timeout=3, threshold=13)
if result == False:
raise Exception("Image not found")
def find_plus_mask():
print('Look for "+" button using a mask...')
result = look_for2("+.png", '"+"', timeout=3, threshold=13, maskPoints= ((217, 460), (293, 529)))
if result == False:
raise Exception("Image not found")
######################################
def find_2():
print('Look for "2" button...')
result = look_for2("2.png", '"2"', timeout=3, threshold=13)
if result == False:
raise Exception("Image not found")
def find_2_mask():
print('Look for "2" button using a mask...')
result = look_for2("2.png", '"2"', timeout=3, threshold=13, maskPoints= ((76, 459), (149, 529)))
if result == False:
raise Exception("Image not found")
######################################
def find_equals():
print('Look for "=" button...')
result = look_for2("equals.png", '"equals"', timeout=3, threshold=7)
if result == False:
raise Exception("Image not found")
def find_equals_mask():
print('Look for "=" button using a mask...')
result = look_for2("equals.png", '"equals"', timeout=3, threshold=6, maskPoints= ((218, 531), (291, 602)))
if result == False:
raise Exception("Image not found")
def find_equals_text_mask():
print('Look for "=" button using OCR and a mask')
result = look_for_text(text="=", timeout=5, invert=True, maskPoints= ((218, 531), (291, 602)))
if result == False:
raise Exception("Image not found")
######################################
def find_answer():
print('Look for answer using OCR...')
result = look_for_text(text="34", timeout=5, maskPoints= ((190, 170), (285, 240)))
if result == False:
raise Exception("Image not found")
######################################
if __name__ == '__main__':
print("Valet Demo")
sys.stdout.flush()
m.home()
get_screenshot()
find_1()
find_1_mask()
# Click 1
m.raw.move_by(60,1370)
m.click()
find_2()
find_2_mask()
# Click 2
m.raw.move_by(200,0)
m.click()
find_plus()
find_plus_mask()
# Click +
m.raw.move_by(410,0)
m.click()
time.sleep(2)
# Click 2
m.raw.move_by(-410,0)
m.click()
m.click()
find_equals()
# Click =
m.raw.move_by(390,190)
m.click()
# Click equals
find_answer()
print("Done")
Raw
2_sample-output.txt
(env) tapster@valet-link:~/Projects/valet-demo $ python demo.py
Using API host: http://localhost:5000
Valet Demo
Look for "1" button...
looking for "1"...
good matches found: 20
found "1"...
image_counter: 2
Look for "1" button using a mask...
looking for "1"...
using mask to find image...
good matches found: 23
found "1"...
image_counter: 3
Look for "2" button...
looking for "2"...
good matches found: 29
found "2"...
image_counter: 4
Look for "2" button using a mask...
looking for "2"...
using mask to find image...
good matches found: 37
found "2"...
image_counter: 5
Look for "+" button...
looking for "+"...
good matches found: 15
found "+"...
image_counter: 6
Look for "+" button using a mask...
looking for "+"...
using mask to find image...
good matches found: 16
found "+"...
image_counter: 7
Look for "=" button...
looking for "equals"...
good matches found: 7
found "equals"...
image_counter: 8
Look for answer using OCR...
looking for "34"...
using mask to find image...
OCR text found: "34"
found 34...
image_counter: 9
Done
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment