I'm trying to learn python by building an image detection bot for a 2d game.
The bot checks to see if it can find any one of a number of images within a cropped section of the game window. If it finds something successfully, this triggers the bot to move the mouse and click a button while logging what's been found into a text file.
I'm very new at this and this is very cobbled together code from a few different tutorials that I'm working my way through.
Have I made any obvious mistakes? Am I wasting significant resources/cycles with nonsense calls? Is there a much simpler way to detect images and send mouse clicks? And any other advice would be appreciated.
Please explain any suggestions rather than simply posting refactored code.
Working Code Below
For testing purposes change 'App'
to the literal name of an application, place some png
images of the application in C:\test\*.png
and make sure to change crop = haystack_img_resized.crop((360,22,600,65))
to a relevant co-ordiantes of your application window that contain your images.
import pyautogui as py
import glob
import cv2 as cv
import os
import numpy as np
import pygetwindow
import time
from PIL import ImageGrab
os.chdir(r'C:\test')
avoid = glob.glob(r"C:\test\*.png")
# Load Images
def loadImages(directory):
# Intialise empty array
image_list = []
# Add images to array
for i in directory:
img = cv.imread(i, cv.IMREAD_REDUCED_GRAYSCALE_2)
image_list.append((img, i))
return image_list
def videoLoop():
# Grab Window and find size
window = pygetwindow.getWindowsWithTitle('App')[0]
x1 = window.left
y1 = window.top
height = window.height
width = window.width
x2 = x1 + width
y2 = y1 + height
# Actual Video Loop, cropped down to the specific window,
# resized to 1/2 size, and converted to BGR for OpenCV
haystack_img = ImageGrab.grab(bbox=(x1, y1, x2, y2))
(width, height) = (haystack_img.width // 2, haystack_img.height // 2)
haystack_img_resized = haystack_img.resize((width, height))
crop = haystack_img_resized.crop((360,22,600,65))
haystack_img_np = np.array(crop)
haystack = cv.cvtColor(haystack_img_np, cv.COLOR_BGR2GRAY)
cv.imshow("Screen", haystack)
return haystack
def objectDetection(image_list, threshold, haystack, a):
# Object Detection
for x in range (0, a):
for i in image_list:
needle_img = i[0]
needle_name = i[1]
sliced_name = needle_name.split("\\")[-1]
result = cv.matchTemplate(haystack, needle_img, cv.TM_CCORR_NORMED)
# Get the best match position
min_val, max_val, min_loc, max_loc = cv.minMaxLoc(result)
# Define top left and bottom right and threshold
(H, W) = i[0].shape[:2]
top_left = max_loc
bottom_right = (top_left[0] + W, top_left[1] + H)
# If something has been detected click keep looking code
if max_val >= threshold:
cv.rectangle(haystack, top_left, bottom_right, 255, 2)
cv.imshow("Screen", haystack)
keep_looking(sliced_name)
if cv.waitKey(1) == ord('q'):
cv.destroyAllWindows()
break
def keep_looking(sliced_name):
py.moveTo(1715,1000,0.1)
py.sleep(0.01)
avoidLog(sliced_name)
py.click()
# Log details of what we've avoided function
def avoidLog(needle_name):
sliced_name = needle_name.split("\\")[-1]
skipTime = time.strftime("%d.%m.%Y - %H.%M.%S")
# Print Avoided Details to terminal
print("Keep Looking - Avoided: " + str(sliced_name) + " at: " + str(skipTime))
avoidLog = open('.Ships Avoided.txt', 'a')
avoidLog.write(("Sccessfully Avoided something") + " at: " + str(skipTime))
avoidLog.write("\n")
avoidLog.close()
#############################
#############################
###### MAIN BOT SCRIPT ######
#############################
#############################
# load images to detect
ships_to_avoid = loadImages(avoid)
while True:
window = videoLoop()
objectDetection(ships_to_avoid, 0.92, window, 10)