From 7523ff6f9b3cbf2b476f987f80c84d711c70dd4c Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Sun, 24 Jan 2021 20:24:35 -0500 Subject: [PATCH 01/44] Update predict.py --- predict.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/predict.py b/predict.py index 1e2f4a4..e851b57 100755 --- a/predict.py +++ b/predict.py @@ -3,6 +3,7 @@ from scipy.misc import imresize def predict(model, X): - X = imresize(X, (150, 150, 3)).astype('float32')/255. + X = imresize(X, (150, 150, 3)).astype('float32') Y = model.predict(X.reshape(1,150,150,3)) + Y = Y.argmax() return Y From 30a0fffd682d01dda03e04150d2b4088cf1f3d53 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Sun, 24 Jan 2021 20:24:51 -0500 Subject: [PATCH 02/44] Update ai.py --- ai.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/ai.py b/ai.py index c99cde1..d13e0df 100755 --- a/ai.py +++ b/ai.py @@ -1,3 +1,4 @@ + # Arda Mavi import os import platform @@ -9,6 +10,8 @@ from game_control import * from keras.models import model_from_json +import pickle + def main(): # Get Model: model_file = open('Data/Model/model.json', 'r') @@ -18,7 +21,9 @@ def main(): model.load_weights("Data/Model/weights.h5") print('AI start now!') - + with open('listfile.data', 'rb') as filehandle: + # read the data as binary data stream + placesList = pickle.load(filehandle) while 1: # Get screenshot: screen = ImageGrab.grab() @@ -26,12 +31,16 @@ def main(): screen = np.array(screen) # 4 channel(PNG) to 3 channel(JPG) Y = predict(model, screen) + print(Y) + Y = placesList[Y] + print(Y) + if Y == [0,0,0,0]: # Not action continue elif Y[0] == -1 and Y[1] == -1: # Only keyboard action. - key = get_key(Y[3]) + key = get_key(int(Y[3])) if Y[2] == 1: # Press: press(key) @@ -40,19 +49,19 @@ def main(): release(key) elif Y[2] == 0 and Y[3] == 0: # Only mouse action. - click(Y[0], Y[1]) + click(int(Y[0]), int(Y[1])) else: # Mouse and keyboard action. # Mouse: - click(Y[0], Y[1]) + click(int(Y[0]), int(Y[1])) # Keyboard: - key = get_key(Y[3]) + key = get_key(int(Y[3])) if Y[2] == 1: # Press: press(key) else: # Release: release(key) - + if __name__ == '__main__': main() From 4ac12708dab2bebf2235c51a1111d8e2cb39f634 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Sun, 24 Jan 2021 20:25:05 -0500 Subject: [PATCH 03/44] Update get_model.py --- get_model.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/get_model.py b/get_model.py index e50271f..44c9657 100755 --- a/get_model.py +++ b/get_model.py @@ -37,16 +37,16 @@ def get_model(): flat_1 = Flatten()(pooling_2) - fc = Dense(1280)(flat_1) + fc = Dense(512)(flat_1) fc = Activation('relu')(fc) fc = Dropout(0.5)(fc) - fc = Dense(4)(fc) + fc = Dense(39)(fc) - outputs = Activation('sigmoid')(fc) + outputs = Activation('softmax')(fc) model = Model(inputs=inputs, outputs=outputs) - model.compile(loss='categorical_crossentropy', optimizer='adadelta', metrics=['accuracy']) + model.compile(loss='binary_crossentropy', optimizer='adadelta', metrics=['accuracy']) return model From d6ce8cf92274982e1f95f841fd315daf85aadf79 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Sun, 24 Jan 2021 20:25:26 -0500 Subject: [PATCH 04/44] Update create_dataset.py --- create_dataset.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/create_dataset.py b/create_dataset.py index 1142edb..fafd208 100644 --- a/create_dataset.py +++ b/create_dataset.py @@ -1,4 +1,5 @@ # Arda Mavi +import torch import os import sys import platform @@ -13,8 +14,9 @@ from multiprocessing import Process from keras.models import model_from_json from pynput.mouse import Listener as mouse_listener -from pynput.keyboard import Listener as key_listener - +import string +from threading import * +import keyboard as kb def get_screenshot(): img = ImageGrab.grab() img = np.array(img)[:,:,:3] # Get first 3 channel from image as numpy array. @@ -56,15 +58,21 @@ def listen_keyboard(): if not os.path.exists(data_path): os.makedirs(data_path) - def on_press(key): - save_event_keyboard(data_path, 1, key) - - def on_release(key): - save_event_keyboard(data_path, 2, key) - - with key_listener(on_press=on_press, on_release=on_release) as listener: - listener.join() - + keys = list(string.ascii_lowercase) + keys.append("space_bar") + + def pressed(key): + while True: + continue_or_na = kb.is_pressed(key) + if continue_or_na: + save_event_keyboard(data_path, 1, key) + + + + + thread1 = [Thread(target=pressed, kwargs={"key":key}, daemon=True) for key in keys] + for thread in thread1: + thread.start() def main(): dataset_path = 'Data/Train_Data/' if not os.path.exists(dataset_path): From 7a46e63b2a776918090f0a5848ddbbe5417759b9 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Sun, 24 Jan 2021 20:25:51 -0500 Subject: [PATCH 05/44] Update get_dataset.py --- get_dataset.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/get_dataset.py b/get_dataset.py index 0f1ca04..394ea78 100755 --- a/get_dataset.py +++ b/get_dataset.py @@ -5,16 +5,24 @@ from scipy.misc import imread, imresize, imsave from sklearn.model_selection import train_test_split +import pickle def get_img(data_path): # Getting image array from path: img = imread(data_path) img = imresize(img, (150, 150, 3)) return img -def save_img(img, path): - imsave(path, img) +def save_img(path, img): + imsave(path+ '.jpg', img) return + +def before(value, a): + # Find first part and return slice before it. + pos_a = value.find(a) + if pos_a == -1: return "" + return value[0:pos_a] + def get_dataset(dataset_path='Data/Train_Data'): # Getting all data from data path: try: @@ -24,6 +32,7 @@ def get_dataset(dataset_path='Data/Train_Data'): labels = os.listdir(dataset_path) # Geting labels X = [] Y = [] + Z = [] count_categori = [-1,''] # For encode labels for label in labels: datas_path = dataset_path+'/'+label @@ -34,6 +43,8 @@ def get_dataset(dataset_path='Data/Train_Data'): if data != count_categori[1]: count_categori[0] += 1 count_categori[1] = data.split(',') + count_categori[1][3] = before(count_categori[1][3], '.jpg') + Z.append(count_categori[1]) Y.append(count_categori[0]) # Create dateset: X = np.array(X).astype('float32')/255. @@ -43,5 +54,8 @@ def get_dataset(dataset_path='Data/Train_Data'): os.makedirs('Data/npy_train_data/') np.save('Data/npy_train_data/X.npy', X) np.save('Data/npy_train_data/Y.npy', Y) + with open('listfile.data', 'wb') as filehandle: + # store the data as binary data stream + pickle.dump(Z, filehandle) X, X_test, Y, Y_test = train_test_split(X, Y, test_size=0.1, random_state=42) return X, X_test, Y, Y_test From cd04260e875f449962bc297d31f8e647321b8b38 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Sun, 24 Jan 2021 20:26:35 -0500 Subject: [PATCH 06/44] Update train.py --- train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/train.py b/train.py index f97d886..22c78ed 100755 --- a/train.py +++ b/train.py @@ -5,8 +5,8 @@ from get_model import get_model, save_model from keras.callbacks import ModelCheckpoint, TensorBoard -epochs = 100 -batch_size = 5 +epochs = 10 +batch_size = 32 def train_model(model, X, X_test, Y, Y_test): checkpoints = [] From 56d4c2bd8f2d291d76aa117aff54342a62c8daf2 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Sun, 24 Jan 2021 20:27:04 -0500 Subject: [PATCH 07/44] Update game_control.py --- game_control.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/game_control.py b/game_control.py index a37b8a9..b61dbc5 100755 --- a/game_control.py +++ b/game_control.py @@ -4,13 +4,21 @@ # For encoding keyboard keys: def get_keys(): - return ['\t', '\n', '\r', ' ', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e','f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', 'accept', 'add', 'alt', 'altleft', 'altright', 'apps', 'backspace', 'browserback', 'browserfavorites', 'browserforward', 'browserhome', 'browserrefresh', 'browsersearch', 'browserstop', 'capslock', 'clear', 'convert', 'ctrl', 'ctrlleft', 'ctrlright', 'decimal', 'del', 'delete', 'divide', 'down', 'end', 'enter', 'esc', 'escape', 'execute', 'f1', 'f10', 'f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f2', 'f20', 'f21', 'f22', 'f23', 'f24', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'final', 'fn', 'hanguel', 'hangul', 'hanja', 'help', 'home', 'insert', 'junja', 'kana', 'kanji', 'launchapp1', 'launchapp2', 'launchmail', 'launchmediaselect', 'left', 'modechange', 'multiply', 'nexttrack', 'nonconvert', 'num0', 'num1', 'num2', 'num3', 'num4', 'num5', 'num6', 'num7', 'num8', 'num9', 'numlock', 'pagedown', 'pageup', 'pause', 'pgdn', 'pgup', 'playpause', 'prevtrack', 'print', 'printscreen', 'prntscrn', 'prtsc', 'prtscr', 'return', 'right', 'scrolllock', 'select', 'separator', 'shift', 'shiftleft', 'shiftright', 'sleep', 'space', 'stop', 'subtract', 'tab', 'up', 'volumedown', 'volumemute', 'volumeup', 'win', 'winleft', 'winright', 'yen', 'command', 'option', 'optionleft', 'optionright'] + return ['\t', '\n', '\r', ' ', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e','f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', 'accept', 'add', 'alt', 'altleft', 'altright', 'apps', 'backspace', 'browserback', 'browserfavorites', 'browserforward', 'browserhome', 'browserrefresh', 'browsersearch', 'browserstop', 'capslock', 'clear', 'convert', 'ctrl', 'ctrlleft', 'ctrlright', 'decimal', 'del', 'delete', 'divide', 'down', 'end', 'enter', 'esc', 'escape', 'execute', 'f1', 'f10', 'f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f2', 'f20', 'f21', 'f22', 'f23', 'f24', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'final', 'fn', 'hanguel', 'hangul', 'hanja', 'help', 'home', 'insert', 'junja', 'kana', 'kanji', 'launchapp1', 'launchapp2', 'launchmail', 'launchmediaselect', 'left', 'modechange', 'multiply', 'nexttrack', 'nonconvert', 'num0', 'num1', 'num2', 'num3', 'num4', 'num5', 'num6', 'num7', 'num8', 'num9', 'numlock', 'pagedown', 'pageup', 'pause', 'pgdn', 'pgup', 'playpause', 'prevtrack', 'print', 'printscreen', 'prntscrn', 'prtsc', 'prtscr', 'return', 'right', 'scrolllock', 'select', 'separator', 'shift', 'shiftleft', 'shiftright', 'sleep', 'space_bar', 'stop', 'subtract', 'tab', 'up', 'volumedown', 'volumemute', 'volumeup', 'win', 'winleft', 'winright', 'yen', 'command', 'option', 'optionleft', 'optionright'] def get_key(id): return get_keys()[id] def get_id(key): - return get_keys().index(key) + try: + print('Key Pressed:',key.char,sep='') + return get_keys().index(key.char) + except AttributeError: + if (str(key)+'') not in get_keys(): + print((str(key)+''),' is not in list') + return + print('Key Pressed:',(str(key)+''),sep='') + return get_keys().index((str(key)+'')) keyboard = Keyboard() mouse = Mouse() From 51a95850694dce29a33800c420c0ff76f44114da Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Sun, 24 Jan 2021 20:27:35 -0500 Subject: [PATCH 08/44] Delete database_process.py --- database_process.py | 35 ----------------------------------- 1 file changed, 35 deletions(-) delete mode 100644 database_process.py diff --git a/database_process.py b/database_process.py deleted file mode 100644 index 73ec22b..0000000 --- a/database_process.py +++ /dev/null @@ -1,35 +0,0 @@ -# Arda Mavi -import os -import sqlite3 - -def set_sql_connect(database_name): - return sqlite3.connect(database_name) -def set_sql_cursor(database_connect): - return database_connect.cursor() - -def close_connect(vt): - if vt: - vt.commit() - vt.close - -def set_connect_and_cursor(path='Data/database.sqlite'): - vt = set_sql_connect(path) - db = set_sql_cursor(vt) - return vt, db - -def create_table(table_name, columns): - vt, db = set_connect_and_cursor() - db.execute("CREATE TABLE IF NOT EXISTS {0} ({1})".format(table_name, columns)) - close_connect(vt) - -def get_data(sql_command): - vt, db = set_connect_and_cursor() - db.execute(sql_command) - gelen_veri = db.fetchall() - close_connect(vt) - return gelen_veri - -def add_data(table, adding): - vt, db = set_connect_and_cursor() - db.execute("INSERT INTO '{0}' VALUES ({1})".format(table, adding)) - close_connect(vt) From ee3fe77755b343f338423e81fa62aea23f2501b7 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Sun, 24 Jan 2021 20:27:59 -0500 Subject: [PATCH 09/44] Update ai.py --- ai.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ai.py b/ai.py index d13e0df..e6d11ce 100755 --- a/ai.py +++ b/ai.py @@ -20,7 +20,7 @@ def main(): model = model_from_json(model) model.load_weights("Data/Model/weights.h5") - print('AI start now!') + print('AI starting now!') with open('listfile.data', 'rb') as filehandle: # read the data as binary data stream placesList = pickle.load(filehandle) From 9a4794c47e72b0785a9e4aeb10603b194bfadb68 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Mon, 25 Jan 2021 10:35:49 -0500 Subject: [PATCH 10/44] Update get_model.py --- get_model.py | 48 +++++++++++++++--------------------------------- 1 file changed, 15 insertions(+), 33 deletions(-) diff --git a/get_model.py b/get_model.py index 44c9657..a6d6dd9 100755 --- a/get_model.py +++ b/get_model.py @@ -1,8 +1,7 @@ # Arda Mavi import os -from keras.models import Model -from keras.optimizers import Adadelta -from keras.layers import Input, Conv2D, Activation, MaxPooling2D, Flatten, Dense, Dropout +from tensorflow.keras.models import Sequential +from tensorflow.keras.layers import Dense, Conv2D, MaxPool2D, Flatten, Dropout def save_model(model): if not os.path.exists('Data/Model/'): @@ -17,36 +16,19 @@ def save_model(model): def get_model(): - inputs = Input(shape=(150, 150, 3)) - - conv_1 = Conv2D(32, (3,3), strides=(1,1))(inputs) - act_1 = Activation('relu')(conv_1) - - conv_2 = Conv2D(64, (3,3), strides=(1,1))(act_1) - act_2 = Activation('relu')(conv_2) - - conv_3 = Conv2D(64, (3,3), strides=(1,1))(act_2) - act_3 = Activation('relu')(conv_3) - - pooling_1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(act_3) - - conv_4 = Conv2D(128, (3,3), strides=(1,1))(pooling_1) - act_4 = Activation('relu')(conv_4) - - pooling_2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(act_4) - - flat_1 = Flatten()(pooling_2) - - fc = Dense(512)(flat_1) - fc = Activation('relu')(fc) - fc = Dropout(0.5)(fc) - fc = Dense(39)(fc) - - outputs = Activation('softmax')(fc) - - model = Model(inputs=inputs, outputs=outputs) - - model.compile(loss='binary_crossentropy', optimizer='adadelta', metrics=['accuracy']) + model = Sequential() + + model.add(Conv2D(10, (3,3))) + model.add(MaxPool2D((2,2))) + for i in range(3): + model.add(Conv2D(16, (2, 2))) + model.add(MaxPool2D((2, 2))) + #model.add(Dropout(0.4)) + model.add(Flatten()) + model.add(Dense(256, activation='relu')) + model.add(Dense(output_neurons, activation='softmax')) + + model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) return model From 4c3213eedb34ce711725bbaf33ac62aa6ffa77aa Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Mon, 25 Jan 2021 10:38:10 -0500 Subject: [PATCH 11/44] Update game_control.py --- game_control.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/game_control.py b/game_control.py index b61dbc5..2308ab8 100755 --- a/game_control.py +++ b/game_control.py @@ -34,11 +34,16 @@ def scroll(x, y): def click(x, y): mouse.press(Button.left) + move(x, y) return # Keyboard: def press(key): - keyboard.press(key) + if key == "space_bar": + keyboard.press(Key.space) + return + else: + keyboard.press(key) return def release(key): From 32f568cd895a54eb7ada886c50a0bbbd8bda0799 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Mon, 25 Jan 2021 10:42:40 -0500 Subject: [PATCH 12/44] Update game_control.py --- game_control.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/game_control.py b/game_control.py index 2308ab8..4706c17 100755 --- a/game_control.py +++ b/game_control.py @@ -1,7 +1,7 @@ # Arda Mavi from pynput.mouse import Button, Controller as Mouse from pynput.keyboard import Controller as Keyboard - +import time # For encoding keyboard keys: def get_keys(): return ['\t', '\n', '\r', ' ', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e','f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', 'accept', 'add', 'alt', 'altleft', 'altright', 'apps', 'backspace', 'browserback', 'browserfavorites', 'browserforward', 'browserhome', 'browserrefresh', 'browsersearch', 'browserstop', 'capslock', 'clear', 'convert', 'ctrl', 'ctrlleft', 'ctrlright', 'decimal', 'del', 'delete', 'divide', 'down', 'end', 'enter', 'esc', 'escape', 'execute', 'f1', 'f10', 'f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f2', 'f20', 'f21', 'f22', 'f23', 'f24', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'final', 'fn', 'hanguel', 'hangul', 'hanja', 'help', 'home', 'insert', 'junja', 'kana', 'kanji', 'launchapp1', 'launchapp2', 'launchmail', 'launchmediaselect', 'left', 'modechange', 'multiply', 'nexttrack', 'nonconvert', 'num0', 'num1', 'num2', 'num3', 'num4', 'num5', 'num6', 'num7', 'num8', 'num9', 'numlock', 'pagedown', 'pageup', 'pause', 'pgdn', 'pgup', 'playpause', 'prevtrack', 'print', 'printscreen', 'prntscrn', 'prtsc', 'prtscr', 'return', 'right', 'scrolllock', 'select', 'separator', 'shift', 'shiftleft', 'shiftright', 'sleep', 'space_bar', 'stop', 'subtract', 'tab', 'up', 'volumedown', 'volumemute', 'volumeup', 'win', 'winleft', 'winright', 'yen', 'command', 'option', 'optionleft', 'optionright'] @@ -41,9 +41,13 @@ def click(x, y): def press(key): if key == "space_bar": keyboard.press(Key.space) + time.sleep(0.025) + keyboard.release(Key.space) return else: keyboard.press(key) + time.sleep(0.025) + keyboard.release(key) return def release(key): From 9b55584814fd98fe559020cb22cbdd2202c63c91 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Mon, 25 Jan 2021 10:43:31 -0500 Subject: [PATCH 13/44] Update train.py --- train.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/train.py b/train.py index 22c78ed..c9281fe 100755 --- a/train.py +++ b/train.py @@ -21,8 +21,8 @@ def train_model(model, X, X_test, Y, Y_test): return model def main(): - X, X_test, Y, Y_test = get_dataset() - model = get_model() + X, X_test, Y, Y_test, action_total = get_dataset() + model = get_model(action_total) model = train_model(model, X, X_test, Y, Y_test) save_model(model) return model From eba2b0cbac50139b2068228d5872e9f2a9f55634 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Mon, 25 Jan 2021 10:44:06 -0500 Subject: [PATCH 14/44] Update get_model.py --- get_model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/get_model.py b/get_model.py index a6d6dd9..6489040 100755 --- a/get_model.py +++ b/get_model.py @@ -15,7 +15,7 @@ def save_model(model): return -def get_model(): +def get_model(action_total): model = Sequential() model.add(Conv2D(10, (3,3))) @@ -26,7 +26,7 @@ def get_model(): #model.add(Dropout(0.4)) model.add(Flatten()) model.add(Dense(256, activation='relu')) - model.add(Dense(output_neurons, activation='softmax')) + model.add(Dense(action_total, activation='softmax')) model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) From 898943974e53556adb5fa40cd214abc1c041f0f1 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Mon, 25 Jan 2021 10:45:02 -0500 Subject: [PATCH 15/44] Update get_dataset.py --- get_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/get_dataset.py b/get_dataset.py index 394ea78..25064c1 100755 --- a/get_dataset.py +++ b/get_dataset.py @@ -58,4 +58,4 @@ def get_dataset(dataset_path='Data/Train_Data'): # store the data as binary data stream pickle.dump(Z, filehandle) X, X_test, Y, Y_test = train_test_split(X, Y, test_size=0.1, random_state=42) - return X, X_test, Y, Y_test + return X, X_test, Y, Y_test, count_categori[0]+1 From 101befcdda1066ea025302330f46181b0979a2f2 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Sun, 31 Jan 2021 14:43:17 -0500 Subject: [PATCH 16/44] Update create_dataset.py --- create_dataset.py | 1 - 1 file changed, 1 deletion(-) diff --git a/create_dataset.py b/create_dataset.py index fafd208..0eb5ac7 100644 --- a/create_dataset.py +++ b/create_dataset.py @@ -1,5 +1,4 @@ # Arda Mavi -import torch import os import sys import platform From 946e90eb605f97255c1303aee7140ce2cc50b3a3 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Sun, 31 Jan 2021 14:47:38 -0500 Subject: [PATCH 17/44] Update predict.py --- predict.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/predict.py b/predict.py index e851b57..5d32998 100755 --- a/predict.py +++ b/predict.py @@ -1,9 +1,9 @@ # Arda Mavi import numpy as np -from scipy.misc import imresize +import imageio def predict(model, X): - X = imresize(X, (150, 150, 3)).astype('float32') + X = imageio.imresize(X, (150, 150, 3)).astype('float32') /255. Y = model.predict(X.reshape(1,150,150,3)) Y = Y.argmax() return Y From 7a64b46004f13b1254bdb7f2648820257febe998 Mon Sep 17 00:00:00 2001 From: BunnieCodeX <89990981+BunnieCodeX@users.noreply.github.com> Date: Thu, 30 Sep 2021 12:28:43 +0530 Subject: [PATCH 18/44] Update game_control.py --- game_control.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/game_control.py b/game_control.py index 4706c17..ff20a17 100755 --- a/game_control.py +++ b/game_control.py @@ -1,6 +1,6 @@ # Arda Mavi from pynput.mouse import Button, Controller as Mouse -from pynput.keyboard import Controller as Keyboard +from pynput.keyboard import Key, Controller as Keyboard import time # For encoding keyboard keys: def get_keys(): From 50e6872eb68bca64bed6058b3dc7c6e68d1d5a50 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Wed, 10 Nov 2021 11:44:37 -0500 Subject: [PATCH 19/44] Update README.md --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 36e728e..71bc303 100644 --- a/README.md +++ b/README.md @@ -42,3 +42,8 @@ Deep Learning is a subfield of machine learning with neural networks inspired by - Run CMD and Input Command `pip3 install -r requirements.txt` ### This project is still being worked on ... + + +### TODO: +- Update/Allow CNN Model to train each output on more observations (For More Accurate and complicated Models) +- Update requirements.txt and Update Code for newer verstions of packages From 5cfccecb6c2f7b9525748fc927155f46663ba920 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Fri, 12 Nov 2021 12:46:52 -0500 Subject: [PATCH 20/44] Update get_dataset.py --- get_dataset.py | 66 +++++++++++++++++++++++++------------------------- 1 file changed, 33 insertions(+), 33 deletions(-) diff --git a/get_dataset.py b/get_dataset.py index 25064c1..3b81d05 100755 --- a/get_dataset.py +++ b/get_dataset.py @@ -1,5 +1,6 @@ # Arda Mavi import os +import sys import numpy as np from keras.utils import to_categorical from scipy.misc import imread, imresize, imsave @@ -13,7 +14,7 @@ def get_img(data_path): return img def save_img(path, img): - imsave(path+ '.jpg', img) + imsave(path + '.jpg', img) return @@ -25,37 +26,36 @@ def before(value, a): def get_dataset(dataset_path='Data/Train_Data'): # Getting all data from data path: - try: - X = np.load('Data/npy_train_data/X.npy') - Y = np.load('Data/npy_train_data/Y.npy') - except: - labels = os.listdir(dataset_path) # Geting labels - X = [] - Y = [] - Z = [] - count_categori = [-1,''] # For encode labels - for label in labels: - datas_path = dataset_path+'/'+label - for data in os.listdir(datas_path): - img = get_img(datas_path+'/'+data) - X.append(img) - # For encode labels: - if data != count_categori[1]: - count_categori[0] += 1 - count_categori[1] = data.split(',') - count_categori[1][3] = before(count_categori[1][3], '.jpg') + labels = os.listdir(dataset_path) # Geting labels + X = [] + Y = [] + Z = [] + count_categori = [-1,''] # For encode labels + for label in labels: + datas_path = dataset_path+'/'+label + for data in os.listdir(datas_path): + img = get_img(datas_path+'/'+data) + X.append(img) + # For encode labels: + current_choice = data.split(',') + del current_choice[4] + if current_choice != count_categori[1]: + count_categori[0] += 1 + count_categori[1] = current_choice + if count_categori[1] not in Z: Z.append(count_categori[1]) - Y.append(count_categori[0]) - # Create dateset: - X = np.array(X).astype('float32')/255. - Y = np.array(Y).astype('float32') - Y = to_categorical(Y, count_categori[0]+1) - if not os.path.exists('Data/npy_train_data/'): - os.makedirs('Data/npy_train_data/') - np.save('Data/npy_train_data/X.npy', X) - np.save('Data/npy_train_data/Y.npy', Y) - with open('listfile.data', 'wb') as filehandle: - # store the data as binary data stream - pickle.dump(Z, filehandle) - X, X_test, Y, Y_test = train_test_split(X, Y, test_size=0.1, random_state=42) + Y.append(count_categori[0]) + with open('listfile.data', 'wb') as filehandle: + # store the data as binary data stream + pickle.dump(Z, filehandle) + X = np.array(X).astype('float32')/255. + Y = np.array(Y).astype('float32') + #print(Y) + Y = to_categorical(Y, count_categori[0]+1) + #print(Y) + if not os.path.exists('Data/npy_train_data/'): + os.makedirs('Data/npy_train_data/') + np.save('Data/npy_train_data/X.npy', X) + np.save('Data/npy_train_data/Y.npy', Y) + X, X_test, Y, Y_test = train_test_split(X, Y, test_size=0.1) return X, X_test, Y, Y_test, count_categori[0]+1 From 72516267c3c5b612e16fef7cf59ebcfba1e2bdef Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Fri, 12 Nov 2021 12:47:05 -0500 Subject: [PATCH 21/44] Update game_control.py --- game_control.py | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/game_control.py b/game_control.py index ff20a17..5f7b955 100755 --- a/game_control.py +++ b/game_control.py @@ -2,9 +2,10 @@ from pynput.mouse import Button, Controller as Mouse from pynput.keyboard import Key, Controller as Keyboard import time +import win32api # For encoding keyboard keys: def get_keys(): - return ['\t', '\n', '\r', ' ', '!', '"', '#', '$', '%', '&', "'", '(', ')', '*', '+', ',', '-', '.', '/', '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', ':', ';', '<', '=', '>', '?', '@', '[', '\\', ']', '^', '_', '`', 'a', 'b', 'c', 'd', 'e','f', 'g', 'h', 'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', 's', 't', 'u', 'v', 'w', 'x', 'y', 'z', '{', '|', '}', '~', 'accept', 'add', 'alt', 'altleft', 'altright', 'apps', 'backspace', 'browserback', 'browserfavorites', 'browserforward', 'browserhome', 'browserrefresh', 'browsersearch', 'browserstop', 'capslock', 'clear', 'convert', 'ctrl', 'ctrlleft', 'ctrlright', 'decimal', 'del', 'delete', 'divide', 'down', 'end', 'enter', 'esc', 'escape', 'execute', 'f1', 'f10', 'f11', 'f12', 'f13', 'f14', 'f15', 'f16', 'f17', 'f18', 'f19', 'f2', 'f20', 'f21', 'f22', 'f23', 'f24', 'f3', 'f4', 'f5', 'f6', 'f7', 'f8', 'f9', 'final', 'fn', 'hanguel', 'hangul', 'hanja', 'help', 'home', 'insert', 'junja', 'kana', 'kanji', 'launchapp1', 'launchapp2', 'launchmail', 'launchmediaselect', 'left', 'modechange', 'multiply', 'nexttrack', 'nonconvert', 'num0', 'num1', 'num2', 'num3', 'num4', 'num5', 'num6', 'num7', 'num8', 'num9', 'numlock', 'pagedown', 'pageup', 'pause', 'pgdn', 'pgup', 'playpause', 'prevtrack', 'print', 'printscreen', 'prntscrn', 'prtsc', 'prtscr', 'return', 'right', 'scrolllock', 'select', 'separator', 'shift', 'shiftleft', 'shiftright', 'sleep', 'space_bar', 'stop', 'subtract', 'tab', 'up', 'volumedown', 'volumemute', 'volumeup', 'win', 'winleft', 'winright', 'yen', 'command', 'option', 'optionleft', 'optionright'] + return ['w', 'a', 's', 'd', 'r', 'f', 'g', 'c', 'Key.space', 'Key.shift'] def get_key(id): return get_keys()[id] @@ -13,10 +14,10 @@ def get_id(key): try: print('Key Pressed:',key.char,sep='') return get_keys().index(key.char) - except AttributeError: + except: if (str(key)+'') not in get_keys(): print((str(key)+''),' is not in list') - return + return 1000 print('Key Pressed:',(str(key)+''),sep='') return get_keys().index((str(key)+'')) @@ -25,7 +26,7 @@ def get_id(key): # Mouse: def move(x, y): - mouse.position = (x, y) + win32api.SetCursorPos((x,y)) return def scroll(x, y): @@ -39,17 +40,19 @@ def click(x, y): # Keyboard: def press(key): - if key == "space_bar": + if key == 'Key.shift': + keyboard.press(Key.shift) + elif key == 'Key.space': keyboard.press(Key.space) - time.sleep(0.025) - keyboard.release(Key.space) - return else: keyboard.press(key) - time.sleep(0.025) - keyboard.release(key) return def release(key): - keyboard.release(key) + if key == 'Key.shift': + keyboard.release(Key.shift) + elif key == 'Key.space': + keyboard.release(Key.space) + else: + keyboard.release(key) return From c472ca69f725bc2ccb53fac71c72ae1aa2318685 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Fri, 12 Nov 2021 12:47:27 -0500 Subject: [PATCH 22/44] Update ai.py --- ai.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/ai.py b/ai.py index e6d11ce..5462ec3 100755 --- a/ai.py +++ b/ai.py @@ -4,12 +4,12 @@ import platform import numpy as np from time import sleep -from PIL import ImageGrab +from PIL import Image from game_control import * from predict import predict from game_control import * from keras.models import model_from_json - +from mss import mss import pickle def main(): @@ -26,21 +26,24 @@ def main(): placesList = pickle.load(filehandle) while 1: # Get screenshot: - screen = ImageGrab.grab() - # Image to numpy array: - screen = np.array(screen) + with mss() as sct: + monitor = sct.monitors[1] + sct_img = sct.grab(monitor) + # Convert to PIL/Pillow Image + screen = Image.frombytes('RGB', sct_img.size, sct_img.bgra, 'raw', 'BGRX') + screen = np.array(screen)[:,:,:3] # Get first 3 channel from image as numpy array. # 4 channel(PNG) to 3 channel(JPG) Y = predict(model, screen) print(Y) Y = placesList[Y] + Y = [int(i) for i in Y] print(Y) - if Y == [0,0,0,0]: # Not action continue elif Y[0] == -1 and Y[1] == -1: # Only keyboard action. - key = get_key(int(Y[3])) + key = get_key(Y[3]) if Y[2] == 1: # Press: press(key) @@ -48,8 +51,9 @@ def main(): # Release: release(key) elif Y[2] == 0 and Y[3] == 0: - # Only mouse action. - click(int(Y[0]), int(Y[1])) + # Click action. + click(Y[0], Y[1]) + ''' else: # Mouse and keyboard action. # Mouse: @@ -62,6 +66,7 @@ def main(): else: # Release: release(key) - + ''' + time.sleep(0.005) if __name__ == '__main__': main() From dc4394c39513156be41071f1e412aff1ff622a63 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Fri, 12 Nov 2021 12:47:54 -0500 Subject: [PATCH 23/44] Update predict.py --- predict.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/predict.py b/predict.py index 5d32998..eb863d2 100755 --- a/predict.py +++ b/predict.py @@ -1,9 +1,9 @@ # Arda Mavi import numpy as np -import imageio +from scipy.misc import imresize def predict(model, X): - X = imageio.imresize(X, (150, 150, 3)).astype('float32') /255. + X = imresize(X, (150, 150, 3)).astype('float32') /255. Y = model.predict(X.reshape(1,150,150,3)) Y = Y.argmax() return Y From 7e0344485ae546490dbcd2e13f12b954c1d066f8 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Fri, 12 Nov 2021 12:48:08 -0500 Subject: [PATCH 24/44] Update create_dataset.py --- create_dataset.py | 58 ++++++++++++++++++++++++++--------------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/create_dataset.py b/create_dataset.py index 0eb5ac7..8d47c9e 100644 --- a/create_dataset.py +++ b/create_dataset.py @@ -1,10 +1,12 @@ # Arda Mavi +import torch import os import sys import platform import numpy as np from time import sleep -from PIL import ImageGrab +import time +from PIL import Image from game_control import * from predict import predict from scipy.misc import imresize @@ -13,24 +15,33 @@ from multiprocessing import Process from keras.models import model_from_json from pynput.mouse import Listener as mouse_listener +from pynput.keyboard import Listener as key_listener import string from threading import * import keyboard as kb +from mss import mss def get_screenshot(): - img = ImageGrab.grab() - img = np.array(img)[:,:,:3] # Get first 3 channel from image as numpy array. - img = imresize(img, (150, 150, 3)).astype('float32')/255. - return img + with mss() as sct: + monitor = sct.monitors[1] + sct_img = sct.grab(monitor) + # Convert to PIL/Pillow Image + img = Image.frombytes('RGB', sct_img.size, sct_img.bgra, 'raw', 'BGRX') + img = np.array(img)[:,:,:3] # Get first 3 channel from image as numpy array. + img = imresize(img, (150, 150, 3)).astype('float32')/255. + return img def save_event_keyboard(data_path, event, key): key = get_id(key) - data_path = data_path + '/-1,-1,{0},{1}'.format(event, key) - screenshot = get_screenshot() - save_img(data_path, screenshot) - return + if key == 1000: + return + else: + data_path = data_path + '/-1,-1,{0},{1},{2}'.format(event, key, time.time()) + screenshot = get_screenshot() + save_img(data_path, screenshot) + return def save_event_mouse(data_path, x, y): - data_path = data_path + '/{0},{1},0,0'.format(x, y) + data_path = data_path + '/{0},{1},0,0,{2}'.format(x, y, time.time()) screenshot = get_screenshot() save_img(data_path, screenshot) return @@ -45,7 +56,7 @@ def on_click(x, y, button, pressed): def on_scroll(x, y, dx, dy): pass - + def on_move(x, y): pass @@ -57,21 +68,15 @@ def listen_keyboard(): if not os.path.exists(data_path): os.makedirs(data_path) - keys = list(string.ascii_lowercase) - keys.append("space_bar") - - def pressed(key): - while True: - continue_or_na = kb.is_pressed(key) - if continue_or_na: - save_event_keyboard(data_path, 1, key) - - - - - thread1 = [Thread(target=pressed, kwargs={"key":key}, daemon=True) for key in keys] - for thread in thread1: - thread.start() + def on_press(key): + save_event_keyboard(data_path, 1, key) + + def on_release(key): + save_event_keyboard(data_path, 2, key) + + with key_listener(on_press=on_press, on_release=on_release) as listener: + listener.join() + def main(): dataset_path = 'Data/Train_Data/' if not os.path.exists(dataset_path): @@ -80,6 +85,7 @@ def main(): # Start to listening mouse with new process: Process(target=listen_mouse, args=()).start() listen_keyboard() + return if __name__ == '__main__': From 9514d03b02b3ee31bd8729415d5a6f0644befabf Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Fri, 12 Nov 2021 12:48:24 -0500 Subject: [PATCH 25/44] Update get_model.py --- get_model.py | 48 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/get_model.py b/get_model.py index 6489040..b2c8767 100755 --- a/get_model.py +++ b/get_model.py @@ -1,7 +1,8 @@ # Arda Mavi import os -from tensorflow.keras.models import Sequential -from tensorflow.keras.layers import Dense, Conv2D, MaxPool2D, Flatten, Dropout +from keras.models import Model +from keras.optimizers import Adadelta +from keras.layers import Input, Conv2D, Activation, MaxPooling2D, Flatten, Dense, Dropout def save_model(model): if not os.path.exists('Data/Model/'): @@ -16,19 +17,36 @@ def save_model(model): def get_model(action_total): - model = Sequential() - - model.add(Conv2D(10, (3,3))) - model.add(MaxPool2D((2,2))) - for i in range(3): - model.add(Conv2D(16, (2, 2))) - model.add(MaxPool2D((2, 2))) - #model.add(Dropout(0.4)) - model.add(Flatten()) - model.add(Dense(256, activation='relu')) - model.add(Dense(action_total, activation='softmax')) - - model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy']) + inputs = Input(shape=(150, 150, 3)) + + conv_1 = Conv2D(32, (3,3), strides=(1,1))(inputs) + #act_1 = Activation('relu')(conv_1) + + conv_2 = Conv2D(64, (3,3), strides=(1,1))(conv_1) + #act_2 = Activation('relu')(conv_2) + + conv_3 = Conv2D(64, (3,3), strides=(1,1))(conv_2) + #act_3 = Activation('relu')(conv_3) + + pooling_1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(conv_3) + + conv_4 = Conv2D(128, (3,3), strides=(1,1))(pooling_1) + #act_4 = Activation('relu')(conv_4) + + pooling_2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(conv_4) + + flat_1 = Flatten()(pooling_2) + + fc = Dense(1024)(flat_1) + fc = Activation('relu')(fc) + fc = Dropout(0.5)(fc) + fc = Dense(action_total)(fc) + + outputs = Activation('sigmoid')(fc) + + model = Model(inputs=inputs, outputs=outputs) + + model.compile(loss='binary_crossentropy', optimizer='adadelta', metrics=['accuracy']) return model From 61555bcafefac16d987fe13b122911d6c8877bc8 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Fri, 12 Nov 2021 12:48:47 -0500 Subject: [PATCH 26/44] Update train.py --- train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/train.py b/train.py index c9281fe..2ff566a 100755 --- a/train.py +++ b/train.py @@ -5,7 +5,7 @@ from get_model import get_model, save_model from keras.callbacks import ModelCheckpoint, TensorBoard -epochs = 10 +epochs = 15 batch_size = 32 def train_model(model, X, X_test, Y, Y_test): @@ -22,6 +22,7 @@ def train_model(model, X, X_test, Y, Y_test): def main(): X, X_test, Y, Y_test, action_total = get_dataset() + print(action_total) model = get_model(action_total) model = train_model(model, X, X_test, Y, Y_test) save_model(model) From 6c3d93399e9125e4def2032462ceaaeb2c3c18e2 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Fri, 12 Nov 2021 12:50:01 -0500 Subject: [PATCH 27/44] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 71bc303..6137f5a 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Deep Learning is a subfield of machine learning with neural networks inspired by 2. Run `python3 ai.py` command in terminal. ### Creating Training Dataset: +1. Change the desired kets in the game_control.py file's get keys function, by changing return keys to desired ones 1. Run `python3 create_dataset.py` command in terminal. 2. Play your desired game. 3. Stop `create_dataset` program with `Cntrl-C` in terminal. From 51a81fef51d10c647db23d365c4fc7df97eab417 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Fri, 12 Nov 2021 17:31:35 -0500 Subject: [PATCH 28/44] Update requirements.txt --- requirements.txt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 508a7f3..5c1ac1e 100755 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,8 @@ numpy scikit-learn scikit-image pillow -tensorflow -keras +tensorflow==1.9.0 +keras==2.2.0 +scipy==1.2.3 pynput h5py From de393d226cfc0514ffc21f843cbd5247645bd954 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Fri, 12 Nov 2021 17:32:48 -0500 Subject: [PATCH 29/44] Update README.md --- README.md | 5 ----- 1 file changed, 5 deletions(-) diff --git a/README.md b/README.md index 6137f5a..69c1e8b 100644 --- a/README.md +++ b/README.md @@ -43,8 +43,3 @@ Deep Learning is a subfield of machine learning with neural networks inspired by - Run CMD and Input Command `pip3 install -r requirements.txt` ### This project is still being worked on ... - - -### TODO: -- Update/Allow CNN Model to train each output on more observations (For More Accurate and complicated Models) -- Update requirements.txt and Update Code for newer verstions of packages From f261de43c8d3aa080e3192e67a51a1982cfb5337 Mon Sep 17 00:00:00 2001 From: benni347 Date: Tue, 1 Feb 2022 13:51:03 +0100 Subject: [PATCH 30/44] Refractored ai.py and updated the gitignore file. --- .gitignore | 165 ++++++++++++++++++++++++++++++++++++++++++++++++++++- ai.py | 97 +++++++++++++++++-------------- 2 files changed, 216 insertions(+), 46 deletions(-) diff --git a/.gitignore b/.gitignore index 56c94cc..0842a22 100755 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,162 @@ -__pycache__ -.DS_Store -Data + +# Created by https://www.toptal.com/developers/gitignore/api/python +# Edit at https://www.toptal.com/developers/gitignore?templates=python + +### Python ### +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# PyCharm +# JetBrains specific template is maintainted in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +# End of https://www.toptal.com/developers/gitignore/api/python + +.idea/ +*.png \ No newline at end of file diff --git a/ai.py b/ai.py index 5462ec3..7a6205b 100755 --- a/ai.py +++ b/ai.py @@ -1,72 +1,83 @@ +""" +This is the main file for the AI. -# Arda Mavi -import os -import platform +Author: Arda Mavi +""" +import pickle + +import time import numpy as np -from time import sleep from PIL import Image -from game_control import * -from predict import predict -from game_control import * from keras.models import model_from_json from mss import mss -import pickle + +from game_control import get_key, press, release, click +from predict import predict + def main(): - # Get Model: - model_file = open('Data/Model/model.json', 'r') - model = model_file.read() - model_file.close() + """ + Main function. + + :return: None + """ + with open("Data/Model/model.json", "r") as model_file: + model = model_file.read() model = model_from_json(model) model.load_weights("Data/Model/weights.h5") - print('AI starting now!') - with open('listfile.data', 'rb') as filehandle: + print("AI starting now!") + with open("listfile.data", "rb") as filehandle: # read the data as binary data stream - placesList = pickle.load(filehandle) + places_list = pickle.load(filehandle) while 1: # Get screenshot: with mss() as sct: monitor = sct.monitors[1] sct_img = sct.grab(monitor) # Convert to PIL/Pillow Image - screen = Image.frombytes('RGB', sct_img.size, sct_img.bgra, 'raw', 'BGRX') - screen = np.array(screen)[:,:,:3] # Get first 3 channel from image as numpy array. + screen = Image.frombytes("RGB", sct_img.size, sct_img.bgra, "raw", + "BGRX") + screen = np.array(screen)[ + :, :, :3 + ] # Get first 3 channel from image as numpy array. # 4 channel(PNG) to 3 channel(JPG) - Y = predict(model, screen) - print(Y) - Y = placesList[Y] - Y = [int(i) for i in Y] - print(Y) - if Y == [0,0,0,0]: + y_ai = predict(model, screen) + print(y_ai) + y_ai = places_list[y_ai] + y_ai = [int(i) for i in y_ai] + print(y_ai) + if y_ai == [0, 0, 0, 0]: # Not action continue - elif Y[0] == -1 and Y[1] == -1: + if y_ai[0] == -1 and y_ai[1] == -1: # Only keyboard action. - key = get_key(Y[3]) - if Y[2] == 1: + key = get_key(y_ai[3]) + if y_ai[2] == 1: # Press: press(key) else: # Release: release(key) - elif Y[2] == 0 and Y[3] == 0: + elif y_ai[2] == 0 and y_ai[3] == 0: # Click action. - click(Y[0], Y[1]) - ''' - else: - # Mouse and keyboard action. - # Mouse: - click(int(Y[0]), int(Y[1])) - # Keyboard: - key = get_key(int(Y[3])) - if Y[2] == 1: - # Press: - press(key) - else: - # Release: - release(key) - ''' + click(y_ai[0], y_ai[1]) + + # else: + # # Mouse and keyboard action. + # # Mouse: + # click(int(y_ai[0]), int(y_ai[1])) + # # Keyboard: + # key = get_key(int(y_ai[3])) + # if y_ai[2] == 1: + # # Press: + # press(key) + # else: + # # Release: + # release(key) + time.sleep(0.005) -if __name__ == '__main__': + + +if __name__ == "__main__": main() From 05ba17bbe08e930c42bc889278cbe8c23a9d9636 Mon Sep 17 00:00:00 2001 From: benni347 Date: Tue, 1 Feb 2022 14:14:25 +0100 Subject: [PATCH 31/44] Refractored create_databse.py --- create_dataset.py | 77 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 25 deletions(-) diff --git a/create_dataset.py b/create_dataset.py index 8d47c9e..2b55676 100644 --- a/create_dataset.py +++ b/create_dataset.py @@ -1,52 +1,71 @@ -# Arda Mavi -import torch +""" +This file will create a dataset of images and labels for training. + +Author: Arda Mavi +""" import os -import sys -import platform -import numpy as np -from time import sleep +from multiprocessing import Process import time + +import numpy as np from PIL import Image -from game_control import * -from predict import predict -from scipy.misc import imresize +from mss import mss +from numpy import size +from pynput.keyboard import Listener as key_listener +from pynput.mouse import Listener as mouse_listener + from game_control import get_id from get_dataset import save_img -from multiprocessing import Process -from keras.models import model_from_json -from pynput.mouse import Listener as mouse_listener -from pynput.keyboard import Listener as key_listener -import string -from threading import * -import keyboard as kb -from mss import mss + + def get_screenshot(): + """ + This function will get the screenshot of the game. + :return: num_array of the screenshot + """ with mss() as sct: monitor = sct.monitors[1] sct_img = sct.grab(monitor) # Convert to PIL/Pillow Image img = Image.frombytes('RGB', sct_img.size, sct_img.bgra, 'raw', 'BGRX') - img = np.array(img)[:,:,:3] # Get first 3 channel from image as numpy array. - img = imresize(img, (150, 150, 3)).astype('float32')/255. + img = np.array(img)[:, :, :3] # Get first 3 channel from image as numpy array. + # resize it with PIL, because scipy.misc.imresize is deprecated. + img = img(Image.fromarray(img).resize((size[0] * 4, size[1] * 4), resample=Image.BICUBIC)) + # img = imresize(img, (150, 150, 3)).astype('float32') / 255. return img + def save_event_keyboard(data_path, event, key): + """ + This function will save the event of the keyboard. + :param data_path: path to save the event + :param event: down or up + :param key: which key is pressed + """ key = get_id(key) - if key == 1000: - return - else: + if key != 1000: data_path = data_path + '/-1,-1,{0},{1},{2}'.format(event, key, time.time()) screenshot = get_screenshot() save_img(data_path, screenshot) - return + def save_event_mouse(data_path, x, y): + """ + This function will save the event of the mouse. + :param data_path: path to save the event + :param x: x coordinate + :param y: y coordinate + """ data_path = data_path + '/{0},{1},0,0,{2}'.format(x, y, time.time()) screenshot = get_screenshot() save_img(data_path, screenshot) - return + def listen_mouse(): + """ + This function will listen the mouse and save the event. + :return: None + """ data_path = 'Data/Train_Data/Mouse' if not os.path.exists(data_path): os.makedirs(data_path) @@ -63,7 +82,12 @@ def on_move(x, y): with mouse_listener(on_move=on_move, on_click=on_click, on_scroll=on_scroll) as listener: listener.join() + def listen_keyboard(): + """ + This function will listen the keyboard and save the event. + :return: None + """ data_path = 'Data/Train_Data/Keyboard' if not os.path.exists(data_path): os.makedirs(data_path) @@ -77,7 +101,11 @@ def on_release(key): with key_listener(on_press=on_press, on_release=on_release) as listener: listener.join() + def main(): + """ + This is the main function. + """ dataset_path = 'Data/Train_Data/' if not os.path.exists(dataset_path): os.makedirs(dataset_path) @@ -86,7 +114,6 @@ def main(): Process(target=listen_mouse, args=()).start() listen_keyboard() - return if __name__ == '__main__': main() From 6977d099a88496dea75684ef5f80d60f2b117e90 Mon Sep 17 00:00:00 2001 From: benni347 Date: Tue, 1 Feb 2022 14:37:10 +0100 Subject: [PATCH 32/44] Refractored game_control.py. Changed that we no longer use pynput for the keyboard events instead we are using pyautogui. --- game_control.py | 109 +++++++++++++++++++++++++++++++++-------------- requirements.txt | 1 + 2 files changed, 79 insertions(+), 31 deletions(-) diff --git a/game_control.py b/game_control.py index 5f7b955..0456d36 100755 --- a/game_control.py +++ b/game_control.py @@ -1,58 +1,105 @@ -# Arda Mavi -from pynput.mouse import Button, Controller as Mouse -from pynput.keyboard import Key, Controller as Keyboard -import time -import win32api +""" +This file contains the game control logic. + +Author: Arda Mavi +""" +import pyautogui + +from pynput.mouse import Controller as Mouse +from pynput.keyboard import Key + + # For encoding keyboard keys: def get_keys(): - return ['w', 'a', 's', 'd', 'r', 'f', 'g', 'c', 'Key.space', 'Key.shift'] + """ + Returns a list of all the keys that can be pressed. + :return: The list of keys. + """ + return ["w", "a", "s", "d", "r", "f", "g", "c", "Key.space", "Key.shift"] + + +def get_key(key_id): + """ + Returns the key that corresponds to the given key id. + :param key_id: Set the key id. + :return: the key that corresponds to the given key id. + """ + return get_keys()[key_id] -def get_key(id): - return get_keys()[id] def get_id(key): + """ + Returns the id of the given key. + :param key: The key. + :return: The id of the given key. + """ try: - print('Key Pressed:',key.char,sep='') + print("Key Pressed:", key.char, sep="") return get_keys().index(key.char) except: - if (str(key)+'') not in get_keys(): - print((str(key)+''),' is not in list') + if (str(key) + "") not in get_keys(): + print((str(key) + ""), " is not in list") return 1000 - print('Key Pressed:',(str(key)+''),sep='') - return get_keys().index((str(key)+'')) + print("Key Pressed:", (str(key) + ""), sep="") + return get_keys().index((str(key) + "")) + -keyboard = Keyboard() mouse = Mouse() + # Mouse: def move(x, y): - win32api.SetCursorPos((x,y)) - return + """ + Moves the mouse to the given coordinates. + :param x: x coordinate. + :param y: y coordinate. + :return: None + """ + pyautogui.moveTo(x, y) + def scroll(x, y): + """ + Scrolls the mouse to the given coordinates. + :param x: The horizontal scroll. + :param y: The vertical scroll. + :TODO: Change the scroll function to scroll with pyautogui. + """ mouse.scroll(x, y) - return + def click(x, y): - mouse.press(Button.left) + """ + Clicks the mouse at the given coordinates. + :param x: The x coordinate. + :param y: The y coordinate. + """ move(x, y) - return + pyautogui.click() + # Keyboard: def press(key): - if key == 'Key.shift': - keyboard.press(Key.shift) - elif key == 'Key.space': - keyboard.press(Key.space) + """ + Presses the given key. + :param key: The key. + """ + if key in ["Key.shift", "shift"]: + pyautogui.keyDown("shift") + elif key in ["Key.space", "space"]: + pyautogui.keyDown("space") else: - keyboard.press(key) - return + pyautogui.keyDown(key) + def release(key): - if key == 'Key.shift': - keyboard.release(Key.shift) - elif key == 'Key.space': - keyboard.release(Key.space) + """ + Releases the given key. + :param key: the key. + """ + if key in ["Key.shift", "shift"]: + pyautogui.keyUp(Key.shift) + elif key in ["Key.space", "space"]: + pyautogui.keyUp(Key.space) else: - keyboard.release(key) - return + pyautogui.keyUp(key) diff --git a/requirements.txt b/requirements.txt index 5c1ac1e..32e104e 100755 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ keras==2.2.0 scipy==1.2.3 pynput h5py +pyautogui From acb96eb141594f10bea83781ebdede1357eddb28 Mon Sep 17 00:00:00 2001 From: benni347 Date: Tue, 1 Feb 2022 14:49:53 +0100 Subject: [PATCH 33/44] Refractored get_dataset.py and changed it from scipy.misc.imread and scipy.misc.imsave to imageio because they were depracted. --- get_dataset.py | 99 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 65 insertions(+), 34 deletions(-) diff --git a/get_dataset.py b/get_dataset.py index 3b81d05..0f93e53 100755 --- a/get_dataset.py +++ b/get_dataset.py @@ -1,61 +1,92 @@ -# Arda Mavi +""" +This file will get the dataset. + +Author: Arda Mavi +""" import os -import sys +import pickle + import numpy as np +from PIL import Image from keras.utils import to_categorical -from scipy.misc import imread, imresize, imsave +from numpy import size from sklearn.model_selection import train_test_split +from imageio import imread, imsave + -import pickle def get_img(data_path): - # Getting image array from path: + """ + Getting image array from path and read it. + :param data_path: The image_array path. + :return: the image + """ img = imread(data_path) - img = imresize(img, (150, 150, 3)) + # resize it with PIL, because scipy.misc.imresize is deprecated. + img = img(Image.fromarray(img).resize((size[0] * 4, size[1] * 4), + resample=Image.BICUBIC)) + # img = imresize(img, (150, 150, 3)) return img + def save_img(path, img): + """ + Saving image to path. + :param path: The path to save the image. + :param img: Which image to save. + """ imsave(path + '.jpg', img) - return def before(value, a): + """ + Find first part and slice it. + :param value: The value to slice. + :param a: The value to slice. + :return: value before a + """ # Find first part and return slice before it. pos_a = value.find(a) - if pos_a == -1: return "" - return value[0:pos_a] + if pos_a == -1: + return "" + return value[:pos_a] + def get_dataset(dataset_path='Data/Train_Data'): - # Getting all data from data path: - labels = os.listdir(dataset_path) # Geting labels - X = [] - Y = [] - Z = [] - count_categori = [-1,''] # For encode labels + """ + Getting all data from data path: + :param dataset_path: The path to the dataset. + :return: x_dataset, y_dataset, x_test, y_test + """ + labels = os.listdir(dataset_path) # Getting labels + x_dataset = [] + y_dataset = [] + z_dataset = [] + count_category = [-1, ''] # For encode labels for label in labels: - datas_path = dataset_path+'/'+label + datas_path = dataset_path + '/' + label for data in os.listdir(datas_path): - img = get_img(datas_path+'/'+data) - X.append(img) + img = get_img(datas_path + '/' + data) + x_dataset.append(img) # For encode labels: current_choice = data.split(',') del current_choice[4] - if current_choice != count_categori[1]: - count_categori[0] += 1 - count_categori[1] = current_choice - if count_categori[1] not in Z: - Z.append(count_categori[1]) - Y.append(count_categori[0]) + if current_choice != count_category[1]: + count_category[0] += 1 + count_category[1] = current_choice + if count_category[1] not in z_dataset: + z_dataset.append(count_category[1]) + y_dataset.append(count_category[0]) with open('listfile.data', 'wb') as filehandle: # store the data as binary data stream - pickle.dump(Z, filehandle) - X = np.array(X).astype('float32')/255. - Y = np.array(Y).astype('float32') - #print(Y) - Y = to_categorical(Y, count_categori[0]+1) - #print(Y) + pickle.dump(z_dataset, filehandle) + x_dataset = np.array(x_dataset).astype('float32') / 255. + y_dataset = np.array(y_dataset).astype('float32') + # print(y_dataset) + y_dataset = to_categorical(y_dataset, count_category[0] + 1) + # print(y_dataset) if not os.path.exists('Data/npy_train_data/'): os.makedirs('Data/npy_train_data/') - np.save('Data/npy_train_data/X.npy', X) - np.save('Data/npy_train_data/Y.npy', Y) - X, X_test, Y, Y_test = train_test_split(X, Y, test_size=0.1) - return X, X_test, Y, Y_test, count_categori[0]+1 + np.save('Data/npy_train_data/x_dataset.npy', x_dataset) + np.save('Data/npy_train_data/y_dataset.npy', y_dataset) + x_dataset, x_test, y_dataset, y_test = train_test_split(x_dataset, y_dataset, test_size=0.1) + return x_dataset, x_test, y_dataset, y_test, count_category[0] + 1 From de3be915f60bf00288f11f461f8d367d544c146c Mon Sep 17 00:00:00 2001 From: benni347 Date: Tue, 1 Feb 2022 14:57:22 +0100 Subject: [PATCH 34/44] Refractored it and made it more correct for pep8. --- get_model.py | 40 +++++++++++++++++++++++++++------------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/get_model.py b/get_model.py index b2c8767..249d04a 100755 --- a/get_model.py +++ b/get_model.py @@ -1,10 +1,19 @@ -# Arda Mavi +""" +This file will get the model from the database and return it to the user. +Author: Arda Mavi +""" import os -from keras.models import Model -from keras.optimizers import Adadelta + from keras.layers import Input, Conv2D, Activation, MaxPooling2D, Flatten, Dense, Dropout +from keras.models import Model + def save_model(model): + """ + This function will save the model to the database. + :param model: Which model to use. + :return: None + """ if not os.path.exists('Data/Model/'): os.makedirs('Data/Model/') model_json = model.to_json() @@ -13,25 +22,29 @@ def save_model(model): # serialize weights to HDF5 model.save_weights("Data/Model/weights.h5") print('Model and weights saved') - return def get_model(action_total): + """ + This function will get the model from the database. + :param action_total: Total number of actions. + :return: model + """ inputs = Input(shape=(150, 150, 3)) - conv_1 = Conv2D(32, (3,3), strides=(1,1))(inputs) - #act_1 = Activation('relu')(conv_1) + conv_1 = Conv2D(32, (3, 3), strides=(1, 1))(inputs) + # act_1 = Activation('relu')(conv_1) - conv_2 = Conv2D(64, (3,3), strides=(1,1))(conv_1) - #act_2 = Activation('relu')(conv_2) + conv_2 = Conv2D(64, (3, 3), strides=(1, 1))(conv_1) + # act_2 = Activation('relu')(conv_2) - conv_3 = Conv2D(64, (3,3), strides=(1,1))(conv_2) - #act_3 = Activation('relu')(conv_3) + conv_3 = Conv2D(64, (3, 3), strides=(1, 1))(conv_2) + # act_3 = Activation('relu')(conv_3) pooling_1 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(conv_3) - conv_4 = Conv2D(128, (3,3), strides=(1,1))(pooling_1) - #act_4 = Activation('relu')(conv_4) + conv_4 = Conv2D(128, (3, 3), strides=(1, 1))(pooling_1) + # act_4 = Activation('relu')(conv_4) pooling_2 = MaxPooling2D(pool_size=(2, 2), strides=(2, 2))(conv_4) @@ -50,5 +63,6 @@ def get_model(action_total): return model + if __name__ == '__main__': - save_model(get_model()) + save_model(get_model(10)) From 0146b33f774bcad43a097114e4a7d9925eb9323f Mon Sep 17 00:00:00 2001 From: benni347 Date: Tue, 1 Feb 2022 15:00:13 +0100 Subject: [PATCH 35/44] Refractored train.py --- train.py | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/train.py b/train.py index 2ff566a..35f0745 100755 --- a/train.py +++ b/train.py @@ -1,32 +1,60 @@ -# Arda Mavi +""" +This will train the model. +Author: Arda Mavi +""" import os -import numpy + +from keras.callbacks import ModelCheckpoint, TensorBoard + from get_dataset import get_dataset from get_model import get_model, save_model -from keras.callbacks import ModelCheckpoint, TensorBoard epochs = 15 batch_size = 32 -def train_model(model, X, X_test, Y, Y_test): - checkpoints = [] + +def train_model(model, x, x_test, y, y_test): + """ + This will train the model. + :param model: Which model to train + :param x: x + :param x_test: x_test + :param y: y + :param y_test: y_test + :return: model + """ if not os.path.exists('Data/Checkpoints/'): os.makedirs('Data/Checkpoints/') - checkpoints.append(ModelCheckpoint('Data/Checkpoints/best_weights.h5', monitor='val_loss', verbose=0, save_best_only=True, save_weights_only=True, mode='auto', period=1)) - checkpoints.append(TensorBoard(log_dir='Data/Checkpoints/./logs', histogram_freq=0, write_graph=True, write_images=False, embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None)) + checkpoints = [ModelCheckpoint( + 'Data/Checkpoints/best_weights.h5', + monitor='val_loss', + verbose=0, + save_best_only=True, + save_weights_only=True, + mode='auto', + period=1, + ), TensorBoard(log_dir='Data/Checkpoints/./logs', histogram_freq=0, write_graph=True, write_images=False, + embeddings_freq=0, embeddings_layer_names=None, embeddings_metadata=None)] - model.fit(X, Y, batch_size=batch_size, epochs=epochs, validation_data=(X_test, Y_test), shuffle=True, callbacks=checkpoints) + model.fit(x, y, batch_size=batch_size, epochs=epochs, validation_data=(x_test, y_test), shuffle=True, + callbacks=checkpoints) return model + def main(): - X, X_test, Y, Y_test, action_total = get_dataset() + """ + The main function. + :return: model + """ + x, x_test, y, y_test, action_total = get_dataset() print(action_total) model = get_model(action_total) model = train_model(model, X, X_test, Y, Y_test) save_model(model) return model + if __name__ == '__main__': main() From bd3e4fe94b537a3d863b1575c45b1f10e4f36ec4 Mon Sep 17 00:00:00 2001 From: benni347 Date: Tue, 1 Feb 2022 15:04:14 +0100 Subject: [PATCH 36/44] Refractoring work on predict and added missing modules to requirements.txt. --- predict.py | 26 +++++++++++++++++++------- requirements.txt | 1 + 2 files changed, 20 insertions(+), 7 deletions(-) diff --git a/predict.py b/predict.py index eb863d2..ffea7bd 100755 --- a/predict.py +++ b/predict.py @@ -1,9 +1,21 @@ -# Arda Mavi -import numpy as np -from scipy.misc import imresize +""" +This file will make a prediction based on the input data. +Author: Arda Mavi +""" +from PIL import Image +from numpy import size + def predict(model, X): - X = imresize(X, (150, 150, 3)).astype('float32') /255. - Y = model.predict(X.reshape(1,150,150,3)) - Y = Y.argmax() - return Y + """ + This function will make a prediction based on the input data. + :param model: Which model to use. + :param X: input data. + :return: y_pred: prediction. + """ + # resize it with PIL, because scipy.misc.imresize is deprecated. + x = X(Image.fromarray(X).resize((size[0] * 4, size[1] * 4), + resample=Image.BICUBIC)) + y = model.predict(x.reshape(1, 150, 150, 3)) + y = y.argmax() + return y diff --git a/requirements.txt b/requirements.txt index 32e104e..c849cbc 100755 --- a/requirements.txt +++ b/requirements.txt @@ -8,3 +8,4 @@ scipy==1.2.3 pynput h5py pyautogui +imageio From 30d5d6ee00f1e78b18bf5b70bd3ef33107d16787 Mon Sep 17 00:00:00 2001 From: benni347 Date: Tue, 1 Feb 2022 15:08:11 +0100 Subject: [PATCH 37/44] Changed the keras... inports to tensorflow.keras.... --- ai.py | 2 +- get_dataset.py | 2 +- get_model.py | 4 ++-- train.py | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/ai.py b/ai.py index 7a6205b..9c3e02e 100755 --- a/ai.py +++ b/ai.py @@ -8,7 +8,7 @@ import time import numpy as np from PIL import Image -from keras.models import model_from_json +from tensorflow.keras.models import model_from_json from mss import mss from game_control import get_key, press, release, click diff --git a/get_dataset.py b/get_dataset.py index 0f93e53..1d9a675 100755 --- a/get_dataset.py +++ b/get_dataset.py @@ -8,7 +8,7 @@ import numpy as np from PIL import Image -from keras.utils import to_categorical +from tensorflow.keras.utils import to_categorical from numpy import size from sklearn.model_selection import train_test_split from imageio import imread, imsave diff --git a/get_model.py b/get_model.py index 249d04a..b84bea3 100755 --- a/get_model.py +++ b/get_model.py @@ -4,8 +4,8 @@ """ import os -from keras.layers import Input, Conv2D, Activation, MaxPooling2D, Flatten, Dense, Dropout -from keras.models import Model +from tensorflow.keras.layers import Input, Conv2D, Activation, MaxPooling2D, Flatten, Dense, Dropout +from tensorflow.keras.models import Model def save_model(model): diff --git a/train.py b/train.py index 35f0745..d1c8941 100755 --- a/train.py +++ b/train.py @@ -4,7 +4,7 @@ """ import os -from keras.callbacks import ModelCheckpoint, TensorBoard +from tensorflow.keras.callbacks import ModelCheckpoint, TensorBoard from get_dataset import get_dataset from get_model import get_model, save_model From 9b077f19df9167c7ad5ea447d24ce83b4fffee9e Mon Sep 17 00:00:00 2001 From: benni347 Date: Thu, 3 Feb 2022 10:56:29 +0100 Subject: [PATCH 38/44] Fixed the create_datset file and added more allowed keys to game_control. --- create_dataset.py | 57 +++++++++++++++++++++++++++++++++++++---------- game_control.py | 13 +++++++---- 2 files changed, 54 insertions(+), 16 deletions(-) mode change 100644 => 100755 create_dataset.py diff --git a/create_dataset.py b/create_dataset.py old mode 100644 new mode 100755 index 2b55676..6e37ca5 --- a/create_dataset.py +++ b/create_dataset.py @@ -4,13 +4,12 @@ Author: Arda Mavi """ import os -from multiprocessing import Process import time +from multiprocessing import Process import numpy as np from PIL import Image from mss import mss -from numpy import size from pynput.keyboard import Listener as key_listener from pynput.mouse import Listener as mouse_listener @@ -30,7 +29,8 @@ def get_screenshot(): img = Image.frombytes('RGB', sct_img.size, sct_img.bgra, 'raw', 'BGRX') img = np.array(img)[:, :, :3] # Get first 3 channel from image as numpy array. # resize it with PIL, because scipy.misc.imresize is deprecated. - img = img(Image.fromarray(img).resize((size[0] * 4, size[1] * 4), resample=Image.BICUBIC)) + img = Image.fromarray(img) + img = img.resize((img.size[0] * 4, img.size[1] * 4), resample=Image.BICUBIC) # img = imresize(img, (150, 150, 3)).astype('float32') / 255. return img @@ -49,14 +49,24 @@ def save_event_keyboard(data_path, event, key): save_img(data_path, screenshot) -def save_event_mouse(data_path, x, y): +def save_event_mouse(data_path, x_coordinate, y_coordinate, button, pressed): """ This function will save the event of the mouse. :param data_path: path to save the event - :param x: x coordinate - :param y: y coordinate + :param x_coordinate: x coordinate + :param y_coordinate: y coordinate """ - data_path = data_path + '/{0},{1},0,0,{2}'.format(x, y, time.time()) + # 539,996,0,0,1643879876.0606766 + # 539 x coordinate + # 996 y coordinate + + # 1643879876.0606766 time since epoch + # button is an enum + + # cut button at dot and keep the last part. + button = button.name.split('.')[-1] + data_path = data_path + '/{0},{1},{2},{3},{4}'.format(x_coordinate, y_coordinate, button, int(pressed), + time.time()) screenshot = get_screenshot() save_img(data_path, screenshot) @@ -70,13 +80,36 @@ def listen_mouse(): if not os.path.exists(data_path): os.makedirs(data_path) - def on_click(x, y, button, pressed): - save_event_mouse(data_path, x, y) - - def on_scroll(x, y, dx, dy): + def on_click(x_coordinate, y_coordinate, button, pressed): + """ + This function will get the x and y coordinate of the mouse, when a click happens. + :param x_coordinate: int + :param y_coordinate: int + :TODO: fix the function. Help from: https://pynput.readthedocs.io/en/latest/mouse.html + """ + print(data_path, x_coordinate, y_coordinate, button, pressed) + save_event_mouse(data_path, x_coordinate, y_coordinate, button, pressed) + + def on_scroll(x_cord, y_cord, dx, dy): + """ + This function will get the new x and y coordinate of the mouse, when a scroll happens. + dx and dy are the amount of scrolling. + :param x_cord: int + :param y_cord: int + :param dx: int + :param dy: int + :return: None + """ pass - def on_move(x, y): + def on_move(x_cord, y_cord): + """ + This function will get the new x and y coordinate of the mouse, when a move happens. + If this callback raises an exception, or returns False, the mouse tracking will be stopped. + :param x_cord: int + :param y_cord: int + :return: None + """ pass with mouse_listener(on_move=on_move, on_click=on_click, on_scroll=on_scroll) as listener: diff --git a/game_control.py b/game_control.py index 0456d36..056826e 100755 --- a/game_control.py +++ b/game_control.py @@ -15,7 +15,13 @@ def get_keys(): Returns a list of all the keys that can be pressed. :return: The list of keys. """ - return ["w", "a", "s", "d", "r", "f", "g", "c", "Key.space", "Key.shift"] + return ["q", "w", "e", "r", "t", "y", "u", "i", "o", "p", + "a", "s", "d", "f", "g", "h", "j", "k", "l", "z", "x", "c", "v", "b", "n", "m", ",", ".", + "Key.space", "Key.shift", "Key.shift_r", "Key.esc", "Key.enter", "Key.backspace", "Key.tab", "Key.ctrl", + "Key.ctrl_r", "Key.caps_lock", "Key.page_up", "Key.page_down", "Key.end", "Key.home", "Key.delete", + "Key.insert", "Key.left", "Key.up", "Key.right", "Key.down", "Key.num_lock", "Key.print_screen", + "Key.f1", "Key.f2", "Key.f3", "Key.f4", "Key.f5", "Key.f6", "Key.f7", "Key.f8", "Key.f9", "Key.f10", + "Key.f11", "Key.f12"] def get_key(key_id): @@ -34,13 +40,13 @@ def get_id(key): :return: The id of the given key. """ try: - print("Key Pressed:", key.char, sep="") + print("Key Pressed:", key.char, sep=" ") return get_keys().index(key.char) except: if (str(key) + "") not in get_keys(): print((str(key) + ""), " is not in list") return 1000 - print("Key Pressed:", (str(key) + ""), sep="") + print("Key Pressed:", (str(key) + ""), sep=" ") return get_keys().index((str(key) + "")) @@ -63,7 +69,6 @@ def scroll(x, y): Scrolls the mouse to the given coordinates. :param x: The horizontal scroll. :param y: The vertical scroll. - :TODO: Change the scroll function to scroll with pyautogui. """ mouse.scroll(x, y) From 67c565b71885fd4d3629dc54596f8a84e9105719 Mon Sep 17 00:00:00 2001 From: benni347 Date: Thu, 3 Feb 2022 11:41:12 +0100 Subject: [PATCH 39/44] Fixed a typo in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 69c1e8b..9d8b590 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Deep Learning is a subfield of machine learning with neural networks inspired by 1. Change the desired kets in the game_control.py file's get keys function, by changing return keys to desired ones 1. Run `python3 create_dataset.py` command in terminal. 2. Play your desired game. -3. Stop `create_dataset` program with `Cntrl-C` in terminal. +3. Stop `create_dataset` program with `ctrl-C` in terminal. ### Model Training: `python3 train.py` From f612ca9d963cc4d0ed5d88142f77f867bfbe6a77 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Mon, 11 Apr 2022 21:59:04 -0400 Subject: [PATCH 40/44] Update create_dataset.py --- create_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_dataset.py b/create_dataset.py index 6e37ca5..5ad1477 100755 --- a/create_dataset.py +++ b/create_dataset.py @@ -30,7 +30,7 @@ def get_screenshot(): img = np.array(img)[:, :, :3] # Get first 3 channel from image as numpy array. # resize it with PIL, because scipy.misc.imresize is deprecated. img = Image.fromarray(img) - img = img.resize((img.size[0] * 4, img.size[1] * 4), resample=Image.BICUBIC) + img = img.resize((img.size[0] / 12, img.size[1] / 12), resample=Image.BICUBIC) # img = imresize(img, (150, 150, 3)).astype('float32') / 255. return img From cfaa532c53f8a468da67d89d429ba061399fc2a3 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Fri, 26 May 2023 21:23:46 -0400 Subject: [PATCH 41/44] Update create_dataset.py --- create_dataset.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/create_dataset.py b/create_dataset.py index 5ad1477..700a804 100755 --- a/create_dataset.py +++ b/create_dataset.py @@ -30,7 +30,7 @@ def get_screenshot(): img = np.array(img)[:, :, :3] # Get first 3 channel from image as numpy array. # resize it with PIL, because scipy.misc.imresize is deprecated. img = Image.fromarray(img) - img = img.resize((img.size[0] / 12, img.size[1] / 12), resample=Image.BICUBIC) + img = img.resize((150, 150), resample=Image.NEAREST) # img = imresize(img, (150, 150, 3)).astype('float32') / 255. return img From 3d775cad5a9e6ad785323458da2347cd27763a95 Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Sat, 27 May 2023 15:14:16 -0400 Subject: [PATCH 42/44] Update get_dataset.py --- get_dataset.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/get_dataset.py b/get_dataset.py index 1d9a675..a9aa50e 100755 --- a/get_dataset.py +++ b/get_dataset.py @@ -11,7 +11,7 @@ from tensorflow.keras.utils import to_categorical from numpy import size from sklearn.model_selection import train_test_split -from imageio import imread, imsave +from imageio import imread, imsave, imresize def get_img(data_path): @@ -21,10 +21,7 @@ def get_img(data_path): :return: the image """ img = imread(data_path) - # resize it with PIL, because scipy.misc.imresize is deprecated. - img = img(Image.fromarray(img).resize((size[0] * 4, size[1] * 4), - resample=Image.BICUBIC)) - # img = imresize(img, (150, 150, 3)) + img = imresize(img, (150, 150)) return img From 53e53f324cde5fe4e8d4b07f16cd166f374bebdf Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Sat, 27 May 2023 15:47:14 -0400 Subject: [PATCH 43/44] Update get_dataset.py --- get_dataset.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/get_dataset.py b/get_dataset.py index a9aa50e..c5360fe 100755 --- a/get_dataset.py +++ b/get_dataset.py @@ -11,7 +11,7 @@ from tensorflow.keras.utils import to_categorical from numpy import size from sklearn.model_selection import train_test_split -from imageio import imread, imsave, imresize +from imageio import imread, imsave def get_img(data_path): @@ -21,7 +21,6 @@ def get_img(data_path): :return: the image """ img = imread(data_path) - img = imresize(img, (150, 150)) return img From 560b2b795ece02f4c8950f1ea04b33c039cd9a8f Mon Sep 17 00:00:00 2001 From: "Elijah V. Schrotenboer" Date: Sat, 27 May 2023 15:53:43 -0400 Subject: [PATCH 44/44] Update train.py --- train.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/train.py b/train.py index d1c8941..5f0e6d3 100755 --- a/train.py +++ b/train.py @@ -51,7 +51,7 @@ def main(): x, x_test, y, y_test, action_total = get_dataset() print(action_total) model = get_model(action_total) - model = train_model(model, X, X_test, Y, Y_test) + model = train_model(model, x, x_test, x, x_test) save_model(model) return model