I have a program that includes sending data to our backend servers upon completion.
Due to the occasional connection issue at either side I have coded into the main script a JSON dump as a backup in case we are unable to connect to the BE server.
The JSON file is simply a Key/Value mirror of the SQL database fields in the table in question.
I will run this program in the background which will periodically check the folder. It will open the JSON and insert / update any files that are in there and remove them if completed.
from json import load from time import sleep from os.path import join from os import remove from glob import glob from sql_tunnel_control import ( SqlTunnelConnect, SSH_KEY, sql_ip, sql_hostname, sql_username, sql_password, sql_main_database, sql_port, ssh_host, ssh_port, ssh_user, ssh_password, table ) production = SqlTunnelConnect( SSH_KEY, None, sql_hostname, sql_username, sql_password, table, sql_port, ssh_host, ssh_port, ssh_user, None, ) def load_be_data(f): """ Save data to json file and allow another program to iterate over that folder until all data is safely in the backend server""" try: with open(join(f), "r") as j: return load(j) except Exception as e: print(e) # pass def get_files(): f = glob("backend_data/*.json") return f def send_to_be(f): """ takes in a file and attempts to send to the server.. Returns False if it fails.. Deletes the file if it succeeds """ print(f) if f["query_type"] == "insert": create_new_component = """INSERT INTO components .... """ try: production.run_query(create_new_component) return True except: return False elif f["query_type"] == "update": update_component = """UPDATE components SET ...""" try: production.run_query(update_component) return True except: return False else: return False # problem with query type def persist_dump(): to_send = get_files() while len(to_send) > 0: for f in to_send: r = send_to_be(load_be_data(f)) if r: remove(f) sleep(1) else: sleep(300) # 5 minutes rest. # all files completed.. sleep for a bit.. then see whats what// sleep(1800) persist_dump() persist_dump()