3
\$\begingroup\$

Any and all suggestions are welcome, mainly looking for suggestions on clean up, and making it more readable as I feel that it is inefficient and not very clean.

Some specific things:

  • A better way to have verbose printout? It's one line as is but it seems so ugly.
  • A better and cleaner way to call the exception handling functions?
  • In general, how to make it more efficient.
  • My exception handling seems very verbose and most of the time it seems like I could do better

It's about 550 lines so I'll post it in text here but I'll also provide a link to it on githb.
GitHub Link

# -*- coding: utf-8 -*-
"""
    File Name: __main__.py
    Author: Ari Madian
    Created: May 25, 2017 10:23 AM
    Python Version: 3.6

    __main__.py - Part of Machine-Learning Repo
    Repo: github.com/akmadian/Machine-Learning


    ----------------
    UOD States
    0 - Up
    1 - Down
    2 - Same
"""
from lxml import html
import requests
import csv
import time
import arrow
import sys
import traceback
import smtplib
import configparser
from string import Template
from twilio.rest import Client
import os
import socket

args = sys.argv[1:]

caught_exceptions = 0
exceptions = []

config = configparser.ConfigParser()
config.read_file(open('config.ini'))

csv_headers = ['entryno.', 'value',
               'time', 'DOW', 'timeperiod',
               'OUDstreaktype', 'OUDstreakno.',
               'looptime']

RDF_Name = 'RDF_0006.csv'
RDF_Path = os.path.dirname(os.path.realpath(sys.argv[0])) + \
            '/Data-Files/Raw-Data-Files/' + \
            RDF_Name

values = [0, 0, 0, 0, 0, 0, 0, 0]
values_cache = [0, 0, 0, 0, 0, 0, 0, 0]
entryno_counter = 0
streak = 0
streak_type = 3
uod_state = 3


def add_exception(type_ = None):
    """Log an exception"""
    global caught_exceptions
    caught_exceptions += 1
    exceptions.append(type_)


def email(**kwargs):
    """ Emails an exception report

    Uses information passed in with kwargs to fill out a template and send
    an email containing the information.

    :param kwargs:
    extype      (str): The type of exception or error
    statuscode  (str): The HTML response code
    traceback   (str): The traceback message
    dir         (str): The path to the .py file the exception occurred in
    values      (str): The values and values_cache list, or any other relevant values
    programname (str): The name of the script, can provide additional info
    time        (str): The time at which the exception occurred
    addinfo     (str): Any additional information about the exception/ error
    :return:
    """
    if '-v' in args: print('Email Log Started')

    args_ = ('exctype', 'statuscode', 'traceback',
             'dir', 'values', 'programname',
             'time', 'addinfo', 'exceptionscaught',
             'exceptions')
    kwargs['dir'] = os.path.dirname(os.path.realpath(sys.argv[0]))
    kwargs['programname'] = sys.argv[0]
    kwargs['exceptionscaught'] = caught_exceptions
    kwargs['exceptions'] = exceptions

    for i in args_:
        if i not in kwargs:
            if '-v' in args: print(str(i) + ' - Not in kwargs')
            kwargs[i] = None

    loginemail  = config['BUGREPORTING']['larloginemail']
    loginpassw  = config['BUGREPORTING']['larloginpassword']
    sendtoemail = config['BUGREPORTING']['lartoemail']

    server = smtplib.SMTP('smtp.gmail.com', 587)
    if '-v' in args: print('Server Connection Established')
    server.ehlo()
    server.starttls()
    if '-v' in args: print('Ehlo and Starttls Successful')
    try:
        server.login(loginemail, loginpassw)
    except TypeError as e:
        print(e)
        print('-== TypeError During Email Report Login ==-')
    if '-v' in args: print('Login Successful')

    t = Template("""Program name.....: __main__.py
Time of exception.....: $time
Exception Type.....: $extype
Callback message.....:
$traceback




Status code.....: $statuscode
Directory.....: $dir
Program name.....: __main__,.py
Values.....:
$values



Exceptions Caught.....: $exceptionscaught
Exceptions.....: $exceptions





Additional Information.....: $addinfo




""")

    mes = 'Subject: Exception occurred and was caught\n\n' + t.substitute(kwargs) + \
          'Local log status.....:' + '\n\n End exception log'
    if '-v' in args: print('Message Defined')
    try:
        server.sendmail(loginemail, sendtoemail, mes)
    except TypeError as e:
        print(e)
        print('-== TypeError During Email Report Send ==-')
    if '-v' in args: print('Message Sent')
    server.quit()
    if '-v' in args: print('Server Connection Broken')
    print('Email Log Successful')


def text_log(**kwargs):
    """Uses SMS to text an error report

    Uses information passed in via kwargs to generate an excpetion/ error report

    :param kwargs:
    See kwargs: in email() for kwargs docs
    :return:
    """
    if '-v' in args: print('Text Log Started')
    account_sid = config['BUGREPORTING']['twilioaccssid']
    auth_token = config['BUGREPORTING']['twilioauthtoken']
    client = Client(account_sid, auth_token)
    if '-v' in args: print('Client Defined')

    t = Template('\n' + """

An exception occurred and was caught

Program name..: __main__.py

Time..: $time

Exception Type..: $extype
""")
    if '-v' in args: print('Message Defined')
    mess = t.substitute(kwargs)
    client.messages.create(
        to=config['BUGREPORTING']['twiliorecievenum'],
        from_=config['BUGREPORTING']['twiliosendnum'],
        body=mess)
    print('Text Log Successful')


def local_log(**kwargs):
    """ Logs an exception/ error report locally

    Longer description is essentially the same as email() and text_log()
    but it's a local text file.

    :param:
    fname       (str): What the log file should be named, do not include file extension
    See kwargs: in email() for the rest of them
    :return:
    """
    Local_Log_Path = os.path.dirname(os.path.realpath(sys.argv[0])) + \
                     '/Local-Err-Logs'

    args_ = ('extype', 'statuscode', 'traceback',
            'dir', 'values', 'programname',
            'time', 'addinfo', 'exceptionscaught',
             'exceptions')
    kwargs['dir'] = os.path.dirname(os.path.realpath(sys.argv[0]))
    kwargs['programname'] = sys.argv[0]
    kwargs['exceptionscaught'] = caught_exceptions
    kwargs['exceptions'] = exceptions
    kwargs['values'] = (values, values_cache)
    kwargs['time'] = arrow_timestamp()

    for i in args_:
        if i not in kwargs:
            kwargs[i] = None
    try:
        filename = Local_Log_Path + kwargs['fname'] + '.txt'
        with open(filename, 'w') as f:
            for arg in args_:
                f.write(str(arg) + ' - ' + str(kwargs[arg]) + '\n')
    except Exception as e:
        print(e)
        print('-== Local Log Unsuccessful ==-')

    print('Local Log Successful')


def scrape():
    """ Scrapes a stock value

    Uses requests and lxml to scrape a stock value
    :return:
    """
    stock_xpath = '//*[@id="cross_rate_1"]/tbody/tr[1]/td[4]'
    stock_site = 'https://www.investing.com/commodities/real-time-futures'

    del values_cache[:]
    if '-v' in args: print('Values Cache Cleared')
    for index in values:
        values_cache.append(index)
    if '-v' in args: print('Values Cache Updated')
    del values[:]
    if '-v' in args: print('Values Cleared')
    values.append(entryno_counter)
    value_ = []

    try:
        socket.create_connection((socket.gethostbyname('www.google.com'), 80), 2)
    except socket.timeout as e:
        print(e)
        add_exception('socket.timeout')
        print('-== Timeout Caught ==-')
        time.sleep(900)
        scrape()

    except Exception as e:
        print(e)
        add_exception('Exception')
        print('-== Generic Exception Caught ==-')
        time.sleep(900)
        scrape()
    else:
        if '-v' in args: print('Good Connection')

    try:
        page = session.get(stock_site, headers={'User-Agent': 'Mozilla/5.0'})
        if '-v' in args: print('Page Recieved')
        code = str(page.status_code)
        if '-v' in args: print('Response Code - ' + code)

        try:
            assert 200 <= int(code) < 300
        except AssertionError as e:
            print(e)
            add_exception('Bad Response Code - ' + code)
            print('-== Bad Response Code Returned ==-')
            print('Status Code - ' + code)
            exc_type, exc_value, exc_traceback = sys.exc_info()
            exc_mes = repr(traceback.format_exception(exc_type, exc_value, exc_traceback))
            email(extype='AssertionError', statuscode=code, traceback=exc_mes,
                  values=[values, values_cache], programname='Machine-Learning/__main__.py',
                  addinfo='Bad status code.', time=arrow_timestamp())
            local_log(fname='__main__errlog', extype='AssertionError',
                      statuscode=code, traceback=exc_mes, programname='Machine-Learning/__main__.py',
                      addinfo='Bad status code')
            text_log(time=arrow_timestamp(), programname='Machine-Learning/__main__.py',
                     extype='AssertionError')

        else:
            if '-v' in args: print('Good Response Code Returned - ' + code)
            tree = html.fromstring(page.content)
            value = tree.xpath(stock_xpath)[0].text
            for letter in value:
                value_.append(letter) if letter.isdigit() else None
            values.append(int(''.join(value_)))
            if '-v' in args: print('Value Appended')
            csv_write()

    except requests.exceptions.Timeout as e:
        print(e)
        add_exception('requests.exceptions.Timeout')
        print('-== Timeout Exception Raised ==-')
        exc_type, exc_value, exc_traceback = sys.exc_info()
        exc_mes = repr(traceback.format_exception(exc_type, exc_value, exc_traceback))
        try:
            # noinspection PyUnboundLocalVariable
            email(extype='requests.exceptions.ConnectionError', statuscode=code,
                  traceback=exc_mes, values=[values, values_cache],
                  programname='Machine-Learning/__main__.py')
        except NameError as e:
            print(e)
            add_exception('NameError')
            print('-== NameError raised during Timeout exception catch ==-')
            email(extype='requests.exceptions.ConnectionError', traceback=exc_mes,
                  values=[values, values_cache], programname='Machine-Learning/__main__.py')
        local_log(fname='__main__errlog', extype='requests.exceptions.ConnectionError',
                  statuscode=code, traceback=exc_mes, programname='Machine-Learning/__main__.py',
                  addinfo='Connection error, see traceback')
        text_log(time=arrow_timestamp(), programname='Machine-Learning/__main__.py',
                 extype='requests.exceptions.ConnectionError')
        time.sleep(900)
        scrape()
        print(exc_mes)

    except requests.exceptions.ConnectionError as e:
        print(e)
        add_exception('requests.exceptions.ConnectionError')
        print('-== ConnectionError Exception Raised ==-')
        exc_type, exc_value, exc_traceback = sys.exc_info()
        exc_mes = repr(traceback.format_exception(exc_type, exc_value, exc_traceback))
        try:
            # noinspection PyUnboundLocalVariable
            email(extype='requests.exceptions.ConnectionError', statuscode=code,
                  traceback=exc_mes, values=[values, values_cache],
                  programname='__main__.py')
        except NameError as e:
            print(e)
            add_exception('NameError')
            print('-== NameError Raised during ConnectionError Handling ==-')
            email(extype='requests.exceptions.ConnectionError',
                  traceback=exc_mes, values=[values, values_cache],
                  programname='__main__.py')

        local_log(fname='__main__errlog', extype='requests.exceptions.ConnectionError',
                  statuscode=code, traceback=exc_mes, programname='__main__.py',
                  addinfo='Connection error, see traceback')
        text_log(time=arrow_timestamp(), programname='__main__.py',
                 extype='requests.exceptions.ConnectionError')
        time.sleep(900)
        scrape()

    except requests.exceptions.RequestException as e:
        print(e)
        add_exception('requests.exceptions.RequestException')
        print('-== Generic RequestException Raised ==-')
        exc_type, exc_value, exc_traceback = sys.exc_info()
        exc_mes = repr(traceback.format_exception(exc_type, exc_value, exc_traceback))
        try:
            # noinspection PyUnboundLocalVariable
            email(extype='requests.exceptions.RequestException', statuscode=code,
                  traceback=exc_mes, values=[values, values_cache], programname='__main__.py')
        except NameError as e:
            print(e)
            add_exception('NameError')
            print('NameError raised during RequestException Catch')
            email(extype='requests.exceptions.RequestException', traceback=exc_mes,
                  values=[values, values_cache], programname='__main__.py')
        local_log(fname='__main__errlog', extype='requests.exceptions.RequestException',
                  statuscode=code, traceback=exc_mes, programname='__main__.py',
                  addinfo='Catchall requests error, see traceback')
        text_log(time=arrow_timestamp(), programname='__main__.py/ COMMODITIES_GOLD.py',
                 extype='requests.exceptions.RequestException')


def uod():
    """ Determines whether the last value was higher, lower, or the same
        as the one before it.

    :return int:    See main file docstring for int key
    """
    global values_cache
    global streak_type
    global uod_state
    global streak
    global values
    try:
        if '-v' in args: print('uod() Now Running')
        if int(values[1]) > int(values_cache[1]):  # If value went up
            if streak_type != 0:  # If it was previously down or same
                uod_state = 0  # Set state to up
                streak = 0  # Reset streak
                streak_type = 0  # Set streak type to up
            if int(values_cache[4]) == 0:  # If last state was up
                streak += 1  # Add one to streak counter
            return 0
        elif int(values[1]) < int(values_cache[1]):  # If value went down
            if streak_type != 1:
                uod_state = 1
                streak = 0
                streak_type = 1
            if int(values_cache[4]) == 1:
                streak += 1
            return 1
        elif int(values[1]) == int(values_cache[1]):  # If value is the same
            if streak_type != 2:
                uod_state = 2
                streak = 0
                streak_type = 2
            if int(values_cache[4]) == 2:
                streak += 1
            return 2
    except TypeError as e:
        print(e)
        add_exception('TypeError - uod')
        print('-== TypeError Caught During uod() Run ==-')
        # Full Reset Of Values
        values_cache = [0, 0, 0, 0, 0, 0, 0]
        values = [0, 0, 0, 0, 0, 0, 0]

    except IndexError as e:
        print(e)
        add_exception('IndexError - uod')
        print('-== IndexError Caught During uod() Run ==-')
        try:
            values_cache = [0, 0, 0, 0, 0, 0, 0]
            if '-v' in args: print('Retrying uod()')
            uod()
        except RecursionError as e:
            print(e)
            add_exception('RecursionError')
            print('-== RecursionError Caught During uod() Retry ==-')
            pass


def istrading():
    """ Determines if the script is running within the stock's trading hours"""
    dow = arrow.utcnow().to('US/Pacific').format('dddd')
    if '-v' in args: print('DOW Defined')
    time_ = arrow.utcnow().to('US/Pacific').format('HHmm')
    if '-v' in args: print('Time Defined')
    if dow != 'Saturday' and 1500 < int(time_) > 1415:
        if '-v' in args: print('Istrading - True')
        return True
    else:
        if '-v' in args: print('Istrading - False')
        return False


def time_period():
    """Gets a custom timestamp"""
    stamp = arrow.utcnow().to('US/Pacific').format('YYYYMMDDdHHmmss')
    if '-v' in args: print('Custom Stamp Defined')
    return stamp


def arrow_timestamp():
    """Gets a generic timestamp"""
    stamp = arrow.utcnow().to('US/Pacific').format('YYYY-MM-DD HH:mm:ss')
    if '-v' in args: print('Generic Stamp Defined')
    return stamp


def csv_write():
    """ Writes to a csv file

    Gets additional information and writes it to a CSV file
    """
    values.append(arrow_timestamp())
    if '-v' in args: print('Timestamp Appended')
    values.append(int(time_period()))
    if '-v' in args: print('Custom Timestamp Appended')
    values.append(uod())
    if '-v' in args: print('uod Appended')
    values.append(streak)
    if '-v' in args: print('Streak Appended')

    with open(RDF_Path, 'a', newline='') as file:
        looptime = time.time() - starttime[:7]
        if '-v' in args: print('Looptime - ' + str(looptime))
        if '-v' in args: print('Lines per second - ' + str(1 / looptime))
        if '-v' in args: print('Lines per minute - ' + str((1 / looptime) * 60))
        if '-v' in args: print('lines per hour - ' + str(((1 / looptime) * 60) * 60))
        values.append(looptime)
        csv.writer(file).writerow(values)
        print(values_cache)
        print(values)
    if '-v' in args: print('Values Written')


def csv_init():
    """ Initializes a csv file with headers and necessary data for PRDF"""
    cases = [[None, None, None, None, 0, 2],
             [None, None, None, None, 1, 2],
             [None, None, None, None, 2, 2]]
    with open(RDF_Path, 'w') as file:
        csv.writer(file).writerow(csv_headers)
        for list_ in cases:
            csv.writer(file).writerow(list_)
        if '-v' in args: print('CSV Initialized')


def argsinterpret(listchoice):
    """Interprets sys.argv data

    :param:
    listchoice  (str): The users choice of which arg list to return"""
    rawargs = sys.argv
    addargs = sys.argv[1:]
    groupedargs = [''.join([thing, next(iter(rawargs))])
    if 'test' in thing else None for thing in iter(rawargs)]
    return rawargs if listchoice == 'rawargs' \
        else addargs if listchoice == 'addargs' \
        else groupedargs if listchoice == 'groupedargs' \
        else 'Not a valid option'


session = requests.Session()
adapter = requests.adapters.HTTPAdapter()
session.mount('https://', adapter)
session.mount('http://', adapter)

while True:
    if True:
        if '-v' in args: print('Is Trading')
        starttime = time.time()
        if '-v' in args: print('~^' * int((len(str(values_cache)) / 2)))
        if '-v' in args and caught_exceptions != 0:
            print('Caught Exceptions - ' + str(caught_exceptions))
            print(exceptions)
        scrape()
        if '-v' in args: print('Loop Successful')
        entryno_counter += 1
    else:
        if '-v' in args: print('Not Trading')
        time.sleep(300)
        continue
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.