2
\$\begingroup\$

I don't really know the best way to deal with long lines. The comments may be a little excessive. To me it seems like too much code for how "little" is actually done. I'm not that experienced and want to improve my coding style (english too). Thanks for the input!

#!/usr/bin/env python2
#titel, version etc.
#python_version  :2.6.6

import sys
import threading
import os
import time
import shlex
import datetime
import logging
import socket
import time
import subprocess as sp


# Define a function that starts to loop over every IP and creates a thread for
# every device
def start_threads(dict_data, devices_to_save):
    started_jobs = 0
    # Loop as long as the global counter for jobs done doesn't reach
    # the amount of devices meant to be save
    while (export_counter + failed_counter) < devices_to_save:
        # Loop as long as the counter for started jobs doesn't reach
        # the amount of devices meant to be save AND till the amount of active
        # jobs reaches the set threshold.
        # Sleep to allow threads to be opened in new process (prevent 100% CPU)
        time.sleep(0.001)
        while (started_jobs < devices_to_save) and (
            threading.active_count() <= parallel_jobs):
            # Create arguments for mkdir command via ssh
            mkdir_args = shlex.split("sshpass -p superpw ssh -o \
            StrictHostKeyChecking=no myuser\@192.168.1.1 mkdir -p /data/loc/\
            tec/configs/%s/" % (data_list[started_jobs][1]))
            sp.call(mkdir_args)
            # Start a thread calling the function "trylogin" with entrys from
            # the data_list/dict
            threading.Thread(target=trylogin, args=(str
                (data_list[started_jobs][0]), dict_data)).start()
            started_jobs += 1


# Define a function to open a SSH connection
def trylogin(ipaddress, dict_data):
    # Use the global user and password
    global user, passwd
    # Set up a SSH channel
    ssh = paramiko.SSHClient()
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    # Try to open a connection with the data provided
    try:
        ssh.connect(ipaddress, username=user, password=passwd)
        logging.info("[+%s] Connected %s" % (threading.currentThread(),
            get_name_by_ip(ipaddress, dict_data)))
    # Catch exceptions
    except paramiko.ssh_exception.SSHException as e:
        export_failed(ipaddress, get_name_by_ip(ipaddress, dict_data))
        logging.info("[-%s] Failed %s %s" % (threading.currentThread(), e,
            get_name_by_ip(ipaddress, dict_data)))
        return
    except socket.timeout as e:
        export_failed(ipaddress, get_name_by_ip(ipaddress, dict_data))
        logging.info("[-%s] Failed %s %s" % (threading.currentThread(), e,
            get_name_by_ip(ipaddress, dict_data)))
        return
    # Execute command to save configuration
    stdin, stdout, stderr = ssh.exec_command("save config")
    # Wait for the "save config" to finish
    exit_status = stdout.channel.recv_exit_status()
    # Transfer the config to the backup server
    stdin, stdout, stderr = ssh.exec_command("sshpass -p superpw scp \
    /tmp/upload/config.tar myuser\@192.168.1.1:/data/loc/tec/\
    configs/%s/" % (get_name_by_ip(ipaddress, dict_data)))
    exit_status = stdout.channel.recv_exit_status()
    # Close the SSH connection
    ssh.close()
    export_done()
    logging.info("[+%s] %s %s" % (threading.currentThread(), export_counter,
        get_name_by_ip(ipaddress, dict_data)))


# Define a function to look up the device name
def get_name_by_ip(ipaddress, dict_data):
    name = dict_data[ipaddress]
    return name


# Define a function to count the created config files
def export_done():
    # Make the variable global
    global export_counter
    export_counter += 1


# Define a function to count failed export attempts
def export_failed(ipaddress, device_name):
    # Make the variable global
    global failed_counter
    global failed_data
    failed_data.append((ipaddress, device_name))
    failed_counter += 1


# Define a function that starts to loop over every IP from failed connections
def start_failed_threads(dict_data, devices_to_save):
    started_jobs = 0
    data_list_failed = []
    # Make new data list for failed IPs and device names
    for key, value in dict_data.iteritems():
        data_list_failed.append((key, value))
    # Loop as long as the combined counters don't reacht the amount of devices
    # meant to be saved
    while (export_counter + failed_counter) < devices_to_save:
        # Loop as long as the counter for started jobs doesn't reach the amount
        # of devices meant to be save AND till the amount of active jobs
        # reaches the set threshold.
        # Sleep to allow threads to be opened in new process
        time.sleep(0.001)
        while (started_jobs < devices_to_save) and (
            threading.active_count() <= parallel_jobs):
            # Start a thread calling the function "trylogin" with data from
            # failed connections
            threading.Thread(target=trylogin, args=(str
                (data_list_failed[started_jobs][0]), dict_data)).start()
            # Count up the started jobs
            started_jobs += 1


if __name__ == "__main__":
    socket.setdefaulttimeout(2)
    now = datetime.datetime.now()
    user = "root"
    passwd = "secpw"
    export_counter = 0
    failed_counter = 0
    failed_data = []
    parallel_jobs = 10
    LOG_FILENAME = "/data/Admin/logs/%s.txt" % (now.strftime("%Y-%m-%d"))
    logging.basicConfig(format="%(asctime)s %(message)s",
        filename=LOG_FILENAME, level=logging.INFO)
    with open("/data/Admin/data/IP_Hostname_.txt") as ips_file:
        # Empty list for hosts
        data_list = []
        # Read lines from IP_Hostname_.txt format is IP;Hostname;
        content = ips_file.readlines()
        # Iterate over all lines, except header
        for line in content[1:]:
            # Split line at ";" and get the first entry (device_ip)
            device_ip = line.split(";")[0]
            device_name = (line.split(";")[1]).replace("\n", "")
            # Append a tuple with host IP and name to the data_list
            data_list.append((device_ip, device_name))
    # Set the amount of devices that are meant to be saved
    devices_to_save = len(data_list)
    # Create dictionary
    dict_data = dict(data_list)
    # Start threading
    start_threads(dict_data, devices_to_save)
    helper_counter = export_counter
    if failed_data:
        # Attempt to save all failed devices, do that three times and wait 10s
        # between trys
        for i in range(3):
            time.sleep(10)
            dict_failed_data = dict(failed_data)
            failed_data = []
            export_counter = 0
            failed_counter = 0
            start_failed_threads(dict_failed_data, len(dict_failed_data))
    for element in failed_data:
        logging.warning("###failed connections" + str(element))
    time.sleep(1)
    logging.info("###devices saved " + str(helper_counter))
    logging.info("###time needed " + str(datetime.datetime.now() - now))
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.