I don't really know the best way to deal with long lines. The comments may be a little excessive. To me it seems like too much code for how "little" is actually done. I'm not that experienced and want to improve my coding style (english too). Thanks for the input!
#!/usr/bin/env python2
#titel, version etc.
#python_version :2.6.6
import sys
import threading
import os
import time
import shlex
import datetime
import logging
import socket
import time
import subprocess as sp
# Define a function that starts to loop over every IP and creates a thread for
# every device
def start_threads(dict_data, devices_to_save):
started_jobs = 0
# Loop as long as the global counter for jobs done doesn't reach
# the amount of devices meant to be save
while (export_counter + failed_counter) < devices_to_save:
# Loop as long as the counter for started jobs doesn't reach
# the amount of devices meant to be save AND till the amount of active
# jobs reaches the set threshold.
# Sleep to allow threads to be opened in new process (prevent 100% CPU)
time.sleep(0.001)
while (started_jobs < devices_to_save) and (
threading.active_count() <= parallel_jobs):
# Create arguments for mkdir command via ssh
mkdir_args = shlex.split("sshpass -p superpw ssh -o \
StrictHostKeyChecking=no myuser\@192.168.1.1 mkdir -p /data/loc/\
tec/configs/%s/" % (data_list[started_jobs][1]))
sp.call(mkdir_args)
# Start a thread calling the function "trylogin" with entrys from
# the data_list/dict
threading.Thread(target=trylogin, args=(str
(data_list[started_jobs][0]), dict_data)).start()
started_jobs += 1
# Define a function to open a SSH connection
def trylogin(ipaddress, dict_data):
# Use the global user and password
global user, passwd
# Set up a SSH channel
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
# Try to open a connection with the data provided
try:
ssh.connect(ipaddress, username=user, password=passwd)
logging.info("[+%s] Connected %s" % (threading.currentThread(),
get_name_by_ip(ipaddress, dict_data)))
# Catch exceptions
except paramiko.ssh_exception.SSHException as e:
export_failed(ipaddress, get_name_by_ip(ipaddress, dict_data))
logging.info("[-%s] Failed %s %s" % (threading.currentThread(), e,
get_name_by_ip(ipaddress, dict_data)))
return
except socket.timeout as e:
export_failed(ipaddress, get_name_by_ip(ipaddress, dict_data))
logging.info("[-%s] Failed %s %s" % (threading.currentThread(), e,
get_name_by_ip(ipaddress, dict_data)))
return
# Execute command to save configuration
stdin, stdout, stderr = ssh.exec_command("save config")
# Wait for the "save config" to finish
exit_status = stdout.channel.recv_exit_status()
# Transfer the config to the backup server
stdin, stdout, stderr = ssh.exec_command("sshpass -p superpw scp \
/tmp/upload/config.tar myuser\@192.168.1.1:/data/loc/tec/\
configs/%s/" % (get_name_by_ip(ipaddress, dict_data)))
exit_status = stdout.channel.recv_exit_status()
# Close the SSH connection
ssh.close()
export_done()
logging.info("[+%s] %s %s" % (threading.currentThread(), export_counter,
get_name_by_ip(ipaddress, dict_data)))
# Define a function to look up the device name
def get_name_by_ip(ipaddress, dict_data):
name = dict_data[ipaddress]
return name
# Define a function to count the created config files
def export_done():
# Make the variable global
global export_counter
export_counter += 1
# Define a function to count failed export attempts
def export_failed(ipaddress, device_name):
# Make the variable global
global failed_counter
global failed_data
failed_data.append((ipaddress, device_name))
failed_counter += 1
# Define a function that starts to loop over every IP from failed connections
def start_failed_threads(dict_data, devices_to_save):
started_jobs = 0
data_list_failed = []
# Make new data list for failed IPs and device names
for key, value in dict_data.iteritems():
data_list_failed.append((key, value))
# Loop as long as the combined counters don't reacht the amount of devices
# meant to be saved
while (export_counter + failed_counter) < devices_to_save:
# Loop as long as the counter for started jobs doesn't reach the amount
# of devices meant to be save AND till the amount of active jobs
# reaches the set threshold.
# Sleep to allow threads to be opened in new process
time.sleep(0.001)
while (started_jobs < devices_to_save) and (
threading.active_count() <= parallel_jobs):
# Start a thread calling the function "trylogin" with data from
# failed connections
threading.Thread(target=trylogin, args=(str
(data_list_failed[started_jobs][0]), dict_data)).start()
# Count up the started jobs
started_jobs += 1
if __name__ == "__main__":
socket.setdefaulttimeout(2)
now = datetime.datetime.now()
user = "root"
passwd = "secpw"
export_counter = 0
failed_counter = 0
failed_data = []
parallel_jobs = 10
LOG_FILENAME = "/data/Admin/logs/%s.txt" % (now.strftime("%Y-%m-%d"))
logging.basicConfig(format="%(asctime)s %(message)s",
filename=LOG_FILENAME, level=logging.INFO)
with open("/data/Admin/data/IP_Hostname_.txt") as ips_file:
# Empty list for hosts
data_list = []
# Read lines from IP_Hostname_.txt format is IP;Hostname;
content = ips_file.readlines()
# Iterate over all lines, except header
for line in content[1:]:
# Split line at ";" and get the first entry (device_ip)
device_ip = line.split(";")[0]
device_name = (line.split(";")[1]).replace("\n", "")
# Append a tuple with host IP and name to the data_list
data_list.append((device_ip, device_name))
# Set the amount of devices that are meant to be saved
devices_to_save = len(data_list)
# Create dictionary
dict_data = dict(data_list)
# Start threading
start_threads(dict_data, devices_to_save)
helper_counter = export_counter
if failed_data:
# Attempt to save all failed devices, do that three times and wait 10s
# between trys
for i in range(3):
time.sleep(10)
dict_failed_data = dict(failed_data)
failed_data = []
export_counter = 0
failed_counter = 0
start_failed_threads(dict_failed_data, len(dict_failed_data))
for element in failed_data:
logging.warning("###failed connections" + str(element))
time.sleep(1)
logging.info("###devices saved " + str(helper_counter))
logging.info("###time needed " + str(datetime.datetime.now() - now))
