I am new to Python.
Currently developing an application, whose purpose is to accept a directory path from user and create a file(sha_generated_file.txt) with SHA256 of each file name inside the passed directory subfolders except those mentioned in ignored list.
Sample entry in sha_generated_file.txt is
SHA256(out_configuration/PRC/afile.bin)= 3c62546a67921acbd4e85c70af794270c095177213781e7d6d1000762d849100
Once the sha_generated_file.txt is generated it will be sent for signing(by invoking another tool, that is not in the scope).
The resultant of signing is a Sample.p7s file, which is parsed using ASN1PARSE and other OpenSSL commands to get some
certificates out of p7s file.
Finally the certificates will be copied to some designated location.
You may not be able to execute the entire program , but individual functions can be executed with minimal modifications ( Sorry I could not possibly come up with version that be run by you, becuase it has dependency of openssl, third party tool etc)
Process goes like this:
The ideal folder structure of of input path is
--configuration
|__PRC
|
|__QRC
|
|__RRC
|
|__XRC
Out of this, i want SHA256 of all the files(only file names, not the contents) from all directories except XRC.
Note: There is a condition for ordering of files in the resultant sha_generated_file.txt
All the files in required directories should be sorted alphabetically (following ascii values)
( This part of code can be found in create_sha_signer_dir,create_sha_file,make_file_list,make_sha_format,fetch_sub_directory_contents etc. )
But before proceeding for SHA calculation, I need to setup some most frequently used directories for temporary processing purpose during the run time of my application.
( This particular code can be found in setup_tool_envdirs)
I have also handled the cases where only parent directory is passed as input or till the required directory is passed (code can be found in check_path_is_valid_config)
There are other set of functions implemented for miscellanios purpose likes extracting bytes from certain offsets and converting different file types(crl to pem etc)
The application has to run in Python2.7 and Python3, It currently works with 2.7 and also mostly it works with Python3 (few errors for byte string conversions etc. )
I am looking for optimizing the code in all possible ways ( optimization/reduce number of lines/correct usage/stanadard/speed/memory/ etc. )
code
import os
import sys
import hashlib
import string
from ctypes import windll
from os.path import dirname, abspath
from shutil import copyfile
import logging
ignored = {"XRC"}
#constants to replaced
OUT_CONFIGURATION = 'out_configuration'
CONTAINS_CONFIG = 'contains_config'
PARENT_HAS_CONFIG = 'parent_has_config'
SHA256_FILE_NAME = 'sha_generated_file.txt'
P7S_FILE_NAME = 'Sample.p7s'
OPENSSL_EXE_PATH = 'C:\\Users\\Documents\\Installs\\openssl.exe'
OFFSETS_LENGTH_FILE = 'crl_offset_n_lengths.txt'
config_dir_path = ''
parent_of_config_dir = ''
signature_file_path = ''
sha_file_path = ''
g_crl_dir_path = ''
# tells whether the input path has(only till parent is passed ) or contains(full path passed)
config_dir_path_flag = ''
def copy_pem_files_to_config_crl():
global g_crl_dir_path
global sha_file_path
crl_dir = sha_file_path
config_crl = g_crl_dir_path
filelist = [ f for f in os.listdir(crl_dir) if f.endswith(".pem") ]
print ( "number of files in crl dir are: {}".format(len(filelist) ))
if len(filelist) > 0:
for f in filelist:
copyfile(os.path.join(crl_dir, f),ref_config_crl+"\\"+f)
return True
else:
return False
def convert_crl_to_pem():
#openssl crl -inform DER -in crl3.crl -outform PEM -out crl3_pem.pem
crl_dir = sha_file_path
filelist = [ f for f in os.listdir(crl_dir) if f.endswith(".crl") ]
logging.info ( "number of files in crl dir are: {}".format(len(filelist)))
if len(filelist) > 0:
for f in filelist:
crl_file = os.path.join(crl_dir, f)
logging.info ( "crl_file = %s", crl_file )
fnamepos = crl_file.find(".crl")
pem_file = crl_file[:fnamepos]
pem_file = pem_file+".pem"
logging.info ( "pem_file = %s", pem_file )
CRL_TO_PEM_CMD = OPENSSL_EXE_PATH + " crl -inform DER -in " + crl_file + " -outform PEM -out " + pem_file
#logging.info ( f"CRL_TO_PEM_CMD ={CRL_TO_PEM_CMD}" )
logging.info ( "CRL_TO_PEM_CMD = %s" + CRL_TO_PEM_CMD )
os.system(CRL_TO_PEM_CMD)
return True
else:
logging.error("There are not CRL files present in %s ",crl_dir)
return False
#sample lines for below 3 functions
#2286:depth=4 hl=4 l=5321 cons: next
def get_length(line):
return int (line.split(':')[1].split(']')[0])
def get_offset(line):
return int (line.split(':')[0].split('[')[1])
def extract_crl(offset,length):
crl_file = '';
#checking this just to avoid creating files with length 4 and below
if length > 4:
crl_file = sha_file_path + "crl_" + str(offset) + ".crl"
logging.info ( "%s created" ,crl_file)
with open(crl_file, 'wb') as fw , open(signature_file_path+P7S_FILE_NAME, 'rb') as fr:
try:
logging.info ( "file open success" )
fr.seek(offset)
crl_content = fr.read(length)
logging.info ( "length of crl_content = %d", len(crl_content))
fw.write(crl_content);
fw.close()
fr.close()
except IOError:
logging.error ( "file open failed %s",crl_file )
return
def create_crls():
olaf = sha_file_path+OFFSETS_LENGTH_FILE
if os.path.exists(olaf):
with open(olaf, 'r') as f:
try:
logging.info ( "file open success" )
line = f.readline()
while len(line.strip()) != 0 :
offset = get_offset(line)
length = get_length(line)
extract_crl(offset,length)
line = f.readline()
f.close();
except IOError:
logging.error ( "file open failed %s",olaf )
else:
logging.error ( "file does not exist %s",olaf )
#get offset and length file
'''
sample contents parse_txt are
2282:depth=3 hl=4 l=12096 cons: start_indi
2286:depth=4 hl=4 l=5321 cons: next
2290:depth=5 hl=4 l=4785 cons: next
extract as long as this condition satifies
next_offset = prev_offset + prev_length
'''
def create_offset_and_length_file(parse_txt):
o_l_f = sha_file_path+OFFSETS_LENGTH_FILE
with open(o_l_f, 'w') as fw:
try:
logging.info ( "file open success" )
except IOError:
logging.error ( "file open failed" )
fo = open(parse_txt);
line = fo.readline()
while len(line.strip()) != 0 :
if "cont [ 1 ]" in line:
line = fo.readline()
offset = line.split(':')[0]
length = line.split('=')[3].split()[0]
length_i = int (length) + 4
offset_i = int (offset)
output = "["+ offset + ":" + str (length_i) + "]"+"\n"
fw.write(output);
line = fo.readline()
while len(line.strip()) != 0 :
if " " + str(offset_i + length_i) + ':' in line:
offset = line.split(':')[0]
length = line.split('=')[3].split()[0]
length_i = int (length) + 4
offset_i = int (offset)
output = "["+ offset + ":" + str (length_i) + "]"+"\n"
fw.write(output);
if str(offset_i + length_i) + ':' in line:
offset = line.split(':')[0]
length = line.split('=')[3].split()[0]
length_i = int (length) + 4
offset_i = int (offset)
output = "["+ offset + ":" + str (length_i) + "]"+"\n"
fw.write(output);
line = fo.readline()
line = fo.readline()
fw.close()
def create_parse_txt(p7sdest_path):
global sha_file_path
cmd = OPENSSL_EXE_PATH
cmd = cmd + " asn1parse -inform DER -in " + p7sdest_path + " >> " + sha_file_path + "parse.txt"
logging.info ( "parse command is %s", cmd )
result = os.system(cmd)
logging.info ( "create_parse_txt result = %d", result )
return result
def copy_signed_p7s():
global sha_file_path
global parent_of_config_dir
global config_dir_path
global signature_file_path
p7s_file_src = sha_file_path + "Sample.p7s"
p7s_file_dest = signature_file_path
logging.info ( "p7s_file_src= %s", p7s_file_src )
logging.info ( "p7s_file_dest=%s", p7s_file_dest )
if os.path.exists(p7s_file_src) == True and os.path.exists(p7s_file_dest) == True:
logging.info ( "both path exists" )
#copyfile should have full path including filename as destination and its blocking so no need to collect return value
if os.path.isfile(p7s_file_src) == True:
copyfile(p7s_file_src, p7s_file_dest+"Sample.p7s")
return True
else:# This should not be reached
logging.error ( "not file" )
else:
logging.error ( " one of the paths does not exist" )
return False
def create_signingP7S_file():
curr_dir = os.getcwd()
logging.info ( "os.getcwd()=%s", curr_dir )
BAT_PATH = "C:\\XXX\\YYY\\Tool\\Test\\"
batdir = os.chdir(BAT_PATH)
logging.info ( "os.getcwd2()= %s", os.getcwd())
TOOL_SCN_PATH = "scenes\\sign_temp.config"
ret = os.system("tool_test.bat"+' -prop_file='+TOOL_SCN_PATH)
if ret == 0:
logging.info ( "Creation of P7S Success" )
if True == copy_signed_p7s():
logging.info ( "p7s copy success" )
return True
else:
logging.error ( "p7s copy failed" )
else:
logging.error ( "Creation of P7S Failed" )
os.chdir(curr_dir)
print ( os.getcwd() )
def write_sha_entry_to_file(sha_entry):
global sha_file_path
sha_file = open(sha_file_path+SHA256_FILE_NAME, "ab")
logging.info (" sha_entry = {}".format(sha_entry))
sha_file.write(str(sha_entry))
sha_file.close()
def calc_hash256(filename):
with open(filename,"rb") as f:
bytes = f.read() # read entire file as bytes
readable_hash = hashlib.sha256(bytes).hexdigest();
return (readable_hash)
def make_sha_format(file):
'''
make the filename format as require by sha file
'''
config_pos = file.find('\\'+OUT_CONFIGURATION)+1
config_path_sha = file[config_pos:]
config_path_sha = config_path_sha.replace("\\","/")
shakey = calc_hash256(file)
shakey_fmt = "SHA256("+config_path_sha+")= "+str(shakey)+'\n'
write_sha_entry_to_file(shakey_fmt)
return shakey_fmt
def fetch_sub_directory_contents(sPath):
for sChild in sorted(os.listdir(sPath)):
if sChild not in ignored:
sChildPath = os.path.join(sPath,sChild)
if os.path.isdir(sChildPath):
fetch_sub_directory_contents(sChildPath)
else:
sChildPath = make_sha_format(sChildPath)
else:
logging.info ( "%s present in ignored list", sChild )
def make_file_list(cwd_rc):
'''
makes the sorted file list from out_configuration directory
'''
logging.info ( cwd_rc )
sorted_list = []
listOfFiles = sorted(os.listdir(cwd_rc))
logging.info ( listOfFiles )
for entry in listOfFiles:
if entry not in ignored:
sorted_list.append(cwd_rc+'\\'+entry)
else:
logging.info ( "%s present in ignored list",entry)
logging.info ( "sorted_list" )
logging.info ( sorted_list )
for entry in sorted_list:
if os.path.isfile(entry):
entry = make_sha_format(entry)
elif os.path.isdir(entry):
fetch_sub_directory_contents(entry)
return True
def create_sha_file(config_dir):
return make_file_list(config_dir)
def get_drives():
drives = []
bitmask = windll.kernel32.GetLogicalDrives()
for letter in string.ascii_uppercase:
if bitmask & 1:
drives.append(letter)
bitmask >>= 1
return drives
def create_sha_signer_dir():
'''
Function to create the directory for SHA256 file of input path
Scans the window system for existing drives and takes the first drive it finds and creates a "temp" folder there
If "temp" is not present it will be created, else checks whether sha256 file is present, if present deletes it and creates new one.
:return the SHA256 path excluding the file name
'''
drives = []
drives = get_drives()
sha_signer_path = drives[0]+':\\temp\\'
if not os.path.exists(sha_signer_path):
print ( " creating ", sha_signer_path )
os.makedirs(sha_signer_path)
else:
print ( sha_signer_path + " already exists, deletes sha file if present" )
isExist = os.path.exists(sha_signer_path+SHA256_FILE_NAME)
if isExist == True:
os.remove(sha_signer_path+SHA256_FILE_NAME)
return sha_signer_path
def startSign():
global config_dir_path
global sha_file_path
global signature_file_path
global parent_of_config_dir
logging.info ("config_dir_path = %s", config_dir_path)
logging.info ("parent_of_config_dir =%s", parent_of_config_dir)
temp_sha_file_path = create_sha_signer_dir()
sha_file_path = temp_sha_file_path
logging.info ( "sha_file_path = %s", sha_file_path )
result = create_sha_file(config_dir_path)
if True == result:
logging.info ( "SHA creation success" )
result = create_signingP7S_file()
if result == True:
logging.info ( "p7s creation success" )
result = create_parse_txt(signature_file_path+P7S_FILE_NAME)
if result == 0:
logging.info ( "parse success" )
create_offset_and_length_file(sha_file_path + "parse.txt")
create_crls()
if convert_crl_to_pem() == True:
copy_pem_files_to_config_crl()
result = True
else:
result = False
else:
logging.error ( "parse failed" )
result = False
else:
logging.error ( "p7s creation failed" )
result = False
else:
logging.error ( "SHA creation failed" )
result = False
return result
def remove_existing_crl_files(crl_dir):
filelist = [ f for f in os.listdir(crl_dir) if f.endswith(".pem") ]
logging.info ( "number of files in crl dir are: {}".format(len(filelist)))
if len(filelist) > 0:
for f in filelist:
os.remove(os.path.join(crl_dir, f))
return True
else:
return False
def create_crl_path(crl_path):
if not os.path.exists(crl_path):
logging.info ( " creating %s", crl_path)
os.makedirs(crl_path)
else:
logging.info ( crl_path + " already exists, deletes crl files if present" )
result = remove_existing_crl_files(crl_path)
if result == True:
logging.info ( "existing crls deleted")
else:
logging.info ( "crl dir is empty")
return crl_path
def create_signature_dir(sign_path):
if not os.path.exists(sign_path):
print ( " creating ", sign_path )
os.makedirs(sign_path)
else:
print ( sign_path + " already exists, deletes p7s file if present" )
isExist = os.path.exists(sign_path+P7S_FILE_NAME)
if isExist == True:
os.remove(sign_path+P7S_FILE_NAME)
return sign_path
def check_path_has_configuration(ipath):
if ipath.endswith("\\"):
config_path = ipath + OUT_CONFIGURATION
else:
config_path = ipath + '\\'+OUT_CONFIGURATION
if os.path.exists(config_path) == True and os.path.isdir(config_path) == True:
return config_path
else:# This should not hit in any case
print (" **parent does not have out_configuration dir** ")
return ''
def get_config_dir(ipath):
global config_dir_path_flag
logging.info ("config_dir_path_flag =%s", config_dir_path_flag)
if config_dir_path_flag == CONTAINS_CONFIG:
return ipath
elif config_dir_path_flag == PARENT_HAS_CONFIG:
config_path = check_path_has_configuration(ipath)
if config_path:
return config_path
else:
return ''
def get_parent_of_config_dir(config_dir):
return dirname(abspath(config_dir))
#input path will be checked for whether it contains out_configuration directory or its parent, in that case returns True
#if none then returns False
def check_path_is_valid_config(ipath):
global config_dir_path_flag
result = ipath.endswith('\\'+OUT_CONFIGURATION) == True or ipath.endswith('\\'+OUT_CONFIGURATION+'\\') == True
if(result == True):
logging.info ("input path contains out_configuration : ")
config_dir_path_flag = CONTAINS_CONFIG
logging.info ("config_dir_path_flag = %s", config_dir_path_flag)
return result
else:
logging.info ("input path looks like parent of out_configuration, checking for out_configuration")
if ipath.endswith("\\"):
config_path = ipath + OUT_CONFIGURATION
else:
config_path = ipath + '\\'+OUT_CONFIGURATION
logging.info (" the created config_path = %s", config_path)
if os.path.exists(config_path) == True and os.path.isdir(config_path) == True:
logging.info ("parent has out_configuration ")
config_dir_path_flag = PARENT_HAS_CONFIG
logging.info ("config_dir_path_flag = %s", config_dir_path_flag)
return True
else:
logging.warning ("out_configuration dir not present for signing")
return False
def setup_tool_envdirs(rc_path):
global config_dir_path
global parent_of_config_dir
global signature_file_path
global g_crl_dir_path
# check for validity of input path
if True == check_path_is_valid_config(rc_path):
config_dir_path = get_config_dir(rc_path)
if config_dir_path:
parent_of_config_dir = get_parent_of_config_dir(config_dir_path)
signature_file_path = config_dir_path + "\\signature\\"
sign_file_path = create_signature_dir(signature_file_path)
crl_files_path = create_crl_path(signature_file_path+"crl");
g_crl_dir_path = crl_files_path
logging.info ( "crl_files_path = %s", crl_files_path )
logging.info ("config_dir_path = %s", config_dir_path)
logging.info ("parent_of_config_dir = %s", parent_of_config_dir)
logging.info ("signature_file_path = %s", signature_file_path)
logging.info ("sign_file_path = %s", sign_file_path)
return True
else:#This part should not be hit
logging.error ("Looks like some corruption in configuration")
return False
else:
logging.error ( "configuration is not present in input path: %s" + ipath )
return False
def parse_cmdline_params():
nargs = len(sys.argv)
logging.info ( "Number of CMD Args:{}".format(nargs) )
if nargs <= 1:
logging.warning ("please pass desired path while running Tool - example: < tool.py path of desired location >")
return False
else:
return True
#tool main entry function
def main_entry():
result = parse_cmdline_params()
if True == result:
input_path = sys.argv[1]
logging.info ("The input path is : %s", input_path)
#First setup the freqently used directory paths
if setup_tool_envdirs(input_path) == True:
logging.info ("setup is successful ")
if startSign() == True:
return True
else:
return False
else:
logging.info ("setup is unsuccessful ")
return False
else:
return
if __name__ == "__main__":
logging.basicConfig(filename='debug_tool.log', encoding='utf-8', level=logging.DEBUG)
if True == main_entry():
print ("**Successful**")
else:
logging.error ("**UnSuccessful**")
