Find all strings matching a giving regex pattern in files in a directory (including all subdirectories)

Question

#! python3
#   `regexSearch`: Finds all lines matching a given regex in each file in a given folder.
#   Usage:
#       The directory to search and regex to be searched for are provided as a command line arguments.
#       The 1st and 2nd command line arguments are the directory and regex pattern respectively.
#       Script prompts the user to enter the regex.
#       After completion, the user is prompted to continue

import re, sys
from os import path, listdir

def regex_search(regex, directory):
    res, lst = {}, listdir(directory)
    for itm in lst:
        pth = path.join(path.abspath(directory), itm)
        if path.isdir(pth):   res.update(regex_search(regex, pth))    #Recursively traverse all sub directories.
        else:
            print(pth)
            with open(pth) as file:  
                tmp = []
                for idx, line in enumerate(file.readlines()):
                    results = regex.findall(line)
                    if results:  tmp.extend([f"Line {idx+1}: {results}"])
                res[pth] = tmp
    return res

if __name__ == "__main__":
    directory, pattern = sys.argv[1:3]
    while not path.isdir(directory):
        print("Error: Please input a valid path for an existing directory:", end = "\t")
        directory = input()
    while True:
        try:
            regex = re.compile(pattern)
            break
        except TypeError:
            print("Error: Please input a valid regex:", end = "\t")
            pattern = input()
        except re.error:
            print("Error: Please input a valid regex:", end = "\t")
            pattern = input()
    matches = regex_search(regex, directory)
    for key in matches:  print(key, "\n".join(matches[key]), sep="\n", end="\n\n")

Answer 1

Some improvements

• Style

  Please indent your file properly, since indentation is important in Python, those lines like

  if path.isdir(pth):   res.update(regex_search(regex, pth))
  

  Are frowned upon, instead do

  if path.isdir(pth):
      res.update(regex_search(regex, pth))
  

• Use glob for listing files in a directory

  With Python3.5+ glob is the easiest way to list all files in a directory and subdirectory, before you should use os.walk()

• Use generators when appropriate

  This will save some memory space, as it doesn't have to append to the temporary list all the time

• Use argparse over sys.argv[]

  Argparse is the module for CLI input, easy to use and has a ton of features I definitely recommend it!

Code

import argparse
import glob
import re
import os
import pathlib

def regex_search(regex, directory):
    for f in glob.glob(f"{directory}**/*.*", recursive=True):
        with open(f) as _file:
            for i, line in enumerate(_file.readlines()):
                if regex.search(line):
                    yield f"In file {f} matched: {line.rstrip()} at position: {i}"

def parse_args():
    parser = argparse.ArgumentParser(
        usage='%(prog)s [options] <regex> <directory>',
        formatter_class=argparse.RawDescriptionHelpFormatter
    )
    parser.add_argument('regex', type=str)
    parser.add_argument('directory', type=str)
    args = parser.parse_args()

    try:
        rgx = re.compile(args.regex)
    except Exception as e:
        parser.error('Regex does not compile')
    directory = pathlib.Path(args.directory)
    if not os.path.isdir(directory):
        parser.error('Directory is not valid')
    return rgx, directory

if __name__ == '__main__':
    regex, directory = parse_args()
    for match in regex_search(regex, directory):
        print(match)

Bonus Round!

grep is a Unix tool that can basically do this by default

grep -Hrn 'search term' path/to/dir

Where:

• -H prints the matching line
• -r Does a recursive search
• -n prints the line number