Walk a folder of files and extract creation date, countries mentioned and a summary based on their text for each

Question

I made a script that scans a folder of 723 documents (pdf, pptx, docx) to extract the 'Name', 'Type', 'NbPages', 'Creation Date', 'Period Date', 'Countries', 'Summary', 'Indicateurs', 'Path'. I think I am not doing it right as I do several loops and each run through the text to extract their respective information. As a result it takes 19 hours. Probably it can be optimized.

Here is my code:

import os
import csv
import PyPDF2
import logging

import pandas as pd
import numpy as np
import missingno as msno

import fitz
from sumy.parsers.plaintext import PlaintextParser
from sumy.nlp.tokenizers import Tokenizer
from sumy.summarizers.lex_rank import LexRankSummarizer

import re
from collections import Counter
from tqdm import tqdm

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
from sklearn.metrics import silhouette_score

import nltk
from nltk.corpus import stopwords
nltk.download('punkt')
nltk.download('stopwords')
import string

from datetime import datetime
import dateparser
import platform
import time
import datefinder

import pycountry

import spacy
import fr_core_news_md

# import matplotlib.pyplot as plt

import docx
import pptx

import zipfile


nlp = fr_core_news_md.load()

# Fonction pour faire un résumé du texte
def summarize_text(text):
    parser = PlaintextParser.from_string(text, Tokenizer("french"))
    summarizer = LexRankSummarizer()
    summary = summarizer(parser.document, 2)  # résumé avec 1 phrases
    return " ".join([str(sentence) for sentence in summary])

def extract_pdf_text(file):
    doc = fitz.open(file)
    text = ""
    for page in doc:
        text += page.get_text("text")
    return text

def compute_percentage_numbers(pdf_reader):
    # Initialiser les compteurs
    total_numbers = 0
    total_text = ''

    # Parcourir chaque page du PDF
    for page_number in range(len(pdf_reader.pages)):
        page_text = pdf_reader.pages[page_number].extract_text()
        total_text += page_text
        numbers = re.findall(r'\b\d+\b', page_text)
        total_numbers += len(numbers)

    # Calculer le pourcentage de nombres
    if len(re.findall(r'\b\w+\b', total_text)) == 0:
        percentage =  0
    else:
        percentage = (total_numbers / len(re.findall(r'\b\w+\b', total_text))) * 100
    return percentage

def extract_text_from_file(file_path):
    if file_path.endswith(".pdf"):
        with open(file_path, "rb") as file:
            pdf_reader = PyPDF2.PdfFileReader(file)
            text = " ".join([pdf_reader.getPage(page_num).extractText()
                          for page_num in range(pdf_reader.numPages)])
            return text
    elif file_path.endswith(".docx"):
        doc = docx.Document(file_path)
        text = " ".join([para.text for para in doc.paragraphs])
        return text
    elif file_path.endswith(".pptx"):
        prs = pptx.Presentation(file_path)
        text = " ".join([slide.shapes[0].text for slide in prs.slides])
        return text

# Fonction pour faire un résumé du texte
def summarize_text(text):
    parser = PlaintextParser.from_string(text, Tokenizer("french"))
    summarizer = LexRankSummarizer()
    summary = summarizer(parser.document, 2)  # résumé avec 1 phrases
    return " ".join([str(sentence) for sentence in summary])

def extract_pdf_text(file):
    doc = fitz.open(file)
    text = ""
    for page in doc:
        text += page.get_text("text")
    return text

countries_possible = ['France','Allemagne','Italie','Espagne','Belgique','Portugal','Suisse',
                      'Norvège','Danemark','Suède','Pays Bas','Finlande','Pologne', 
                      'UK','Irlande','USA','Japon','Australie','Chine','Hong Kong',
                      'Corée du Sud','Canada','Singapour','Brésil','Chili','Mexique','Argentine','Russie']
# countries_possible = list(map(lambda x: x.upper(), countries_possible))                     

def extract_countries(text):
    countries = []
    for country in pycountry.countries:
        if country.name in text:
            countries.append(country.name)
    # countries = [c for c in countries_possible if c.lower() in text]

    if countries:
        return countries
    else:
        return np.nan

list_translate_month = [
    ('janvier', 'january'),
    ('février', 'february'),
    ('mars','march'),
    ('avril', 'april'),
    ('mai', 'may'),
    ('juin', 'june'),
    ('juillet', 'july'),
    ('août', 'august'),
    ('septembre', 'september'),
    ('octobre', 'october'),
    ('novembre', 'november'),
    ('decembre', 'december')
]

list_translate_day = [
    ('lundi', 'monday'),
    ('mardi', 'tuesday'),
    ('mercredi', 'wednesday'),
    ('jeudi','thursday'),
    ('vendredi', 'friday'),
    ('samedi', 'saturday'),
    ('dimanche', 'sunday')
]

def extract_dates(text):
    # https://stackoverflow.com/questions/3276180/extracting-date-from-a-string-in-python

    for month_fr, month_en in list_translate_month:
        text = text.replace(month_fr, month_en)

    for day_fr, day_en in list_translate_day:
        text = text.replace(day_fr, day_en)
    
    matches = list(datefinder.find_dates(text))
    valid_matches = [m for m in matches if m >= datetime(1980, 1, 1) and m <= datetime.today()]
    return valid_matches

def get_creation_date(file_path):
    # https://stackoverflow.com/questions/237079/how-do-i-get-file-creation-and-modification-date-times

    return datetime.fromtimestamp(os.path.getctime(file_path)).strftime('%Y-%m-%d')

def infer_creation_date(dates):
    return max(dates)

def find_period(dates):
    # remove creation_date
    dates.remove(max(dates))
    if dates:
        return min(dates).strftime('%Y-%m-%d'), max(dates).strftime('%Y-%m-%d')
    else:
        return np.nan

    mapping_path = "some_file.xlsx"
    df_indicateurs = pd.read_excel(mapping_path, sheet_name="Liste indicateurs")
    indicateurs = df_indicateurs['Data (issues de un ou plusieurs documents)'].values
    
    # Suppression des doublons dans la colonne "Data"
    df = df_indicateurs.drop_duplicates(subset=['Data (issues de un ou plusieurs documents)'])
    
    # Création d'une liste vide pour stocker les indicateurs
    indicateurs = []
    
    # Boucle pour extraire les indicateurs de chaque ligne
    for line in df_indicateurs['Data (issues de un ou plusieurs documents)']:
        # Divise la ligne en une liste de sous-chaînes séparées par des virgules
        try:
            sous_chaines = line.split(',')
        except:
            print(line)
        
        # Boucle pour parcourir chaque sous-chaîne
        for sous_chaine in sous_chaines:
            # Supprime les espaces en début et fin de sous-chaîne
            indicateur = sous_chaine.strip()
            
            # Vérifie si l'indicateur n'a pas déjà été ajouté à la liste
            if indicateur not in indicateurs:
                indicateurs.append(indicateur)

folder_path = root_directory
data = []
files = []
for root, dirs, files_ in os.walk(folder_path):
    files.extend(os.path.join(root, f) for f in files_)

## for test
# files = files[:10]
for file_path in tqdm(files, desc="Processing files in '{}'".format(folder_path)):
    num_pages, creation_date, period_date, countries, summary = np.nan, np.nan, np.nan, np.nan, np.nan

    name, format = os.path.splitext(file_path)
    name = name.split('/')[-1]
    # test pour ne prendre que les fichiers qui sont communs aux deux sources
    # if name in fichiers_existant:
    format = format[1:]

    if file_path.endswith(".pdf"):
        with open(file_path, 'rb') as pdf_file:
            pdf_reader = PyPDF2.PdfReader(file_path)
            num_pages = len(pdf_reader.pages)
            
    elif file_path.endswith(".docx"):
        # python-docx est faux (1 ou 2), 
        # une methode avec zip est tres approximatif
        # seule la conversion payante en pdf semble valable
        pass

    elif file_path.endswith(".pptx"):
        # Ouvrir le fichier pptx
        prs = pptx.Presentation(file_path)
        # Trouver le nombre de pages
        num_pages = len(prs.slides)
    else:
        num_pages = np.nan

    first_text = ''
    if file_path.endswith(".pdf"):
        text = extract_pdf_text(os.path.join(root, file_path))
        text = preprocess_text(text)
        pdf_reader = PyPDF2.PdfReader(file_path)
        first_text = pdf_reader.pages[0].extract_text()
        first_text = preprocess_text(first_text)

        dates = extract_dates(text)

        if dates:
            creation_date = infer_creation_date(dates)
            period_date = find_period(dates)
        else:
            creation_date = get_creation_date(file_path)

        countries = extract_countries(text)
        
        summary = summarize_text(text)

        indicateurs = get_indicateurs(text)

    file_info = [name, format, num_pages, creation_date, period_date, countries, summary, indicateurs, file_path]
    data.append(file_info)

# df_features = pd.DataFrame(data, columns=['Name', 'Type', 'NbPages', 'Creation Date', 'Period Date', 'Countries', 'Summary', 'Path'])
df_features = pd.DataFrame(data, columns=['Name', 'Type', 'NbPages', 'Creation Date', 'Period Date', 'Countries', 'Summary', 'Indicateurs', 'Path'])

I'm running it on colaboratory, maybe I should run it on a more powerful machine? Or do some parallelization?