Slow parsing of mutiple XML files to CSV

Question

The code below is written as part of a parser, which files would be also added here: XMLParser, parser, Name, GenericElement, attrib

It reads an XML files from a folder, turn them into a dataframe using pandas and then parse it into a CSV file.

from generic_parser import XMLParser
import pandas as pd
import os

opdatas = []
df = pd.DataFrame()
tags_to_leave = ['Description', 'FileX', 'FileY']
value_data = ['SetData']

def dropper(obj):
    return obj.name not in tags_to_leave + value_data, True

def opdata_collector(obj):
    cnt = 0
    global df
    if obj.name == 'START':
        obj.recurs_drop(dropper)
        obj.recurs_change(rename_attribs)
        obj = rename_attribs(obj)
        # opdatas.append(obj)
        final_dict = dict(obj.attribs)
        # final_dict.update({'ValueData' : []})
        value_data_dict = {}
        for child in obj.children:
            if str(child.name) in tags_to_leave:
                final_dict.update(dict(child.attribs))
            if str(child.name) in value_data:
                for key_name in child.attribs.keys():
                    print(key_name)
                    final_dict['SetData' + key_name + str(cnt)] = child.attribs[key_name]
                cnt += 1

        df = df.append(final_dict, ignore_index=True)

def rename_attribs(obj):
    if str(obj.name) == 'Description':
        obj.attribs["text"] = obj.text
    obj.attribs.sub_keys(r"^(.*)$", str(obj.name) + r"_\1")
    return obj



if __name__ == '__main__':
    #xml = XMLParser.parse(r'./work.xml')
    #xml.recurs_collect(opdata_collector)
    #df.to_csv("./original.csv")
    directory= 'C:/Users/232/Desktop/32/Files/test'
    for filename in os.listdir(directory):
        if filename.endswith(".xml"):
            xml.recurs_collect(opdata_collector)
            df.to_csv("./output.csv")
            continue
        else:
            continue

This is the XML file

  <?xml version="1.0" encoding="utf-8"?>
<ProjectData>
<FINAL>
    <START id="DG0003" service_code="0x517B">
      <Docs Docs_type="START">
        <Rational>23423</Rational>
        <Qualify>342342</Qualify>
      </Docs>
      <Description num="3423423f3423">The third</Description>
      <SetFile dg="" dg_id="">
        <FileX dg="" axis_pts="2" name="" num="" dg_id="" />
        <FileY tint="" axis_pts="20" name="TOOLS" num="23423" tint_id="" />
        <SetData x="E1" value="21259" />
        <SetData x="E2" value="0" />
      </SetFile>
    </START>
    <START id="ID0048" service_code="0x5198">
      <Docs Docs_type="START">
        <Rational>225198</Rational>
        <Quality >343243324234234</Quality >
      </Docs>
      <Description num="434234234">The forth</Description>
      <Normal tint="m" tint_id="FEDS">
        <FileX be="dssad" get="false" axis_pts="19" name="asdas" num="SDF3" tint_id="SDGFDS" />
        <SetData xin="sdf" xax="233" value="323" />
        <SetData xin="123" xax="213" value="232" />
        <SetData xin="2321" xax="232" value="23" />
        <SetData xin="-inf" xax="45" value="423" />
        <SetData xin="234" xax="3" value="4523" />
        <SetData xin="324" xax="423" value="6456234" />
        <SetData xin="30" xax="45234" value="34532" />
        <SetData xin="5345" xax="2345" value="453" />
        <SetData xin="234" xax="345" value="4543" />
        <SetData xin="2321" xax="2345" value="45" />
        <SetData xin="3423432" xax="34" value="453" />
        <SetData xin="21" xax="34" value="45" />
        <SetData xin="12" xax="34" value="345" />
        <SetData xin="43" xax="423" value="7" />
        <SetData xin="64" xax="32" value="0" />
        <SetData xin="434" xax="254" value="5" />
        <SetData xin="343" xax="322" value="3" />
        <SetData xin="36" xax="2" value="0" />
        <SetData xin="23" xax="done" value="0" />
      </Normal>
    </START>
</FINAL>
</ProjectData>

The code works fine however the speed is not optimal. It takes quite some time to parse multiple XML files into one CSV.

The screenshot below shows the output file (I just removed the data for personal reasons). The columns show how the data is distributed

Answer 1

An easy speed-up is not using a global object that you continually modify. This is especially important since you also save the output file over and over, which is completely unnecessary (unless you care about the current state if you abort the program).

from itertools import chain
from pathlib import Path
...

def opdata_collector(obj):
    if obj.name != 'START':
        return {}
    count = 0
    obj.recurs_drop(dropper)
    obj.recurs_change(rename_attribs)
    obj = rename_attribs(obj)
    final_dict = dict(obj.attribs)
    for child in obj.children:
        if str(child.name) in tags_to_leave:
            final_dict.update(dict(child.attribs))
        elif str(child.name) in value_data:
            final_dict.update({f"ValueData{key_name}{count}": value
                               for key_name, value in child.attribs.items()})
            count += 1
    return final_dict


if __name__ == '__main__':
    directory = Path('C:/Users/z647818/Desktop/Erdi/Files/test')
    data = (XMLParser.parse(directory / file_name).recurs_collect(opdata_collector)
            for file_name in os.listdir(directory)
            if file_name.endswith(".xml"))
    df = pd.DataFrame(chain.from_iterable(data))
    df.dropna().to_csv("./output.csv")

On top of that the actual opdata_collector can probably also be improved, but this should give you a nice boost.

I also removed unneeded lines, used a dictionary comprehension in the inner loop, turned the if into an elif (you probably don't want a tag to be processed twice), spelled out count (no need to conserve bytes), used an f-string instead of string addition and packed the dataframe generation in one big generator expression.