Python AWS Lambda, why did updating in place instead of creating and appending to a new list not decrease memory usage?

Question

I have a lambda function that ingests a CSV, reads it as a list of dicts, modifies them (updating the Keys, adding a few values) and submits them to a firehose. Initially I had the following code:

import json
import boto3
import sys
import csv
import io
import logging

s3 = boto3.client('s3', 'us-east-1')
firehoseClient = boto3.client('firehose','us-east-1')
logger=logging.getLogger()
logger.setLevel(logging.INFO)

fieldMapper = {
    Dict that maps old column names to new ones
}


def lambda_handler(event, context):
    print(f"Received Event: {event}")
    bucket = event['Records'][0]['s3']['bucket']['name']
    key = event['Records'][0]['s3']['object']['key']
    stream = 'stream'
    logger.info(f'Reading {key} from {bucket}')

    obj = s3.get_object(Bucket = bucket, Key = key)
    f = io.StringIO(obj['Body'].read().decode('utf-8'))

    reader = csv.DictReader(f)
    list_of_json = [dict(device) for device in reader]
    f.close()
    logger.info(f'{key} successfully parsed')

    reformDictList = []
    logger.info('Reformatting dicts')
    for i in list_of_json:
        newDict = {}
        for k, v in i.items():
            if k in fieldMapper.keys():
                newDict[fieldMapper[k]] = v
        newDict['ZIPCODE'] = f"{i['ZIP']}-{i['ZIP4']}"
        newDict['CSV'] = f"{i['CITY']}, {i['STATE']} {newDict['ZIPCODE']}"
        newDict['mail_filename'] = key
        newDict['printer_name'] = 'Printer'
        reformDictList.append(newDict)
    logger.info('Dicts reformatted successfully')

    batch = []
    batch_ct = 1
    for i in jlist1:
        i['mail_filename'] = key.split('/')[1]
        text = json.dumps(i)
        if len(text) > 1:
            text_bytes = bytes(text,'utf-8')
            dict_bytes = {"Data":text}
            batch.append(dict_bytes)
        if len(batch) == 500:
            print('Sending batch at line number ' + str(500*batch_ct))
            # try:
            result = firehoseClient.put_record_batch(DeliveryStreamName = stream, Records = batch)
            # except Exception as x:
            #     logging.error(x)

            num_failures = result['FailedPutCount']

            try:
                if num_failures:
                    logging.info(f'resending {num_failures} failed records')
                    rec_index = 0
                    for record in result['RequestResponses']:
                        if 'ErrorCode' in record:
                            firehoseClient.put_record(DeliveryStreamName=stream,Record=batch[rec_index])
                            num_failures -= 1
                            if not num_failures:
                                break
                        rec_index += 1
            except Exception as y:
                logging.error(y)
            batch_ct += 1
            batch.clear()

    if batch:
        print('Sending leftover records')
        try:
            result = firehoseClient.put_record_batch(DeliveryStreamName = stream, Records = batch)
        except Exception as x:
            logging.error(x)

        num_failures = result['FailedPutCount']

        try:
            if num_failures:
                logging.info(f'resending {num_failures} failed records')
                rec_index = 0
                for record in result['RequestResponses']:
                    if 'ErrorCode' in record:
                        firehoseClient.put_record(DeliveryStreamName=stream,Record=batch[rec_index])
                        num_failures -= 1
                        if not num_failures:
                            break
                    rec_index += 1

        except Exception as y:
            logging.error(y)

But it consumed a max memory of 856mb. So I decided to update the code to modify the original JSON in place to avoid creating a new list:

for i in list_of_json:
    i['ZIP'] = f"{i['ZIP']}-{i['ZIP4']}"
    for k in list(i.keys()):
        if k in fieldMapper.keys():
            i[fieldMapper[k]] = i.pop(k)
        else:
            del i[k]
    i['csv'] = f"{i['CITY']}, {i['STATE']} {i['ZIPCODE']}"
    i['mail_filename'] = key
    i['printer_name'] = 'Printer'

Yet to my surprise this made no difference in memory. Why might this be? The size of the CSV is 51.8mb.

Answer 1

New list here just contains links to list_of_json objects, it shouldn't consume much memory, so getting rid of it won't save much either. Variables that reference to all objects (here dicts) are kept until the function is active. I'd suggest split the data flow into chain of functions to reduce temp objects reference count to 0 and make garbage collector get rid of them.