I have the following fully working code that
- imports JSON files,
- parses the tweets contained in JSONs,
- records them in a table in a data frame.
Considering that per run I currently analyze 1,400 JSONs (about 1.5Gb), the code takes quite some time to run. Please suggest if there is a plausible way to optimize the code in order to increase its speed. Thanks!
import os
import json
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import os
tweets = []
for dirs, subdirs, files in os.walk('/Users/mymac/Documents/Dir'):
for file in files:
if file.endswith('.json'):
print(file)
for line in open(file) :
try:
tweet = json.loads(line)
tweets.append(tweet)
except:
continue
tweet = tweets[0]
ids = [tweet['id_str'] for tweet in tweets if 'id_str' in tweet]
text = [tweet['text'] for tweet in tweets if 'text' in tweet]
lang = [tweet['lang'] for tweet in tweets if 'lang' in tweet]
geo = [tweet['geo'] for tweet in tweets if 'geo' in tweet]
place = [tweet['place'] for tweet in tweets if 'place' in tweet]
df=pd.DataFrame({'Ids':pd.Index(ids),
'Text':pd.Index(text),
'Lang':pd.Index(lang),
'Geo':pd.Index(geo),
'Place':pd.Index(place)})
df