I have the following code:
items = set() # use a set to not have duplicate items
a = requests.get(f"{BASE_URL}?{BASE_QUERIES}&cursor=*")
amount = a.json()["totalResults"] # in the range of 30 million
items.update(item["guid"].split("?")[0] for item in a.json()["items"]) # we only want the url before the query strings
cursor = a.json()["nextCursor"] # we have a cursor as each request only returns 100 items, the cursor shows use where to start
while len(items) < amount: # ensure we get all the items
try:
a = requests.get(f"{BASE_URL}?{BASE_QUERIES}&cursor={cursor}")
items.update(item["guid"].split("?")[0] for item in a.json()["items"])
except:
continue
try:
cursor = urllib.parse.quote(a.json()["nextCursor"])
except KeyError:
if len(items) == amount: # when we reach the final iteration the cursor will not be there
break
headers = {
"user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.150 Safari/537.36"
}
for link in items: # iterate over each item
for _ in range(3):
response = requests.get(link, headers=headers, stream=True)
response.raw.decode_content = True
parser = etree.HTMLParser()
tree = etree.parse(response.raw, parser)
setting = tree.xpath(
"/html/body/div[1]/div/div/main/div/div[2]/div/div[2]/div/div/div[1]/a/span[1]/text()"
)[0].strip()
try:
image = tree.xpath(
"/html/body/div[1]/div/div/main/div/div[2]/div/div[1]/div[1]/div/div/a/@href"
)[0] # most of the time this works
except:
try:
image = tree.xpath(
"/html/body/div[1]/div/div/main/div/div[2]/div/div[1]/div[1]/div[1]/a/@href"
)[0] # about 1 in 10 times the above fails and this works
except:
print(f"Image not found for link: {link}")
break
title = tree.findtext(".//title")
title_for_file = (
fr"{os.getcwd()}\IMAGES\IMAGE - " + "".join(c for c in title if c in valid_chars) + ".jpeg"
) # sometimes the file contains characters which aren't allowed in file names
description = "".join(
tree.xpath(
"/html/body/div[1]/div/div/main/div/div[3]/div[1]/div/div/div/div/p/text()"
)
)
if setting not in [x, y]: # we only want items which have a specific setting
break
try:
image_req = requests.get(image)
with open(title_for_file, "wb") as f:
f.write(image_req.content) # write the image to the directory
img = pyexiv2.Image(title_for_file)
metadata = {"Xmp.xmp.Title": title, "Xmp.xmp.Description": description} # edit the metadata
img.modify_xmp(metadata)
img.close()
break
except Exception as e:
print(
f"ERROR parsing image: {title_for_file}, it is most likely corrupt, "
f"retrying ({link}), "
f"error: {e}"
)
os.remove(title_for_file)
else:
print("ERROR more than 3 times moving to next")
The code has some fairly detailed comments to explain what it does, my issue is that it takes about 7 hours to do all the requests to get the links in items
, then another (I'm not 100% sure but, I think about 1500 hours) to check each image and add the metadata. Is there anything in this code that can be sped up/refined to speed up this process?
Similarly, is there anywhere I could reduce memory usage as I suspect this will use alot of memory with the amount of data it is parsing.
EDIT
A possible consideration would be to use Threading
to download multiple images at once, how would one find the optimal number of threads to run at once? Or perhaps I would start each Thread
with a small delay, perhaps 0.5 seconds?
x
ory
\$\endgroup\$