This Python module contains helper functions to download the result page of SEDE queries and extract columns from it, most prominently:
fetch_sede_soup(label, url):
download the page of SEDE results, cache it or else use previously cached content, and return as a BeautifulSoup instancefetch_table(label, url):
fetch a SEDE page as a table, returned as a tuple of:- column meta data as a dictionary of name -> data
- rows as a list of lists
- other helper functions used by the above main functions
Here's the code:
import json
import logging
import os
import requests
from bs4 import BeautifulSoup
BASE_DIR = os.path.dirname(__file__)
CACHE_DIR = os.path.join(BASE_DIR, '.cache')
def fetch_sede_soup(label, url):
"""
Download the result page of a SEDE query and create a BeautifulSoup from it.
If the page contains results, cache it in a file.
If the page doesn't contain results, use the cache instead.
Note: this happens when the SEDE query is not executed in the browser
for a few days.
:param label: a simple name to represent the URL, it will be used as the cache filename
:param url: the URL to download
:return: a BeautifulSoup instance from the URL
"""
cache_path = os.path.join(CACHE_DIR, '{}.html'.format(label))
debug_cache_path = os.path.join(CACHE_DIR, '{}-debug.html'.format(label))
if not os.path.isdir(CACHE_DIR):
os.mkdir(CACHE_DIR)
logging.info('fetching {} as {}'.format(label, url))
html = requests.get(url).text
soup = BeautifulSoup(html)
def is_valid(soup):
for script in soup.findAll('script'):
if 'resultSets' in script.text:
return True
return False
def write_cache(path, html):
with open(path, 'w') as fh:
fh.write(html)
if is_valid(soup):
logging.info('updating cache')
write_cache(cache_path, html)
return soup
else:
logging.warning('result not valid')
write_cache(debug_cache_path, html)
if os.path.exists(cache_path):
logging.info('using previous cache')
with open(cache_path) as fh:
return BeautifulSoup(fh)
else:
logging.error('no previous cache: you must download the page manually')
return BeautifulSoup()
def fetch_table(label, url):
"""
Fetch a URL using `fetch_soup` and extract a table as a tuple of {cols} and [rows].
- {cols} is a mapping of column names to column meta data
- [rows] is a list of rows in the table
:param label: a simple name to represent the URL, it will be used as the cache filename
:param url: the URL to download
:return: a tuple of ({cols}, [rows])
"""
soup = fetch_sede_soup(label, url)
if not soup:
return {}, []
return extract_table(soup)
def transform_columns_meta(se_columns_meta):
"""
Transform SE column meta data, for example,
from:
[
{'name': 'User Link', 'type': 'User'},
{'name': 'Post Link', 'type': 'Post'}
]
to:
{
'User Link': {'name': 'User Link', 'type': 'User', 'index': 0},
'Post Link': {'name': 'Post Link', 'type': 'Post', 'index': 1}
}
:param se_columns_meta: list of dictionaries describing the fields
:return: dictionary of dictionaries, with index added
"""
columns_meta = {}
for index, se_col_meta in enumerate(se_columns_meta):
col_meta = {'index': index}
col_meta.update(se_col_meta)
columns_meta[se_col_meta['name']] = col_meta
return columns_meta
def extract_table(soup):
"""
Return a tuple of:
- meta data of columns as a dictionary
- list of rows
:param soup: a bs4 (BeautifulSoup) object
:return: a tuple of ({cols}, [rows])
"""
for script in soup.findAll('script'):
result_sets_col = 'resultSets'
if result_sets_col in script.text:
start = script.text.rindex('{', 0, script.text.index(result_sets_col))
end = script.text.index('}', script.text.index('querySetId')) + 1
data = json.loads(script.text[start:end])
results = data[result_sets_col][0]
columns = transform_columns_meta(results['columns'])
rows = results['rows']
return columns, rows
return {}, []
def extract_column(soup, colname):
"""
Return a generator of cell values in selected column.
For simple columns like timestamp, a cell value can be simple,
for example: 1414433013197
For more complex columns like Post Link, a cell value can be an object,
for example:
{
"id": 68102,
"title": "Bash Script - File Comment out & Notate"
}
:param soup: a bs4 (BeautifulSoup) object
:param colname: name of the SEDE column to extract
:return: generator of cell values in selected column
"""
cols, rows = extract_table(soup)
if colname not in cols:
return
index = cols[colname]['index']
for row in rows:
yield row[index]
An example usage of the module, getting the post ids:
cols, rows = sede.fetch_table('naruto', 'http://data.stackexchange.com/codereview/query/264586/naruto-accepted-answer-with-zero-score')
answer_id_index = cols['Post Link']['index']
post_ids = [row[answer_id_index]['id'] for row in rows]
I'm looking for feedback and criticism of any aspect of the code.