7
\$\begingroup\$

I’ve written a script for automatically backing up my Goodreads data. This mimics the Export function on the Goodreads website, which returns a CSV file – but this doesn’t require going through a website, and provides JSON instead of CSV. The idea is that I could run this on a cron job, and have regular backups of my user data.

The script can be invoked without any arguments, or with one argument to specify the filename to write the backup to:

$ python3 backup_goodreads.py
$ python3 backup_goodreads.py my backup.json

I’d be interested in any feedback, but particularly around:

  • My use of xml.etree. The Goodreads API mostly serves XML, but I’m not very familiar with this library, so I might not be making the best use of it. For ease of reviewing, I’ve posted an API response in a Gist so you can see what the API responses look like.

  • Is the code easy to follow? It makes a lot of sense to me, but I wrote it! I’m not sure whether it’s easily comprehensible to somebody else.

  • Uncaught errors. Are there any obvious error cases I’m failing to handle correctly? (Assume I trust that if I get a successful response, the XML will have the right structure.)

Here’s the code:

#!/usr/bin/env python3
# -*- encoding: utf-8 -*-
"""
A script for backing up your Goodreads library.  Writes a backup to
`goodreads_backup.json`, or you can provide an alternative name as the
first command-line argument.

See the README for more details.
"""

from datetime import datetime, timezone
import json
import sys
import xml.etree.ElementTree as ET

import keyring
import requests


# Goodreads user ID
USER_ID = '**redacted**'            # keyring.get_password('goodreads', 'user_id')

# Goodreads API key.  Obtain one from https://www.goodreads.com/api
API_KEY = '**redacted**'     # keyring.get_password('goodreads', 'api_key')


class TagDescriptor(object):
    """
    Used internally by the Review class.  The review class has an ETree
    element representing an XML document of the form:

        <review>
          <tag1>value1</tag1>
          <tag2>value2</tag2>
          <book>
            <book_tag1>book_value1</book_tag1>
            <book_tag2>book_value2</book_tag2>
          </book>
        </review>

    This descriptor provides a cleaner interface for reading values from
    the XML.

    :param tag_name: Name of the tag in the XML.  If prefixed with 'book/',
        reads the attribute from within <book>.
    :param factory: Applied to the value before returning, if supplied.
        For example, passing ``factory=int`` would cast the value to
        an integer.  Otherwise the value is passed through as a string.
    """
    def __init__(self, tag_name, factory=None):
        self.tag_name = tag_name
        self.factory = factory
        if factory is None:
            self.factory = lambda x: x

    def __get__(self, obj, type):
        assert type is Review
        if self.tag_name.startswith('book/'):
            val = obj._data.find('book').find(self.tag_name.split('/')[1]).text
        else:
            val = obj._data.find(self.tag_name).text
        return self.factory(val)


def date_factory(date_str):
    """
    Convert a date string returned by the Goodreads API into an
    ISO-8601 UTC string.
    """
    # We may get ``None`` if the API is missing any information for a
    # particular date field -- for example, the ``date_read`` field is
    if date_str is None:
        return None
    else:
        # In the API responses, dates are returned in the form
        # "Mon Oct 24 12:26:31 -0700 2016"
        date_obj = datetime.strptime(date_str, '%a %b %d %H:%M:%S %z %Y')
        return str(date_obj.astimezone(timezone.utc))


class Review(object):
    """
    Wrapper class around an xml.etree.ElementTree.Element object that
    contains the data from a review.  Does all the messy handling of
    getting the interesting bits out of the XML response.
    """

    def __init__(self, data):
        self._data = data

    book_id             = TagDescriptor('book/id')
    title               = TagDescriptor('book/title')
    isbn                = TagDescriptor('book/isbn')
    isbn13              = TagDescriptor('book/isbn13')
    average_rating      = TagDescriptor('book/average_rating')
    publisher           = TagDescriptor('book/publisher')
    binding             = TagDescriptor('book/format')
    page_count          = TagDescriptor('book/num_pages', int)
    year_published      = TagDescriptor('book/publication_year')
    orig_year_published = TagDescriptor('book/published')
    date_read           = TagDescriptor('read_at', date_factory)
    date_added          = TagDescriptor('date_added', date_factory)
    review              = TagDescriptor('body', lambda x: x.strip())

    @property
    def authors(self):
        return [
            a.find('name').text
            for a in self._data.find('book').find('authors').findall('author')
        ]

    @property
    def my_rating(self):
        # The Goodreads API returns '0' to indicate an unrated book; make
        # this a proper null type.
        rating = self._data.find('rating').text
        if rating == '0':
            return None
        else:
            return rating

    @property
    def bookshelves(self):
        return [
            shelf.attrib['name']
            for shelf in self._data.find('shelves').findall('shelf')
        ]


def asdict(obj):
    """
    Given an object, return a dictionary of its public attributes suitable
    for JSON serialisation.
    """
    return {
        key: getattr(obj, key)
        for key in dir(obj) if not key.startswith('_')
    }


def get_reviews(page_no=1):
    """
    Generate all the reviews associated with a Goodreads account.

    :param page_no: (optional) API results are paginated in batches of 200,
        which page to start on.
    """
    # reviews.list (https://www.goodreads.com/api/index#reviews.list) gets
    # all the books on somebody's shelf.
    req = requests.get('https://www.goodreads.com/review/list.xml', params={
        'v': '2',
        'key': API_KEY,
        'id': USER_ID,
        'page': str(page_no)
    })
    if req.status_code != 200:
        print(
            "Unexpected error code from Goodreads API: %s\n"
            "Error message: %r" % (req.status_code, req.text),
            file=sys.stderr
        )
        sys.exit(1)

    root = ET.fromstring(req.text)
    for review in root.find('reviews').findall('review'):
        yield Review(review)

    # Do we need to get the next page?
    # TODO: Have enough books to test this step!
    total = int(root.find('reviews').attrib['total'])
    end = int(root.find('reviews').attrib['end'])
    if end < total:
        yield from get_reviews(page_no=page_no+1)


def main():
    try:
        path = sys.argv[1]
    except IndexError:
        path = 'goodreads_backup.json'
    json_str = json.dumps(
        [asdict(rev) for rev in get_reviews()],
        indent=2,
        sort_keys=True
    )
    with open(path, 'w', encoding='utf-8') as f:
        f.write(json_str)
    print('Written backup to %s' % path)


if __name__ == '__main__':
    main()

The master version of this code, along with a README and a requirements.txt, is on GitHub.

\$\endgroup\$
3
  • \$\begingroup\$ @Graipher Thanks for the redaction. It’s actually a dummy API key, but I suppose I could have made that clearer in the comments. =) \$\endgroup\$ – alexwlchan Nov 7 '16 at 18:08
  • 1
    \$\begingroup\$ Fair enough. Didn't know about that keyring module before, looks good. Was still typing my previous comment when you replied. \$\endgroup\$ – Graipher Nov 7 '16 at 18:25
  • \$\begingroup\$ @Graipher Sure, sorry for any confusion. \$\endgroup\$ – alexwlchan Nov 7 '16 at 18:37
3
\$\begingroup\$

1. Review

  1. This code uses xml.etree.ElementTree. But the documentation says:

    Warning The xml.etree.ElementTree module is not secure against maliciously constructed data. If you need to parse untrusted or unauthenticated data see XML vulnerabilities.

    Perhaps you're happy to trust the Goodreads API. But I think it's worth mentioning the concern.

  2. It would make sense to pass in the user id and API key as parameters to the get_reviews function.

  3. The function name date_factory could be improved: what this function does is to convert a date from Goodreads format to your preferred format. So I would call it convert_date.

  4. If the results span multiple pages, get_reviews calls itself recursively. But Python doesn't have tail-recursion elimination, so this will eventually result in a stack overflow if there are enough pages. It would be better to write a loop. See the revised code below.

  5. The code reads the API output as an XML document, and then builds Review objects, each of which wraps a element in the XML document. The Review class knows how to search the XML document for each of the properties it wants (via the TagDescriptor descriptors, which call find on the XML element).

    I don't like this approach because (i) it seems overly complex (why keep the XML document around as your data store, when you could convert it once to some more convenient representation?); and (ii) it has the wrong complexity: finding the attributes for a review should be an \$O(n)\$ operation (it should only be necessary to pass over the children of each XML review element once), but the implementation searches for each attribute, making this \$Ω(n^2)\$.

    I think the code would be simpler and clearer if it iterated once over the children of each <review> element, applying a converter function to each attribute as it is encountered. This would allow you to drop the TagDescriptor and Review classes, and the asdict function too, reducing the length and complexity of the code.

2. Revised code

(Untested, but if I made mistakes then I'm sure you can figure out how to fix them.)

# Identity function (for attributes that don't need conversion).
identity = lambda x: x

def convert_rating(element):
    # The Goodreads API returns '0' to indicate an unrated book; make
    # this a proper null type.
    rating = element.text
    if rating == '0':
        return None
    else:
        return rating

def convert_authors(element):
    return [a.find('name').text for a in element.findall('author')]

def convert_shelves(element):
    return [s.find('name').text for s in element.findall('shelf')]

# Map from <review> tag name to pair (output key, converter)
REVIEW_TAGS = {
    'read_at': ('date_read', convert_date),
    'date_added': ('date_added', convert_date),
    'body': ('review', str.strip),
    'rating': ('my_rating', convert_rating),
    'shelves': ('bookshelves', convert_shelves),
}

# Map from <book> tag name to pair (output key, converter)
BOOK_TAGS = {
    'authors': ('authors', convert_authors),
    'id': ('book_id', identity),
    'title': ('title', identity),
    'isbn': ('isbn', identity),
    'isbn13': ('isbn13', identity),
    'average_rating': ('average_rating', identity),
    'publisher': ('publisher', identity),
    'format': ('binding', identity),
    'num_pages': ('page_count', int),
    'publication_year': ('year_published', identity),
    'published': ('orig_year_published', identity),
}

from itertools import count

def get_reviews(user_id, api_key):
    """Generate reviews associated with a Goodreads user as dictionaries.

    :param user_id: The id of the Goodreads user.
    :param api_key: The Goodreads API key.

    """
    for page_no in count(1):
        # ... code for calling the API goes here ...
        reviews = ET.fromstring(req.text).find('reviews')
        for review in reviews.findall('review')
            data = {}
            def convert(element, tag_mapping):
                if element.tag in tag_mapping:
                    key, converter = tag_mapping[element.tag]
                    data[key] = converter(element.text)
            for review_elt in review:
                if review_elt.tag == 'book':
                    for book_elt in review_elt:
                        convert(book_elt, BOOK_TAGS)
                else:
                    convert(review_elt, REVIEW_TAGS)
            yield data
        if int(reviews.attrib['end']) >= int(reviews.attrib['total']):
            break
\$\endgroup\$

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Not the answer you're looking for? Browse other questions tagged or ask your own question.