RESTful HTTP Post with falcon

Question

I wrote a small aggregator app that aggregates values from a json http post request and outputs the aggregated values. Now the aggregator function is somewhat large but the output appears correct.

I start it with gunicorn:

$ gunicorn --workers 2 --threads 4 aggregator:api

I test it with curl :

$ curl -X POST -d @aggregator.json http://localhost:8000/aggregate

It outputs the aggregated values.

import falcon
import json
from datetime import datetime
from dateutil import tz
import requests
import time

from_zone = tz.gettz('UTC')
to_zone = tz.gettz('Europe/Stockholm')

class DataService:

    def on_post(self, req, resp):
        data = json.loads(req.stream.read().decode('utf-8'))
        # output the data, we could write it to persistent storage here
        print(data)

class AggregatorService:

    def zero_if_none(self, value):
        if value is not None:
            x = int(value)
        else:
            x = 0
        return x

    # return the start day of consumption e.g. 1 for 2014-12-01
    def get_day_start(self, hours):
        return int(datetime.fromtimestamp(
            int(hours[0][0])
        ).strftime('%d'))

    # return the month number e.g. 2 for February
    def get_month_start(self, hours):
        return int(datetime.fromtimestamp(
            int(hours[0][0])
        ).strftime('%m'))

    # return the day of month number e.g. 5 for 2015-01-05
    def get_day(self, hour):
        return int(datetime.fromtimestamp(
                int(hour[0])
            ).strftime('%d'))

    # return the month number for a timestamp
    def get_month(self, hour):
        return int(datetime.fromtimestamp(
                int(hour[0])
            ).strftime('%m'))

    def on_post(self, req, resp):

        data = json.loads(req.stream.read().decode('utf-8'))
        hours = data['hours']

        day_start = self.get_day_start(hours)
        month_start = self.get_month_start(hours)

        aggr_daily_wh = 0
        aggr_monthly_wh = 0

        aggr_daily_th = 0
        aggr_monthly_th = 0
        jdict = {}

        jdict['user'] = data['user']

        jhours = []
        jdays = []
        jmonths = []

        last_h = 0
        last_day_wh = 0
        last_day_th = 0
        last_month_wh = 0
        last_month_th = 0

        for hour in hours:

            day = self.get_day(hour)
            print("day %d" % day)
            month = self.get_month(hour)
            print("month %d" % month)

            utime = datetime.fromtimestamp(
                int(hour[0])
            ).strftime('%Y-%m-%d %H:%M:%S')

            utc = datetime.strptime(utime, '%Y-%m-%d %H:%M:%S')

            # Tell the datetime object that it's in UTC time zone since
            # datetime objects are 'naive' by default
            utc = utc.replace(tzinfo=from_zone)

            # Convert time zone and change the timestamp
            tstamp = int(time.mktime(utc.astimezone(to_zone).timetuple()))

            # consumption is 0 if there is no value
            wh = self.zero_if_none(hour[1])
            th = self.zero_if_none(hour[2])

            # append hourly comsumption
            jhours.append([tstamp, wh, th])

            if day == day_start:  # aggregate daily comsumption
                aggr_daily_wh += wh
                aggr_daily_th += th
            else:  # new day
                # append daily comsumption
                jdays.append([tstamp, aggr_daily_wh, aggr_daily_th])
                # begin new day
                day_start = day
                aggr_daily_wh = 0
                aggr_daily_th = 0
                aggr_daily_wh += wh
                aggr_daily_th += th

            if month == month_start:  # aggregate monthly consumption
                print("adding from month %d" % month)
                aggr_monthly_wh += wh
                aggr_monthly_th += th
            else:  # new month
                # append monthly comsumption
                jmonths.append([int(tstamp), aggr_monthly_wh, aggr_monthly_th])
                # begin new month
                month_start = month
                aggr_monthly_wh = 0
                aggr_monthly_th = 0
                aggr_monthly_wh += wh
                aggr_monthly_th += th

            # make the values from the last iteration visible outside the loop
            last_h = tstamp
            last_month_wh = aggr_monthly_wh
            last_month_th = aggr_monthly_th
            last_day_wh = aggr_daily_wh
            last_day_th = aggr_daily_th

        # append the last values
        jdays.append([last_h, last_day_wh, last_day_th])
        jmonths.append([last_h, last_month_wh, last_month_th])

        # create the json dictionary
        jdict['hours'] = jhours
        jdict['days'] = jdays
        jdict['months'] = jmonths

        r = requests.post('http://localhost:8000/store', json=jdict)

api = falcon.API()
api.add_route('/aggregate', AggregatorService())
api.add_route('/store', DataService())

aggregator.json

{
  "user": 42,
  "hours": [
    [
      1417392000,
      1,
      null
    ],
    [
      1417395600,
      2,
      3
    ],
    [
      1417647600,
      3,
      2
    ],
    [
      1417651200,
      null,
      6
    ],
    [
      1422835200,
      6,
      2
    ],
    [
      1423094400,
      4,
      3
    ]
  ]
}

Answer 1

Store disimillar data as a dict, not a list

In each hour in hours of your JSON, you have a list that stores:

• a date in Unix format
• two integers that can also be null

It would be clearer and more standard if, instead of a list, you used a dict. As in:

  "hours": [
     {
       (your_label_1): 1417392000,
       (your_label_2): 1,
       (your_label_3): null
     },

This would aid greatly in getting across what the code is doing. (I still don't exactly what the second and third values represent. Neither of the wh and th abreviations are really very helpful.)

Perform all time operations in Unix time

An advantage of Unix time is that dates and times can be added and subtracted without separate handlers to take care of days, hours etc.

This would dramatically shorten your on_post function.

So, first do all operations in Unix time format (UTC time zone), and finally convert to days/hours/minutes only when you need to display the time information to the user.

(Note: Your wh and th variables (in units of hours in your code) would need to be changed to units of seconds. This could either be done by extracting the hour value from the JSON and multiplying by 60, or changing the JSON format to seconds to match the Unix times.)

(Also note: If you have no control over the JSON format, then most of my points above won't exactly apply to you.)