3
\$\begingroup\$

This is built with Python 2.7.15.

The goal of this script is to count the number of words spoken by each Senator on the floor of Congress between given dates.

It pulls from the Congressional Record, cuts out the sections not spoken on the floor (eg lists of Amendments) and returns a count of words by senator. The current list of senators is from the 115th Congress, from 2017 to 2018

# -*- coding: utf-8 -*-
from requests import get  # to make GET request
import time
from datetime import date, timedelta
import os.path
import textract
import re
import csv
import sys

#download URLs
def downloadPDF(url, file_name):
    #get request
    response = get(url)
    content_type = response.headers.get('content-type')

    if content_type == "application/pdf":
        # open in binary mode
        save_path = '/Users/One/Document/Workspace/Projects/Senate/Congressional_Record/'
        path = os.path.join(save_path, file_name)

        with open(path, "wb") as file:
            #write to file
            file.write(response.content)
            return 'SUCCESS'

#download congressional records and extract text
def downloadConRecords(startDate, endDate): #YYYYMMDD
    #initialize lists
    congressionalRecords = []
    dates = []

    #convert args to date-type
    startDate = str(startDate)
    endDate = str(endDate)
    startDate = date(int(startDate[0:4]),int(startDate[4:6]),int(startDate[6:8]))
    endDate = date(int(endDate[0:4]),int(endDate[4:6]),int(endDate[6:8]))

    #list all the days to check
    range = endDate - startDate
    i=0
    while i <= range.days:
        dates += [str(startDate + timedelta(days=i))]
        i+=1

    #try URL, download PDF, add to list object
    for i in dates:
        YYYY = i[0:4]
        MM = i[5:7]
        DD = i[8:10]
        url = ('https://www.congress.gov/crec/' + YYYY + '/'+ MM + '/' + DD + '/CREC-' + YYYY + '-' + MM + '-' + DD + '-senate.pdf')
        filename = (YYYY+MM+DD+".pdf")

        time.sleep(20)

        if downloadPDF(url,filename) == "SUCCESS":
            print(url)
            textExtract = []
            textExtract = textract.process('/Users/One/Document/Workspace/Projects/Senate/Congressional_Record/' + filename)
            #makes a list with the filename and the text
            entry = [url, filename[:-4].strip(), textExtract]
            congressionalRecords += [entry]

    return congressionalRecords

#process the congressional records
def cleanRecords(congressionalRecords, phraseDelete, segmentStart, segmentEnd):
    #eliminate n/, phrases, segments
    for record in congressionalRecords:
        clean = []
        dump = []
        clean = record[2].split('\n')
        clean = " ".join(clean)

        for i in phraseDelete:
            clean = clean.replace(i,'')

        for start in enumerate(segmentStart):
            erase = re.findall('{}.*?{}'.format(start[1], segmentEnd), clean)
            clean = re.sub('{}.*?{}'.format(start[1], segmentEnd), ' ', clean)
            dump += [erase]

        record += [clean]
        record += [dump]

    return congressionalRecords

#split the text into sections based on the speaker
def multiDelimStringSplitter(aString, separators):
    # separators is an array of strings that are being used to split the the string.
    # sort separators in order of descending length
    separators.sort(key=len)
    listToReturn = []
    rows = []
    l = []
    i = 0

    while i < len(aString):
        theSeparator = ""
        for current in separators:
            if current == aString[i:i+len(current)]:
                #if this section is the separateor, then set the variable to that separator
                theSeparator = current
        if theSeparator != "":
            listToReturn += [theSeparator]
            i = i + len(theSeparator)
        else:
            if listToReturn == []:
                listToReturn = [""]
            if(listToReturn[-1] in separators):
                listToReturn += [""]
            #adds the text one character at a time
            listToReturn[-1] += aString[i]
            i += 1

    i = 0
    listToReturn = listToReturn[1:len(listToReturn)]

    while (i+1) < len(listToReturn):
        #initialize entry with speaker and text
        entry = listToReturn[i:i+2]
        #get word count and add it to the entry
        text = listToReturn[i+1]
        #strip punctuation out of the string
        text = re.sub(r'[^\w\s]','',text)
        wordcount = str(len(text.split()))
        entry += [wordcount]
        #add the entry as its own list
        rows.append(entry)
        i += 2

    return rows

#get information on the records, speakers and word counts
def writeAnalytics(congressionalRecords, senators):
    textExport = [['url','date','extractRaw','extractClean']]
    textDump = [['date','extractDump']]
    speechExport = [['date','speaker','text','wordCount']]
    speakerExport = [['date','speaker','wordCount']]
    speakerAnalytics = [['startDate', 'endDate', 'speaker', 'wordCount']]
    perSpeakerTotal = []
    recordDates = []

    for i in senators:
        # wordsperSpeech += [i,0]
        perSpeakerTotal += [i,0]

    #add delimiter separated text extracts
    for record in congressionalRecords:
        splitLines = []
        wordsperSpeaker = []

        #initialize the lists
        for i in senators:
            # wordsperSpeech += [i,0]
            wordsperSpeaker += [i,0]

        #split the extracts into each speech
        splitLines = multiDelimStringSplitter(record[3],senators)
        record += [splitLines]

        #prep the dump export
        for i in record[4]:
            if len(i) > 0:
                textDump += [[record[1], i]]

        #add the words per speaker
        for speech in record[5]:
            speechExport += [[record[1],speech[0],speech[1],speech[2]]]
            #find the speaker -> pull that person's wordsperSpeaker and add the speech's word count
            wordsperSpeaker[wordsperSpeaker.index(speech[0])+1] += int(speech[2])

        record += [wordsperSpeaker]
        textExport += [[record[0],record[1],record[2],record[3]]]
        recordDates += [record[1]]

        #prepare export for the words per speaker
        for i in senators:
            speakerExport += [[record[1],i, wordsperSpeaker[wordsperSpeaker.index(i)+1]]]
            #add word counts for senators who spoke
            #find senator's index in perSpeaker total -> add the words spoken in that session (record)
            perSpeakerTotal[perSpeakerTotal.index(i)+1] += record[6][record[6].index(i)+1]

    for i in senators:
        speakerAnalytics += [[min(recordDates), max(recordDates), i, perSpeakerTotal[perSpeakerTotal.index(i)+1]]]

    return textExport, textDump, speechExport, speakerExport, speakerAnalytics

#export to CSV
def exportCSV(flatList, filename):
    with open(filename+".csv", "wb") as f:
        writer = csv.writer(f)
        writer.writerows(flatList)

phraseDelete = [
',',
'This ‘‘bullet’’ symbol identifies statements or insertions which are not spoken by a Member of the Senate on the floor.'
]

segmentStart = [
' PRAYER ',
' PLEDGE OF ALLEGIANCE ',
' APPOINTMENT OF ACTING PRESIDENT PRO TEMPORE ',
' CERTIFICATES OF ELECTION ',
' MESSAGE FROM THE HOUSE ',
' MESSAGES FROM THE PRESIDENT ',
' EXECUTIVE MESSAGES REFERRED ',
' LIST OF SENATORS BY STATES ',
' PRESIDENTIAL MESSAGES ',
' ENROLLED BILLS PRESENTED ',
' MEASURES REFERRED ',
' EXECUTIVE AND OTHER COMMUNICATIONS ',
' REPORTS OF COMMITTEES ',
' EXECUTIVE REPORTS OF COMMITTEES ',
' AMENDMENTS SUBMITTED AND PROPOSED ',
' INTRODUCTION OF BILLS AND JOINT RESOLUTIONS ',
' ADDITIONAL COSPONSORS ',
' SUBMITTED RESOLUTIONS ',
' SUBMISSION OF CONCURRENT AND SENATE RESOLUTIONS ',
' SENATE RESOLUTION ',
' TEXT OF AMENDMENTS ',
' APPOINTMENT ',
' AUTHORITY FOR COMMITTEES TO MEET ',
' CONFIRMATION ',
' NOMINATION '
]

segmentEnd = ' f '

senators = ['Mr. SESSIONS', 'Mr. STRANGE', 'Mr. JONES', 'Mr. SHELBY', 'Mr. SULLIVAN',  'Ms. MURKOWSKI',
'Mr. FLAKE', 'Mr. MCCAIN', 'Mr. KYL', 'Mr. COTTON', 'Mr. BOOZMAN', 'Mrs. FEINSTEIN', 'Ms. HARRIS',
'Mr. GARDNER', 'Mr. BENNET', 'Mr. MURPHY', 'Mr. BLUMENTHAL', 'Mr. CARPER', 'Mr. COONS', 'Mr. NELSON', 'Mr. RUBIO',
'Mr. PERDUE', 'Mr. ISAKSON', 'Ms. HIRONO', 'Mr. SCHATZ', 'Mr. RISCH', 'Mr. CRAPO', 'Mr. DURBIN',
'Ms. DUCKWORTH', 'Mr. DONNELLY', 'Mr. YOUNG', 'Mrs. ERNST', 'Mr. GRASSLEY', 'Mr. ROBERTS', 'Mr. MORAN',
'Mr. MCCONNELL', 'Mr. PAUL', 'Mr. CASSIDY', 'Mr. KENNEDY', 'Mr. KING', 'Ms. COLLINS', 'Mr. CARDIN',
'Mr. VAN HOLLEN', 'Ms. WARREN', 'Mr. MARKEY', 'Ms. STABENOW', 'Mr. PETERS', 'Ms. KLOBUCHAR',
'Mr. FRANKEN', 'Ms. SMITH', 'Mr. WICKER', 'Mr. COCHRAN', 'Mrs. HYDE-SMITH',  'Mrs. MCCASKILL',
'Mr. BLUNT', 'Mr. TESTER', 'Mr. DAINES', 'Mrs. FISCHER', 'Mr. SASSE', 'Mr. HELLER', 'Ms. CORTEZ MASTO',
'Mrs. SHAHEEN', 'Ms. HASSAN',  'Mr. MENENDEZ', 'Mr. BOOKER', 'Mr. HEINRICH', 'Mr. UDALL',
'Mrs. GILLIBRAND', 'Mr. SCHUMER', 'Mr. TILLIS', 'Mr. BURR', 'Ms. HEITKAMP', 'Mr. HOEVEN',
'Mr. BROWN', 'Mr. PORTMAN', 'Mr. INHOFE', 'Mr. LANKFORD', 'Mr. MERKLEY', 'Mr. WYDEN',
'Mr. CASEY', 'Mr. TOOMEY', 'Mr. WHITEHOUSE', 'Mr. REED', 'Mr. GRAHAM',
'Mr. SCOTT', 'Mr. ROUNDS', 'Mr. THUNE', 'Mr. CORKER', 'Mr. ALEXANDER', 'Mr. CRUZ',
'Mr. CORNYN', 'Mr. HATCH', 'Mr. LEE', 'Mr. SANDERS', 'Mr. LEAHY', 'Mr. KAINE',
'Mr. WARNER', 'Ms. CANTWELL', 'Mrs. MURRAY', 'Mr. MANCHIN', 'Mrs. CAPITO',
'Ms. BALDWIN', 'Mr. JOHNSON', 'Mr. BARRASSO', 'Mr. ENZI',
'The ACTING PRESIDENT', 'The PRESIDING OFFICER', 'The VICE PRESIDENT'
'Executive nominations confirmed by']

startDate = '20170106'
endDate = '20170130'
conRecords = downloadConRecords(startDate, endDate)
cleanConRecords = cleanRecords(conRecords, phraseDelete, segmentStart, segmentEnd)
textExport, textDump, speechExport, speakerExport, speakerAnalytics = writeAnalytics(conRecords, senators)

exportCSV(textExport,'textExport')
exportCSV(textDump,'textDump')
exportCSV(speechExport,'speechExport')
exportCSV(speakerExport,'speakerExport')
exportCSV(speakerAnalytics,'speakerAnalytics')
\$\endgroup\$
  • \$\begingroup\$ Anyone have suggestions on better ways to write this? \$\endgroup\$ – Sebastian Feb 6 at 18:51

Your Answer

By clicking “Post Your Answer”, you agree to our terms of service, privacy policy and cookie policy

Browse other questions tagged or ask your own question.