Pythonスクレーピング：データから必要な情報だけを抽出する方法

!mkdir pay && mkdir pay/tmp

!mkdir pay/tmp/capublicpay

# The title of the highest paid California city government position in 2010
# note, the code below makes it easy to extend "years" to include multiple years
import csv
import os.path
import requests
from shutil import unpack_archive
LOCAL_DATADIR = "pay/tmp/capublicpay"
YEARS = range(2009,2017) # i.e. just 2010
def foosalary(row):
    return float(row['Total Wages']) if row['Total Wages'] else 0

for year in YEARS:
    bfname = '%s_City' % year
    url = 'https://publicpay.ca.gov/RawExport/%s.zip' % bfname
    zname = os.path.join("pay/tmp", bfname + '.zip')
    cname = os.path.join(LOCAL_DATADIR, bfname + '.csv')

    if not os.path.exists(zname):
        print("Downloading", url, 'to', zname)
        data = requests.get(url).content
        with open(zname, 'wb') as f:
            f.write(data)
    # done downloading, now unzip files
    print("Unzipping", zname, 'to', LOCAL_DATADIR)
    unpack_archive(zname, LOCAL_DATADIR, format = 'zip')

    with open(cname, encoding = 'latin-1') as f:
        # first four lines are:
        # “Disclaimer
        #
        # The information presented is posted as submitted by the reporting entity. The State Controller's Office is not responsible for the accuracy of this information.”
        data = list(csv.DictReader(f.readlines()))
        topitem = max(data, key = foosalary)
        print(topitem['Entity Name'], topitem['Department / Subdivision'],
            topitem['Position'], topitem['Total Wages'])

ls pay/tmp/capublicpay

a = !head -n 1 pay/tmp/capublicpay/2009_City.csv
print(a)

import pandas as pd
a = pd.read_csv("pay/tmp/capublicpay/2009_City.csv", nrows=0)
print(a)

# The title of the highest paid California city government position in 2010
# note, the code below makes it easy to extend "years" to include multiple years
import csv
import os.path
import requests
from shutil import unpack_archive
LOCAL_DATADIR = "pay/tmp/capublicpay"
YEARS = range(2009,2018) # i.e. just 2010
def foosalary(row):
    return float(row['TotalWages']) if row['TotalWages'] else 0

for year in YEARS:
    bfname = '%s_City' % year
    url = 'https://publicpay.ca.gov/RawExport/%s.zip' % bfname
    zname = os.path.join("pay/tmp", bfname + '.zip')
    cname = os.path.join(LOCAL_DATADIR, bfname + '.csv')

    if not os.path.exists(zname):
        print("Downloading", url, 'to', zname)
        data = requests.get(url).content
        with open(zname, 'wb') as f:
            f.write(data)
    # done downloading, now unzip files
    print("Unzipping", zname, 'to', LOCAL_DATADIR)
    unpack_archive(zname, LOCAL_DATADIR, format = 'zip')

    with open(cname, encoding = 'latin-1') as f:
        # first four lines are:
        # “Disclaimer
        #
        # The information presented is posted as submitted by the reporting entity. The State Controller's Office is not responsible for the accuracy of this information.”
        data = list(csv.DictReader(f.readlines()))
        topitem = max(data, key = foosalary)
        print(topitem['EmployerCounty'], topitem['DepartmentOrSubdivision'],
            topitem['Position'], topitem['TotalWages'])