From 00c5d345b177f3ef96fe8b79ba0a59857dee12e3 Mon Sep 17 00:00:00 2001 From: prasket Date: Sun, 6 Sep 2020 19:19:23 -0700 Subject: [PATCH] removed the work on filte for now, cleaned up files and repo structure. This branch is complete and ready to be merged into main. Also releasing this version to the site. --- README.md | 27 ++++- api.py | 100 ++++-------------- .../create_covid-19_zip.py | 0 load_db.py => database/load_db.py | 0 .../source_data}/covid-19.xlsx | Bin .../source_data}/covid-19_20200810_zip.csv | 0 .../source_data}/covid-19_20200824_zip.csv | 0 .../source_data}/covid-19_20200831_zip.csv | 0 ...ovid-data-extract-geography-august-10.xlsx | Bin ...ovid-data-extract-geography-august-24.xlsx | Bin ...ovid-data-extract-geography-august-31.xlsx | Bin 11 files changed, 43 insertions(+), 84 deletions(-) rename create_covid-19_zip.py => database/create_covid-19_zip.py (100%) rename load_db.py => database/load_db.py (100%) rename {source_data => database/source_data}/covid-19.xlsx (100%) rename {source_data => database/source_data}/covid-19_20200810_zip.csv (100%) rename {source_data => database/source_data}/covid-19_20200824_zip.csv (100%) rename {source_data => database/source_data}/covid-19_20200831_zip.csv (100%) rename {source_data => database/source_data}/covid-data-extract-geography-august-10.xlsx (100%) rename {source_data => database/source_data}/covid-data-extract-geography-august-24.xlsx (100%) rename {source_data => database/source_data}/covid-data-extract-geography-august-31.xlsx (100%) diff --git a/README.md b/README.md index 85ef265..44d2af4 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,6 @@ API to help make public data easier to access. Idea started with need for King County, WA COVID-19 data in API format while only having an excel file for source data. - ## Development Setup ### Datastore @@ -20,6 +19,22 @@ docker ps # to find the CONTAINER ID docker exec -it 05b3a3471f6f bash ``` +From the shell login to the db and setup initial user and database + +```bash +psql -U postgres +``` + +Create your creds.py from the creds_sample.py + +```bash +cp creds_sample.py creds.py +``` + +Edit creds.py with correct info for your db. + +More info coming soon on using the scripts in database/ to create tables and load them with data. + ### Development Env. Project is using Python 3.7 @@ -29,8 +44,16 @@ In the project we use [pipenv](https://pipenv.pypa.io/en/latest/) when developin ```bash pipenv install && pipenv shell ``` +### Production + +When running on Debian server libpq-dev is required, this is postgresl library. + +```bash +sudo apt-get install libpq-dev +``` ### Data Available -- King County, Washington COVID-19 Data +- King County, Washington COVID-19 Data + - [King County, WS Covid-19 by ZipCode](https://api.seattlematrix.org/v1/kingcounty/covid-19/zipcode/all) - sample source url https://www.kingcounty.gov/depts/health/covid-19/data/~/media/depts/health/communicable-diseases/documents/C19/data/covid-data-extract-geography-august-24.ashx \ No newline at end of file diff --git a/api.py b/api.py index 0f65c6e..e6ca440 100644 --- a/api.py +++ b/api.py @@ -7,104 +7,40 @@ app = flask.Flask(__name__) app.config["DEBUG"] = True import creds -# assign secrets to variables +# Assign secrets to variables, should look into making this +# a function. Plan to move this to Vault from creds file. dbhost = creds.dblogin['host'] database = creds.dblogin['db'] username = creds.dblogin['username'] password = creds.dblogin['password'] port = creds.dblogin['port'] - -# Create some test data this will eventually be in a Postgresql db. -zipcodes = [ - {'id': 0, - 'date_added': '20200810', - 'date_data': '20200810', - 'zipcode': '98121', - 'population': '22274', - 'tests': '3182', - 'test_rate': '142.86', - 'positives': '64', - 'positive_rate': '2.87', - 'hospitalizations': '4', - 'hospitalization_rate': '0.18', - 'deaths INTEGER': '2', - 'death_rate':'0.09' - }, - {'id': 1, - 'date_added': '20200810', - 'date_data': '20200810', - 'zipcode': '98109', - 'population': '34939', - 'tests': '6142', - 'test_rate': '175.79', - 'positives': '206', - 'positive_rate': '5.9', - 'hospitalizations': '38', - 'hospitalization_rate': '1.09', - 'deaths INTEGER': '19', - 'death_rate':'0.54' - }, - {'id': 2, - 'date_added': '20200810', - 'date_data': '20200810', - 'zipcode': '98116', - 'population': '26979', - 'tests': '5259', - 'test_rate': '194.93', - 'positives': '121', - 'positive_rate': '4.48', - 'hospitalizations': '5', - 'hospitalization_rate': '0.19', - 'deaths INTEGER': '2', - 'death_rate':'0.07' - } -] - - @app.route('/', methods=['GET']) def home(): - return "

Public API offered by the team at Seattle Matrix.

This \ + # TODO: Look into putting all this into a template or something. + return """

Public API offered by the team at Seattle Matrix

This \ site is a collection of api's from various public sources collected \ - by the team at Seattle Matrix

" + by the team at SeattleMatrix.org.

Current Data: \ +
King County, WA Covid-19 By ZipCode - Current Dates: 20200810, 20200824, 20200831 + """ -# A route to return all of the available records for King County +# A route to return all of the available records for King County COVID-19 @app.route('/v1/kingcounty/covid-19/zipcode/all', methods=['GET']) def api_zipcode_all(): - conn = psycopg2.connect(f"host={dbhost} dbname={database} user={username} password={password} port={port}") + """ This route returns all the data in the king_county_covid19_zip + table. This data is currently manually updated. + """ + conn = psycopg2.connect(f"host={dbhost} dbname={database} user={username} password={password} port={port} sslmode='require'") cur = conn.cursor() cur.execute("select row_to_json(king_county_covid19_zip) from king_county_covid19_zip;") rows = cur.fetchall() - # returning rv throws an error and jsonify returns double nested + # FIXME: returning row throws an error and jsonify returns double nested # json result need to figure this out. return jsonify(rows) -@app.route('/v1/kingcounty/covid-19/zipcode', methods=['GET']) -def api_zipcode(): - # Check to see if 'zip' was provided in URL - # If no zip display message stating this - if 'zip' in request.args: - provided_zipcode = request.args['zip'] - else: - return "Error: No zipcode [zip] field provided. Please specify an zipcode." - - # Create an empty list for our results - results = [] - - # Loop through the data and match results that fit the requested Zipcode. - # zipcodes are unique, but other fields might return many results - for zipcode in zipcodes: - if zipcode['zipcode'] == provided_zipcode: - results.append(zipcode) - - # If no zipcodes matched and results are empty show message rather than - # empty page, maybe this isnt good to do in an API that is going to be used - # for human and computer consumption, need to check into that. - if not results: - results = "The Zipcode you entered was not in the API, please try again" - - # Use the jsonify function from Flask to convert our list of - # Python dictionaries to the JSON format. - return jsonify(results) +if __name__ == "__main__": + app.run(host='0.0.0.0') -app.run() +# TODO: Create route for filtering on zipcode and date +# TODO: Put all above into function for covid-19 data +# TODO: Look into making a Class for db connection \ No newline at end of file diff --git a/create_covid-19_zip.py b/database/create_covid-19_zip.py similarity index 100% rename from create_covid-19_zip.py rename to database/create_covid-19_zip.py diff --git a/load_db.py b/database/load_db.py similarity index 100% rename from load_db.py rename to database/load_db.py diff --git a/source_data/covid-19.xlsx b/database/source_data/covid-19.xlsx similarity index 100% rename from source_data/covid-19.xlsx rename to database/source_data/covid-19.xlsx diff --git a/source_data/covid-19_20200810_zip.csv b/database/source_data/covid-19_20200810_zip.csv similarity index 100% rename from source_data/covid-19_20200810_zip.csv rename to database/source_data/covid-19_20200810_zip.csv diff --git a/source_data/covid-19_20200824_zip.csv b/database/source_data/covid-19_20200824_zip.csv similarity index 100% rename from source_data/covid-19_20200824_zip.csv rename to database/source_data/covid-19_20200824_zip.csv diff --git a/source_data/covid-19_20200831_zip.csv b/database/source_data/covid-19_20200831_zip.csv similarity index 100% rename from source_data/covid-19_20200831_zip.csv rename to database/source_data/covid-19_20200831_zip.csv diff --git a/source_data/covid-data-extract-geography-august-10.xlsx b/database/source_data/covid-data-extract-geography-august-10.xlsx similarity index 100% rename from source_data/covid-data-extract-geography-august-10.xlsx rename to database/source_data/covid-data-extract-geography-august-10.xlsx diff --git a/source_data/covid-data-extract-geography-august-24.xlsx b/database/source_data/covid-data-extract-geography-august-24.xlsx similarity index 100% rename from source_data/covid-data-extract-geography-august-24.xlsx rename to database/source_data/covid-data-extract-geography-august-24.xlsx diff --git a/source_data/covid-data-extract-geography-august-31.xlsx b/database/source_data/covid-data-extract-geography-august-31.xlsx similarity index 100% rename from source_data/covid-data-extract-geography-august-31.xlsx rename to database/source_data/covid-data-extract-geography-august-31.xlsx