diff --git a/.vscode/settings.json b/.vscode/settings.json index 236981053..ca20f9585 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,7 +17,6 @@ "files.autoSave": "afterDelay", "screencastMode.onlyKeyboardShortcuts": true, "terminal.integrated.fontSize": 18, - "workbench.activityBar.visible": true, "workbench.colorTheme": "Visual Studio Dark", "workbench.fontAliasing": "antialiased", "workbench.statusBar.visible": true diff --git a/data/nyc_parking_violations.db b/data/nyc_parking_violations.db index 19e587528..7a63444e1 100644 Binary files a/data/nyc_parking_violations.db and b/data/nyc_parking_violations.db differ diff --git a/data/prod_nyc_parking_violations.db b/data/prod_nyc_parking_violations.db index 9a2bd4ad7..cc541647f 100644 Binary files a/data/prod_nyc_parking_violations.db and b/data/prod_nyc_parking_violations.db differ diff --git a/nyc_parking_violations/dbt_project.yml b/nyc_parking_violations/dbt_project.yml index ec48605d3..e2ce205e5 100644 --- a/nyc_parking_violations/dbt_project.yml +++ b/nyc_parking_violations/dbt_project.yml @@ -49,4 +49,4 @@ models: gold: +materialized: table tests: - +store_failures: true + +store_failures: true # help store failed test diff --git a/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql b/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql index 217a5f263..e2f7cdc6a 100644 --- a/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql +++ b/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql @@ -1,7 +1,7 @@ -SELECT - code AS violation_code, - definition, - manhattan_96th_st_below, - all_other_areas -FROM - parking_violation_codes +SELECT + code AS violation_code, + definition, + manhattan_96th_st_below, + all_other_areas +From + parking_violation_codes \ No newline at end of file diff --git a/nyc_parking_violations/models/docs/docs_blocks.md b/nyc_parking_violations/models/docs/docs_block.md similarity index 99% rename from nyc_parking_violations/models/docs/docs_blocks.md rename to nyc_parking_violations/models/docs/docs_block.md index cd410941a..aa403dff5 100644 --- a/nyc_parking_violations/models/docs/docs_blocks.md +++ b/nyc_parking_violations/models/docs/docs_block.md @@ -104,4 +104,4 @@ The total number of tickets issued for a specific violation code. {% docs total_revenue_usd %} The total revenue accumulated from tickets, based on the violation code. This sum is represented in USD. -{% enddocs %} +{% enddocs %} \ No newline at end of file diff --git a/nyc_parking_violations/models/docs/schema.yml b/nyc_parking_violations/models/docs/schema.yml index c1dfcb9fc..0524962a6 100644 --- a/nyc_parking_violations/models/docs/schema.yml +++ b/nyc_parking_violations/models/docs/schema.yml @@ -185,4 +185,4 @@ models: - name: registration_state description: '{{ doc("registration_state") }}' - name: ticket_count - description: '{{ doc("ticket_count") }}' + description: '{{ doc("ticket_count") }}' \ No newline at end of file diff --git a/nyc_parking_violations/models/example/ref_model.sql b/nyc_parking_violations/models/example/ref_model.sql deleted file mode 100644 index 9bb4f51c6..000000000 --- a/nyc_parking_violations/models/example/ref_model.sql +++ /dev/null @@ -1,4 +0,0 @@ -SELECT - COUNT(*) -FROM - {{ref('first_model')}} diff --git a/nyc_parking_violations/models/example/first_model.sql b/nyc_parking_violations/models/examples/first_model.sql similarity index 100% rename from nyc_parking_violations/models/example/first_model.sql rename to nyc_parking_violations/models/examples/first_model.sql diff --git a/nyc_parking_violations/models/examples/ref_model.sql b/nyc_parking_violations/models/examples/ref_model.sql new file mode 100644 index 000000000..57d6efc39 --- /dev/null +++ b/nyc_parking_violations/models/examples/ref_model.sql @@ -0,0 +1,4 @@ +SELECT + COUNT(*) +FROM + {{ref('first_model')}} \ No newline at end of file diff --git a/nyc_parking_violations/profiles.yml b/nyc_parking_violations/profiles.yml index cabe18c54..b63e2b766 100644 --- a/nyc_parking_violations/profiles.yml +++ b/nyc_parking_violations/profiles.yml @@ -1,12 +1,12 @@ -nyc_parking_violations: +nyc_parking_violations: #always remember to name correctly, change from default outputs: dev: type: duckdb - path: '../data/nyc_parking_violations.db' + path: '../data/nyc_parking_violations.db' #always add the path prod: type: duckdb - # note that path is slightly different as GitHub actions + # note that path is slightldbty different as GitHub actions # start in the root directory and not in the # nyc_parking_violations directory - path: './data/prod_nyc_parking_violations.db' + path: '../data/prod_nyc_parking_violations.db' target: dev \ No newline at end of file diff --git a/nyc_parking_violations/seeds/.gitkeep b/nyc_parking_violations/seeds/.gitkeep deleted file mode 100644 index e69de29bb..000000000 diff --git a/run_sql_queries_here.ipynb b/run_sql_queries_here.ipynb index f7c54e6c9..12dfa1f03 100644 --- a/run_sql_queries_here.ipynb +++ b/run_sql_queries_here.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -12,17 +12,527 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
name
0bronze_parking_violation_codes
1bronze_parking_violations
2first_model
3gold_ticket_metrics
4gold_vehicles_metrics
5parking_violation_codes
6parking_violations_2023
7ref_model
8silver_parking_violation_codes
9silver_parking_violations
10silver_violation_tickets
11silver_violation_vehicles
\n", + "
" + ], + "text/plain": [ + " name\n", + "0 bronze_parking_violation_codes\n", + "1 bronze_parking_violations\n", + "2 first_model\n", + "3 gold_ticket_metrics\n", + "4 gold_vehicles_metrics\n", + "5 parking_violation_codes\n", + "6 parking_violations_2023\n", + "7 ref_model\n", + "8 silver_parking_violation_codes\n", + "9 silver_parking_violations\n", + "10 silver_violation_tickets\n", + "11 silver_violation_vehicles" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "sql_query = '''\n", "show tables\n", + "\n", + "\n", + "'''\n", + "\n", + "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", + " display(con.sql(sql_query).df())" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "sql_query_import_1 = '''\n", + "CREATE OR REPLACE TABLE parking_violation_codes AS\n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/dof_parking_violation_codes.csv',\n", + " normalize_names=True\n", + ") \n", + "'''\n", + "\n", + "sql_query_import_2 = '''\n", + "CREATE OR REPLACE TABLE parking_violations_2023 AS \n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/parking_violations_issued_fiscal_year_2023_sample.csv',\n", + " normalize_names=True\n", + ")\n", + "'''\n", + "#run individual query with this\n", + "with duckdb.connect('data/prod_nyc_parking_violations.db') as con:\n", + " con.sql(sql_query_import_1)\n", + " con.sql(sql_query_import_2)\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
summons_numberregistration_stateplate_typeissue_dateviolation_codevehicle_body_typevehicle_makeissuing_agencyvehicle_expiration_dateviolation_location...issuer_precinctissuer_codeissuer_commandissuer_squadviolation_timeviolation_countyviolation_legal_codevehicle_colorvehicle_yearis_manhattan_96th_st_below
09010912681CAPAS2022-10-1117SUBNFORDT2022078850.0...50365910T201D0825ABXNaNBLACK0False
14858762841NYPAS2023-08-21364DSDHONDAV0NaN...00NoneNone1036ABKTrueGY2003False
24854645684FLPAS2023-07-2636UTBMWV0NaN...00NoneNone0126PQNTrueWHI2022False
39044582707NYPAS2023-04-1021SUBNSUBART2023121779.0...79370326T301H0924AKNaNGY2017False
49041503330NYPAS2023-03-21214DSDCHEVRT2025032026.0...26373097T103C1155ANYNaNBK2018False
58964996320NYCOM2023-06-1221VANCHEVRT8888888846.0...46374612T201E0923ABXNaNTAN0False
69019285804NYSRF2022-11-2040SUBNVOLKST2024022020.0...20357701T103P0230PNYNaNGY2011False
79000744076CTCOM2022-07-1969SUBNLINCOT888888886.0...6373104T102I0349PNYNaNBLACK0False
84839205334NYPAS2023-04-3036SUBNTESLAV0NaN...00NoneNone1154ABKTrueRD2021False
99006263692NYPAS2022-09-2016SUBNCHEVRT2024032384.0...84368636T301P0308PKNaNBK2017False
\n", + "

10 rows × 21 columns

\n", + "
" + ], + "text/plain": [ + " summons_number registration_state plate_type issue_date violation_code \\\n", + "0 9010912681 CA PAS 2022-10-11 17 \n", + "1 4858762841 NY PAS 2023-08-21 36 \n", + "2 4854645684 FL PAS 2023-07-26 36 \n", + "3 9044582707 NY PAS 2023-04-10 21 \n", + "4 9041503330 NY PAS 2023-03-21 21 \n", + "5 8964996320 NY COM 2023-06-12 21 \n", + "6 9019285804 NY SRF 2022-11-20 40 \n", + "7 9000744076 CT COM 2022-07-19 69 \n", + "8 4839205334 NY PAS 2023-04-30 36 \n", + "9 9006263692 NY PAS 2022-09-20 16 \n", + "\n", + " vehicle_body_type vehicle_make issuing_agency vehicle_expiration_date \\\n", + "0 SUBN FORD T 20220788 \n", + "1 4DSD HONDA V 0 \n", + "2 UT BMW V 0 \n", + "3 SUBN SUBAR T 20231217 \n", + "4 4DSD CHEVR T 20250320 \n", + "5 VAN CHEVR T 88888888 \n", + "6 SUBN VOLKS T 20240220 \n", + "7 SUBN LINCO T 88888888 \n", + "8 SUBN TESLA V 0 \n", + "9 SUBN CHEVR T 20240323 \n", + "\n", + " violation_location ... issuer_precinct issuer_code issuer_command \\\n", + "0 50.0 ... 50 365910 T201 \n", + "1 NaN ... 0 0 None \n", + "2 NaN ... 0 0 None \n", + "3 79.0 ... 79 370326 T301 \n", + "4 26.0 ... 26 373097 T103 \n", + "5 46.0 ... 46 374612 T201 \n", + "6 20.0 ... 20 357701 T103 \n", + "7 6.0 ... 6 373104 T102 \n", + "8 NaN ... 0 0 None \n", + "9 84.0 ... 84 368636 T301 \n", + "\n", + " issuer_squad violation_time violation_county violation_legal_code \\\n", + "0 D 0825A BX NaN \n", + "1 None 1036A BK True \n", + "2 None 0126P QN True \n", + "3 H 0924A K NaN \n", + "4 C 1155A NY NaN \n", + "5 E 0923A BX NaN \n", + "6 P 0230P NY NaN \n", + "7 I 0349P NY NaN \n", + "8 None 1154A BK True \n", + "9 P 0308P K NaN \n", + "\n", + " vehicle_color vehicle_year is_manhattan_96th_st_below \n", + "0 BLACK 0 False \n", + "1 GY 2003 False \n", + "2 WHI 2022 False \n", + "3 GY 2017 False \n", + "4 BK 2018 False \n", + "5 TAN 0 False \n", + "6 GY 2011 False \n", + "7 BLACK 0 False \n", + "8 RD 2021 False \n", + "9 BK 2017 False \n", + "\n", + "[10 rows x 21 columns]" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sql_query = '''\n", + "SELECT * FROM silver_parking_violations LIMIT 10\n", + "\n", + "\n", "'''\n", "\n", "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", " display(con.sql(sql_query).df())" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": {