diff --git a/.vscode/settings.json b/.vscode/settings.json index 236981053..ca20f9585 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,7 +17,6 @@ "files.autoSave": "afterDelay", "screencastMode.onlyKeyboardShortcuts": true, "terminal.integrated.fontSize": 18, - "workbench.activityBar.visible": true, "workbench.colorTheme": "Visual Studio Dark", "workbench.fontAliasing": "antialiased", "workbench.statusBar.visible": true diff --git a/data/nyc_parking_violations.db b/data/nyc_parking_violations.db index 19e587528..47376e57f 100644 Binary files a/data/nyc_parking_violations.db and b/data/nyc_parking_violations.db differ diff --git a/data/nyc_parking_violations2.db b/data/nyc_parking_violations2.db new file mode 100644 index 000000000..4965164bb Binary files /dev/null and b/data/nyc_parking_violations2.db differ diff --git a/data/prod_nyc_parking_violations.db b/data/prod_nyc_parking_violations.db index 9a2bd4ad7..e9a88dd87 100644 Binary files a/data/prod_nyc_parking_violations.db and b/data/prod_nyc_parking_violations.db differ diff --git a/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql b/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql deleted file mode 100644 index 217a5f263..000000000 --- a/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql +++ /dev/null @@ -1,7 +0,0 @@ -SELECT - code AS violation_code, - definition, - manhattan_96th_st_below, - all_other_areas -FROM - parking_violation_codes diff --git a/nyc_parking_violations/models/bronze/bronze_parking_violations.sql b/nyc_parking_violations/models/bronze/bronze_parking_violations.sql deleted file mode 100644 index 46525da2a..000000000 --- a/nyc_parking_violations/models/bronze/bronze_parking_violations.sql +++ /dev/null @@ -1,23 +0,0 @@ -SELECT - summons_number, - registration_state, - plate_type, - issue_date, - violation_code, - vehicle_body_type, - vehicle_make, - issuing_agency, - vehicle_expiration_date, - violation_location, - violation_precinct, - issuer_precinct, - issuer_code, - issuer_command, - issuer_squad, - violation_time, - violation_county, - violation_legal_code, - vehicle_color, - vehicle_year, -FROM - parking_violations_2023 \ No newline at end of file diff --git a/nyc_parking_violations/models/example/ref_model.sql b/nyc_parking_violations/models/example/ref_model.sql deleted file mode 100644 index 9bb4f51c6..000000000 --- a/nyc_parking_violations/models/example/ref_model.sql +++ /dev/null @@ -1,4 +0,0 @@ -SELECT - COUNT(*) -FROM - {{ref('first_model')}} diff --git a/nyc_parking_violations/.gitignore b/nyc_parking_violations2/.gitignore similarity index 73% rename from nyc_parking_violations/.gitignore rename to nyc_parking_violations2/.gitignore index 9a7943d2d..49f147cb9 100644 --- a/nyc_parking_violations/.gitignore +++ b/nyc_parking_violations2/.gitignore @@ -1,4 +1,4 @@ -.user.yml + target/ dbt_packages/ logs/ diff --git a/nyc_parking_violations2/.user.yml b/nyc_parking_violations2/.user.yml new file mode 100644 index 000000000..f127973aa --- /dev/null +++ b/nyc_parking_violations2/.user.yml @@ -0,0 +1 @@ +id: 23096e09-b919-420a-854b-199a5ab84b4f diff --git a/nyc_parking_violations/README.md b/nyc_parking_violations2/README.md similarity index 100% rename from nyc_parking_violations/README.md rename to nyc_parking_violations2/README.md diff --git a/nyc_parking_violations/analyses/.gitkeep b/nyc_parking_violations2/analyses/.gitkeep similarity index 100% rename from nyc_parking_violations/analyses/.gitkeep rename to nyc_parking_violations2/analyses/.gitkeep diff --git a/nyc_parking_violations/dbt_project.yml b/nyc_parking_violations2/dbt_project.yml similarity index 93% rename from nyc_parking_violations/dbt_project.yml rename to nyc_parking_violations2/dbt_project.yml index ec48605d3..79b6cc6ae 100644 --- a/nyc_parking_violations/dbt_project.yml +++ b/nyc_parking_violations2/dbt_project.yml @@ -2,12 +2,12 @@ # Name your project! Project names should contain only lowercase characters # and underscores. A good package name should reflect your organization's # name or the intended use of these models -name: 'nyc_parking_violations' +name: 'nyc_parking_violations2' version: '1.0.0' config-version: 2 # This setting configures which "profile" dbt uses for this project. -profile: 'nyc_parking_violations' +profile: 'nyc_parking_violations2' # These configurations specify where dbt should look for different types of files. # The `model-paths` config, for example, states that models in this project can be @@ -31,7 +31,7 @@ clean-targets: # directories to be removed by `dbt clean` # directory as views. These settings can be overridden in the individual model # files using the `{{ config(...) }}` macro. models: - nyc_parking_violations: + nyc_parking_violations2: # Config indicated by + and applies to all files under models/example/ example: +materialized: ephemeral @@ -49,4 +49,4 @@ models: gold: +materialized: table tests: - +store_failures: true + +store_failures: true \ No newline at end of file diff --git a/nyc_parking_violations/macros/.gitkeep b/nyc_parking_violations2/macros/.gitkeep similarity index 100% rename from nyc_parking_violations/macros/.gitkeep rename to nyc_parking_violations2/macros/.gitkeep diff --git a/nyc_parking_violations2/models/bronze/bronze_parking_violation_codes.sql b/nyc_parking_violations2/models/bronze/bronze_parking_violation_codes.sql new file mode 100644 index 000000000..9e054fc8d --- /dev/null +++ b/nyc_parking_violations2/models/bronze/bronze_parking_violation_codes.sql @@ -0,0 +1,7 @@ +SELECT + code AS violation_code, + definition, + manhattan_96th_st_below, + all_other_areas +FROM + parking_violation_codes \ No newline at end of file diff --git a/nyc_parking_violations2/models/bronze/bronze_parking_violations.sql b/nyc_parking_violations2/models/bronze/bronze_parking_violations.sql new file mode 100644 index 000000000..12ec574d5 --- /dev/null +++ b/nyc_parking_violations2/models/bronze/bronze_parking_violations.sql @@ -0,0 +1,23 @@ +SELECT + summons_number, + registration_state, + plate_type, + issue_date, + violation_code, + vehicle_body_type, + vehicle_make, + issuing_agency, + vehicle_expiration_date, + violation_location, + violation_precinct, + issuer_precinct, + issuer_code, + issuer_command, + issuer_squad, + violation_time, + violation_county, + violation_legal_code, + vehicle_color, + vehicle_year, +FROM + parking_violations_2023 \ No newline at end of file diff --git a/nyc_parking_violations/models/docs/docs_blocks.md b/nyc_parking_violations2/models/docs/docs_block.md similarity index 99% rename from nyc_parking_violations/models/docs/docs_blocks.md rename to nyc_parking_violations2/models/docs/docs_block.md index cd410941a..aa403dff5 100644 --- a/nyc_parking_violations/models/docs/docs_blocks.md +++ b/nyc_parking_violations2/models/docs/docs_block.md @@ -104,4 +104,4 @@ The total number of tickets issued for a specific violation code. {% docs total_revenue_usd %} The total revenue accumulated from tickets, based on the violation code. This sum is represented in USD. -{% enddocs %} +{% enddocs %} \ No newline at end of file diff --git a/nyc_parking_violations/models/docs/schema.yml b/nyc_parking_violations2/models/docs/schema.yml similarity index 99% rename from nyc_parking_violations/models/docs/schema.yml rename to nyc_parking_violations2/models/docs/schema.yml index c1dfcb9fc..0524962a6 100644 --- a/nyc_parking_violations/models/docs/schema.yml +++ b/nyc_parking_violations2/models/docs/schema.yml @@ -185,4 +185,4 @@ models: - name: registration_state description: '{{ doc("registration_state") }}' - name: ticket_count - description: '{{ doc("ticket_count") }}' + description: '{{ doc("ticket_count") }}' \ No newline at end of file diff --git a/nyc_parking_violations/models/example/first_model.sql b/nyc_parking_violations2/models/example/first_model.sql similarity index 100% rename from nyc_parking_violations/models/example/first_model.sql rename to nyc_parking_violations2/models/example/first_model.sql diff --git a/nyc_parking_violations2/models/example/ref_model.sql b/nyc_parking_violations2/models/example/ref_model.sql new file mode 100644 index 000000000..5624b1b7a --- /dev/null +++ b/nyc_parking_violations2/models/example/ref_model.sql @@ -0,0 +1,4 @@ +SELECT + COUNT(*) +FROM + {{ref('first_model')}} \ No newline at end of file diff --git a/nyc_parking_violations/models/gold/gold_ticket_metrics.sql b/nyc_parking_violations2/models/gold/gold_ticket_metrics.sql similarity index 100% rename from nyc_parking_violations/models/gold/gold_ticket_metrics.sql rename to nyc_parking_violations2/models/gold/gold_ticket_metrics.sql diff --git a/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql b/nyc_parking_violations2/models/gold/gold_vehicle_metrics.sql similarity index 100% rename from nyc_parking_violations/models/gold/gold_vehicles_metrics.sql rename to nyc_parking_violations2/models/gold/gold_vehicle_metrics.sql diff --git a/nyc_parking_violations/models/silver/silver_parking_violation_codes.sql b/nyc_parking_violations2/models/silver/silver_parking_violation_codes.sql similarity index 100% rename from nyc_parking_violations/models/silver/silver_parking_violation_codes.sql rename to nyc_parking_violations2/models/silver/silver_parking_violation_codes.sql diff --git a/nyc_parking_violations/models/silver/silver_parking_violations.sql b/nyc_parking_violations2/models/silver/silver_parking_violations.sql similarity index 100% rename from nyc_parking_violations/models/silver/silver_parking_violations.sql rename to nyc_parking_violations2/models/silver/silver_parking_violations.sql diff --git a/nyc_parking_violations/models/silver/silver_violation_tickets.sql b/nyc_parking_violations2/models/silver/silver_violation_tickets.sql similarity index 100% rename from nyc_parking_violations/models/silver/silver_violation_tickets.sql rename to nyc_parking_violations2/models/silver/silver_violation_tickets.sql diff --git a/nyc_parking_violations/models/silver/silver_violation_vehicles.sql b/nyc_parking_violations2/models/silver/silver_violation_vehicles.sql similarity index 100% rename from nyc_parking_violations/models/silver/silver_violation_vehicles.sql rename to nyc_parking_violations2/models/silver/silver_violation_vehicles.sql diff --git a/nyc_parking_violations/profiles.yml b/nyc_parking_violations2/profiles.yml similarity index 77% rename from nyc_parking_violations/profiles.yml rename to nyc_parking_violations2/profiles.yml index cabe18c54..57c76bcf9 100644 --- a/nyc_parking_violations/profiles.yml +++ b/nyc_parking_violations2/profiles.yml @@ -1,4 +1,4 @@ -nyc_parking_violations: +nyc_parking_violations2: outputs: dev: type: duckdb @@ -8,5 +8,5 @@ nyc_parking_violations: # note that path is slightly different as GitHub actions # start in the root directory and not in the # nyc_parking_violations directory - path: './data/prod_nyc_parking_violations.db' + path: '../data/prod_nyc_parking_violations.db' target: dev \ No newline at end of file diff --git a/nyc_parking_violations/seeds/.gitkeep b/nyc_parking_violations2/seeds/.gitkeep similarity index 100% rename from nyc_parking_violations/seeds/.gitkeep rename to nyc_parking_violations2/seeds/.gitkeep diff --git a/nyc_parking_violations/snapshots/.gitkeep b/nyc_parking_violations2/snapshots/.gitkeep similarity index 100% rename from nyc_parking_violations/snapshots/.gitkeep rename to nyc_parking_violations2/snapshots/.gitkeep diff --git a/nyc_parking_violations/tests/.gitkeep b/nyc_parking_violations2/tests/.gitkeep similarity index 100% rename from nyc_parking_violations/tests/.gitkeep rename to nyc_parking_violations2/tests/.gitkeep diff --git a/nyc_parking_violations/tests/generic/generic_not_null.sql b/nyc_parking_violations2/tests/generic/generic_not_null.sql similarity index 100% rename from nyc_parking_violations/tests/generic/generic_not_null.sql rename to nyc_parking_violations2/tests/generic/generic_not_null.sql diff --git a/nyc_parking_violations/tests/violation_codes_revenue.sql b/nyc_parking_violations2/tests/violation_codes_revenue.sql similarity index 85% rename from nyc_parking_violations/tests/violation_codes_revenue.sql rename to nyc_parking_violations2/tests/violation_codes_revenue.sql index 091ddfdd4..9426a0ba9 100644 --- a/nyc_parking_violations/tests/violation_codes_revenue.sql +++ b/nyc_parking_violations2/tests/violation_codes_revenue.sql @@ -8,4 +8,4 @@ FROM GROUP BY violation_code HAVING - NOT(total_revenue_usd >= 1) + NOT(total_revenue_usd >= 1) \ No newline at end of file diff --git a/run_sql_queries_here.ipynb b/run_sql_queries_here.ipynb index f7c54e6c9..48b53cfcd 100644 --- a/run_sql_queries_here.ipynb +++ b/run_sql_queries_here.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ @@ -10,19 +10,195 @@ "import pandas as pd" ] }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
violation_codetotal_revenue_usd
0410.0
\n", + "
" + ], + "text/plain": [ + " violation_code total_revenue_usd\n", + "0 41 0.0" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sql_query = '''\n", + "select * from \"nyc_parking_violations\".\"main_dbt_test__audit\".\"violation_codes_revenue\"\n", + "'''\n", + "\n", + "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", + " display(con.sql(sql_query).df())" + ] + }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], + "source": [ + "sql_query_import_1 = '''\n", + "CREATE OR REPLACE TABLE parking_violations_code AS \n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/dof_parking_violation_codes.csv',\n", + " normalize_names=True\n", + " )\n", + "'''\n", + "\n", + "sql_query_import_2 = '''\n", + "CREATE OR REPLACE TABLE parking_violations_2023 AS \n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/parking_violations_issued_fiscal_year_2023_sample.csv',\n", + " normalize_names=True\n", + " )\n", + "'''\n", + "\n", + "with duckdb.connect('data/prod_nyc_parking_violations2.db') as con:\n", + " con.sql(sql_query_import_1)\n", + " con.sql(sql_query_import_2)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
name
0bronze_parking_violation_codes
1bronze_parking_violations
2gold_ticket_metrics
3gold_vehicles_metrics
4parking_violation_codes
5parking_violations_2023
6parking_violations_code
7silver_violation_tickets
8silver_violation_vehicles
\n", + "
" + ], + "text/plain": [ + " name\n", + "0 bronze_parking_violation_codes\n", + "1 bronze_parking_violations\n", + "2 gold_ticket_metrics\n", + "3 gold_vehicles_metrics\n", + "4 parking_violation_codes\n", + "5 parking_violations_2023\n", + "6 parking_violations_code\n", + "7 silver_violation_tickets\n", + "8 silver_violation_vehicles" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "sql_query = '''\n", "show tables\n", "'''\n", "\n", - "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", + "with duckdb.connect('data/prod_nyc_parking_violations.db') as con:\n", " display(con.sql(sql_query).df())" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": {