diff --git a/.github/workflows/run-dbt-prod.yml b/.github/workflows/run-dbt-prod.yml index 8fc42d164..40e5bb780 100644 --- a/.github/workflows/run-dbt-prod.yml +++ b/.github/workflows/run-dbt-prod.yml @@ -2,6 +2,7 @@ name: run_dbt_prod on: + workflow_dispatch: push: branches: [ "main" ] pull_request: diff --git a/.vscode/settings.json b/.vscode/settings.json index 236981053..a82add75e 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,8 +17,7 @@ "files.autoSave": "afterDelay", "screencastMode.onlyKeyboardShortcuts": true, "terminal.integrated.fontSize": 18, - "workbench.activityBar.visible": true, "workbench.colorTheme": "Visual Studio Dark", "workbench.fontAliasing": "antialiased", "workbench.statusBar.visible": true -} +} \ No newline at end of file diff --git a/data/nyc_parking_violations.db b/data/nyc_parking_violations.db index 19e587528..96e2269be 100644 Binary files a/data/nyc_parking_violations.db and b/data/nyc_parking_violations.db differ diff --git a/data/prod_nyc_parking_violations.db b/data/prod_nyc_parking_violations.db index 9a2bd4ad7..3cb368d6c 100644 Binary files a/data/prod_nyc_parking_violations.db and b/data/prod_nyc_parking_violations.db differ diff --git a/nyc_parking_violations/models/example/first_model.sql b/nyc_parking_violations/models/example/first_model.sql index 942e472df..1c044d16c 100644 --- a/nyc_parking_violations/models/example/first_model.sql +++ b/nyc_parking_violations/models/example/first_model.sql @@ -1 +1 @@ -SELECT * FROM parking_violation_codes \ No newline at end of file +SELECT * FROM parking_violation_codes; \ No newline at end of file diff --git a/nyc_parking_violations/models/example/ref_model.sql b/nyc_parking_violations/models/example/ref_model.sql index 9bb4f51c6..81da9dcee 100644 --- a/nyc_parking_violations/models/example/ref_model.sql +++ b/nyc_parking_violations/models/example/ref_model.sql @@ -1,4 +1,4 @@ SELECT - COUNT(*) + COUNT(*) FROM - {{ref('first_model')}} + {{ref('first_model')}} \ No newline at end of file diff --git a/nyc_parking_violations/profiles.yml b/nyc_parking_violations/profiles.yml index cabe18c54..b4a77bc5c 100644 --- a/nyc_parking_violations/profiles.yml +++ b/nyc_parking_violations/profiles.yml @@ -9,4 +9,10 @@ nyc_parking_violations: # start in the root directory and not in the # nyc_parking_violations directory path: './data/prod_nyc_parking_violations.db' - target: dev \ No newline at end of file + target: dev + +default: + outputs: + dev: + type: duckdb + target: dev diff --git a/run_sql_queries_here.ipynb b/run_sql_queries_here.ipynb index f7c54e6c9..4cf5888ae 100644 --- a/run_sql_queries_here.ipynb +++ b/run_sql_queries_here.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ @@ -12,22 +12,188 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
violation_codetotal_revenue_usd
0410.0
\n", + "
" + ], + "text/plain": [ + " violation_code total_revenue_usd\n", + "0 41 0.0" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sql_query = '''\n", + "show tables;\n", + "select * from \"nyc_parking_violations\".\"main_dbt_test__audit\".\"violation_codes_revenue\"\n", + "\n", + "'''\n", + "\n", + "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", + " display(con.sql(sql_query).df())" + ] + }, + { + "cell_type": "code", + "execution_count": 22, "metadata": {}, "outputs": [], + "source": [ + "sql_query_import_1 = '''\n", + "CREATE OR REPLACE TABLE parking_violation_codes AS\n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/dof_parking_violation_codes.csv',\n", + " normalize_names=True\n", + " )\n", + "'''\n", + "\n", + "sql_query_import_2 = '''\n", + "CREATE OR REPLACE TABLE parking_violations_2023 AS\n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/parking_violations_issued_fiscal_year_2023_sample.csv',\n", + " normalize_names=True\n", + " )\n", + "'''\n", + "\n", + "with duckdb.connect('data/prod_nyc_parking_violations.db') as con:\n", + " con.sql(sql_query_import_1)\n", + " con.sql(sql_query_import_2)" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
name
0bronze_parking_violation_codes
1bronze_parking_violations
2gold_ticket_metrics
3gold_vehicles_metrics
4parking_violation_codes
5parking_violations_2023
6silver_violation_tickets
7silver_violation_vehicles
\n", + "
" + ], + "text/plain": [ + " name\n", + "0 bronze_parking_violation_codes\n", + "1 bronze_parking_violations\n", + "2 gold_ticket_metrics\n", + "3 gold_vehicles_metrics\n", + "4 parking_violation_codes\n", + "5 parking_violations_2023\n", + "6 silver_violation_tickets\n", + "7 silver_violation_vehicles" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "sql_query = '''\n", "show tables\n", "'''\n", "\n", - "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", + "with duckdb.connect('data/prod_nyc_parking_violations.db') as con:\n", " display(con.sql(sql_query).df())" ] } ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": ".venv", "language": "python", "name": "python3" }, @@ -43,12 +209,7 @@ "pygments_lexer": "ipython3", "version": "3.10.12" }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" - } - } + "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2