diff --git a/.github/workflows/run-dbt-prod.yml b/.github/workflows/run-dbt-prod.yml index 8fc42d164..e69de29bb 100644 --- a/.github/workflows/run-dbt-prod.yml +++ b/.github/workflows/run-dbt-prod.yml @@ -1,38 +0,0 @@ -# .github/workflows/run-dbt-prod.yml -name: run_dbt_prod - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - # schedule: - # - cron: '0 8 * * *' - -env: - DBT_PROFILES_DIR: ./nyc_parking_violations - DBT_PROJECT_DIR: ./nyc_parking_violations - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Run dbt Prod - run: | - dbt debug - dbt compile --target prod - dbt run --target prod - - name: Test dbt Prod - run: | - dbt test --target prod \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 236981053..ca20f9585 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,7 +17,6 @@ "files.autoSave": "afterDelay", "screencastMode.onlyKeyboardShortcuts": true, "terminal.integrated.fontSize": 18, - "workbench.activityBar.visible": true, "workbench.colorTheme": "Visual Studio Dark", "workbench.fontAliasing": "antialiased", "workbench.statusBar.visible": true diff --git a/data/nyc_parking_violations.db b/data/nyc_parking_violations.db index 19e587528..1c60561cf 100644 Binary files a/data/nyc_parking_violations.db and b/data/nyc_parking_violations.db differ diff --git a/data/prod_nyc_parking_violations.db b/data/prod_nyc_parking_violations.db index 9a2bd4ad7..cabaebac7 100644 Binary files a/data/prod_nyc_parking_violations.db and b/data/prod_nyc_parking_violations.db differ diff --git a/nyc_parking_violations/models/docs/docs_block.md b/nyc_parking_violations/models/docs/docs_block.md new file mode 100644 index 000000000..4131970da --- /dev/null +++ b/nyc_parking_violations/models/docs/docs_block.md @@ -0,0 +1,53 @@ +{% docs violation_code %} Code representing the specific parking violation.{% enddocs %} + +{% docs definition %} Description of the violation for a respective code.{% enddocs %} + +{% docs manhattan_96th_st_below %} The fee in $USD for a violation or below Manhattan 96th street.{% enddocs %} + +{% docs all_other_areas %} The fee in $USD for a violation not on or below Manhattan 96th street. {% enddocs %} + +{% docs summons_number %} Unique identifier for each summons issued for a parking violation.{% enddocs %} + +{% docs registration_state %} The state where the vehicle is registered. {% enddocs %} + +{% docs plate_type %} The type of license plate. {% enddocs %} + +{% docs issue_date %} The date when the summons was issued. {% enddocs %} + +{% docs vehicle_body_type %} The body type of the vehicle involved in the violation. {% enddocs %} + +{% docs vehicle_make %} The make or brand of the vehicle. {% enddocs %} + +{% docs issuing_agency %} The agency that issued the summons. {% enddocs %} + +{% docs vehicle_expiration_date %} The date when the vehicle's registration expires. {% enddocs %} + +{% docs violation_location %} General location where the violation occurred. {% enddocs %} + +{% docs violation_precinct %} Precinct where the violation was identified. {% enddocs %} + +{% docs issuer_precinct %} Precinct of the officer or official who issued the summons. {% enddocs %} + +{% docs issuer_code %} Unique code identifying the issuer. {% enddocs %} + +{% docs issuer_command %} Command or unit of the issuer. {% enddocs %} + +{% docs issuer_squad %} Squad detail for the issuer. {% enddocs %} + +{% docs violation_time %} Time when the violation occurred. {% enddocs %} + +{% docs violation_county %} County where the violation took place. {% enddocs %} + +{% docs violation_legal_code %} Legal code associated with the violation. {% enddocs %} + +{% docs vehicle_color %} Color of the vehicle involved in the violation. {% enddocs %} + +{% docs vehicle_year %} Manufacturing year of the vehicle. {% enddocs %} + +{% docs fee_usd %} The fee charged for a parking violation, specified in USD. This fee varies depending on the location of the violation. {% enddocs %} + +{% docs is_manhattan_96th_st_below %} A boolean value indicating whether the violation occurred in Manhattan on or below 96th Street. {% enddocs %} + +{% docs ticket_count %} The total number of tickets issued for a specific violation code. {% enddocs %} + +{% docs total_revenue_usd %} The total revenue accumulated from tickets, based on the violation code. This sum is represented in USD. {% enddocs %} diff --git a/nyc_parking_violations/models/docs/schema.yml b/nyc_parking_violations/models/docs/schema.yml index c1dfcb9fc..e69de29bb 100644 --- a/nyc_parking_violations/models/docs/schema.yml +++ b/nyc_parking_violations/models/docs/schema.yml @@ -1,188 +0,0 @@ -models: - - name: bronze_parking_violation_codes - description: Raw data representing the violation codes and their fees. - columns: - - name: violation_code - description: '{{ doc("violation_code") }}' - - name: definition - description: '{{ doc("definition") }}' - - name: manhattan_96th_st_below - description: '{{ doc("manhattan_96th_st_below") }}' - - name: all_other_areas - description: '{{ doc("all_other_areas") }}' - - - name: bronze_parking_violations - description: Raw data related to parking violations in 2023, encompassing various details about each violation. - columns: - - name: summons_number - description: '{{ doc("summons_number") }}' - tests: - - unique - - not_null - - generic_not_null - - name: registration_state - description: '{{ doc("registration_state") }}' - - name: plate_type - description: '{{ doc("plate_type") }}' - - name: issue_date - description: '{{ doc("issue_date") }}' - - name: violation_code - description: '{{ doc("violation_code") }}' - - name: vehicle_body_type - description: '{{ doc("vehicle_body_type") }}' - - name: vehicle_make - description: '{{ doc("vehicle_make") }}' - - name: issuing_agency - description: '{{ doc("issuing_agency") }}' - - name: vehicle_expiration_date - description: '{{ doc("vehicle_expiration_date") }}' - - name: violation_location - description: '{{ doc("violation_location") }}' - - name: violation_precinct - description: '{{ doc("violation_precinct") }}' - - name: issuer_precinct - description: '{{ doc("issuer_precinct") }}' - - name: issuer_code - description: '{{ doc("issuer_code") }}' - - name: issuer_command - description: '{{ doc("issuer_command") }}' - - name: issuer_squad - description: '{{ doc("issuer_squad") }}' - - name: violation_time - description: '{{ doc("violation_time") }}' - - name: violation_county - description: '{{ doc("violation_county") }}' - - name: violation_legal_code - description: '{{ doc("violation_legal_code") }}' - - name: vehicle_color - description: '{{ doc("vehicle_color") }}' - - name: vehicle_year - description: '{{ doc("vehicle_year") }}' - - - name: silver_parking_violation_codes - description: "This model unifies violation codes, providing a comprehensive view of violations, indicating whether they occurred on/below 96th St in Manhattan or in other areas, along with the respective fees in USD." - columns: - - name: violation_code - description: '{{ doc("violation_code") }}' - - name: definition - description: '{{ doc("definition") }}' - - name: is_manhattan_96th_st_below - description: '{{ doc("is_manhattan_96th_st_below") }}' - - name: fee_usd - description: '{{ doc("fee_usd") }}' - - - name: silver_parking_violations - description: "Enhanced view of parking violations, enriched with details and specific indicators such as the flag for violations in Manhattan on or below 96th Street." - columns: - - name: summons_number - description: '{{ doc("summons_number") }}' - - name: registration_state - description: '{{ doc("registration_state") }}' - - name: plate_type - description: '{{ doc("plate_type") }}' - - name: issue_date - description: '{{ doc("issue_date") }}' - - name: violation_code - description: '{{ doc("violation_code") }}' - - name: vehicle_body_type - description: '{{ doc("vehicle_body_type") }}' - - name: vehicle_make - description: '{{ doc("vehicle_make") }}' - - name: issuing_agency - description: '{{ doc("issuing_agency") }}' - - name: vehicle_expiration_date - description: '{{ doc("vehicle_expiration_date") }}' - - name: violation_location - description: '{{ doc("violation_location") }}' - - name: violation_precinct - description: '{{ doc("violation_precinct") }}' - - name: issuer_precinct - description: '{{ doc("issuer_precinct") }}' - - name: issuer_code - description: '{{ doc("issuer_code") }}' - - name: issuer_command - description: '{{ doc("issuer_command") }}' - - name: issuer_squad - description: '{{ doc("issuer_squad") }}' - - name: violation_time - description: '{{ doc("violation_time") }}' - - name: violation_county - description: '{{ doc("violation_county") }}' - - name: violation_legal_code - description: '{{ doc("violation_legal_code") }}' - - name: vehicle_color - description: '{{ doc("vehicle_color") }}' - - name: vehicle_year - description: '{{ doc("vehicle_year") }}' - - name: is_manhattan_96th_st_below - description: '{{ doc("is_manhattan_96th_st_below") }}' - - - name: silver_violation_tickets - description: "Consolidated information on parking violations, enriched with associated fee details." - columns: - - name: summons_number - description: '{{ doc("summons_number") }}' - - name: issue_date - description: '{{ doc("issue_date") }}' - - name: violation_code - description: '{{ doc("violation_code") }}' - - name: is_manhattan_96th_st_below - description: '{{ doc("is_manhattan_96th_st_below") }}' - - name: issuing_agency - description: '{{ doc("issuing_agency") }}' - - name: violation_location - description: '{{ doc("violation_location") }}' - - name: violation_precinct - description: '{{ doc("violation_precinct") }}' - - name: issuer_precinct - description: '{{ doc("issuer_precinct") }}' - - name: issuer_code - description: '{{ doc("issuer_code") }}' - - name: issuer_command - description: '{{ doc("issuer_command") }}' - - name: issuer_squad - description: '{{ doc("issuer_squad") }}' - - name: violation_time - description: '{{ doc("violation_time") }}' - - name: violation_county - description: '{{ doc("violation_county") }}' - - name: violation_legal_code - description: '{{ doc("violation_legal_code") }}' - - - name: silver_violation_vehicles - description: "Details of the vehicles involved in parking violations." - columns: - - name: summons_number - description: '{{ doc("summons_number") }}' - - name: registration_state - description: '{{ doc("registration_state") }}' - - name: plate_type - description: '{{ doc("plate_type") }}' - - name: vehicle_body_type - description: '{{ doc("vehicle_body_type") }}' - - name: vehicle_make - description: '{{ doc("vehicle_make") }}' - - name: vehicle_expiration_date - description: '{{ doc("vehicle_expiration_date") }}' - - name: vehicle_color - description: '{{ doc("vehicle_color") }}' - - name: vehicle_year - description: '{{ doc("vehicle_year") }}' - - - name: gold_ticket_metrics - description: "Aggregated metrics representing the total tickets and revenue by violation code." - columns: - - name: violation_code - description: '{{ doc("violation_code") }}' - - name: ticket_count - description: '{{ doc("ticket_count") }}' - - name: total_revenue_usd - description: '{{ doc("total_revenue_usd") }}' - - - name: gold_vehicles_metrics - description: "Aggregated metrics detailing the number of tickets per vehicle, identified by the plate ID." - columns: - - name: registration_state - description: '{{ doc("registration_state") }}' - - name: ticket_count - description: '{{ doc("ticket_count") }}' diff --git a/nyc_parking_violations/models/gold/gold_ticket_metrics.sql b/nyc_parking_violations/models/gold/gold_ticket_metrics.sql index 861de4d61..b1a8917dc 100644 --- a/nyc_parking_violations/models/gold/gold_ticket_metrics.sql +++ b/nyc_parking_violations/models/gold/gold_ticket_metrics.sql @@ -1,7 +1,7 @@ SELECT - violation_code, - COUNT(summons_number) AS ticket_count, - SUM(fee_usd) AS total_revenue_usd + violation_code + ,COUNT(summons_number) AS ticket_count + ,SUM(fee_usd) AS total_revenue_usd FROM {{ref('silver_violation_tickets')}} GROUP BY diff --git a/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql b/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql index af796bfa9..ba45fba6b 100644 --- a/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql +++ b/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql @@ -1,6 +1,6 @@ SELECT - registration_state, - COUNT(summons_number) AS ticket_count, + registration_state + ,COUNT(summons_number) AS ticket_count FROM {{ref('silver_violation_vehicles')}} WHERE diff --git a/nyc_parking_violations/profiles.yml b/nyc_parking_violations/profiles.yml index cabe18c54..62d136ae5 100644 --- a/nyc_parking_violations/profiles.yml +++ b/nyc_parking_violations/profiles.yml @@ -1,12 +1,9 @@ nyc_parking_violations: outputs: - dev: - type: duckdb - path: '../data/nyc_parking_violations.db' - prod: - type: duckdb - # note that path is slightly different as GitHub actions - # start in the root directory and not in the - # nyc_parking_violations directory - path: './data/prod_nyc_parking_violations.db' - target: dev \ No newline at end of file + dev: + type: duckdb + path: "../data/nyc_parking_violations.db" + prod: + type: duckdb + path: "../data/prod_nyc_parking_violations.db" + target: dev diff --git a/nyc_parking_violations/tests/generic/generic_not_null.sql b/nyc_parking_violations/tests/generic/generic_not_null.sql index e49c2b087..e69de29bb 100644 --- a/nyc_parking_violations/tests/generic/generic_not_null.sql +++ b/nyc_parking_violations/tests/generic/generic_not_null.sql @@ -1,8 +0,0 @@ --- source: https://docs.getdbt.com/guides/best-practices/writing-custom-generic-tests#generic-tests-with-default-config-values -{% test generic_not_null(model, column_name) %} - - select * - from {{ model }} - where {{ column_name }} is null - -{% endtest %} \ No newline at end of file diff --git a/nyc_parking_violations/tests/violation_codes_revenue.sql b/nyc_parking_violations/tests/violation_codes_revenue.sql index 091ddfdd4..e69de29bb 100644 --- a/nyc_parking_violations/tests/violation_codes_revenue.sql +++ b/nyc_parking_violations/tests/violation_codes_revenue.sql @@ -1,11 +0,0 @@ -{{ config(severity = 'warn') }} - -SELECT - violation_code, - SUM(fee_usd) AS total_revenue_usd -FROM - {{ref('silver_parking_violation_codes')}} -GROUP BY - violation_code -HAVING - NOT(total_revenue_usd >= 1) diff --git a/run_sql_queries_here.ipynb b/run_sql_queries_here.ipynb index f7c54e6c9..1995adb17 100644 --- a/run_sql_queries_here.ipynb +++ b/run_sql_queries_here.ipynb @@ -12,17 +12,120 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
name
0bronze_parking_violation_codes
1bronze_parking_violations
2first_model
3gold_ticket_metrics
4gold_vehicles_metrics
5parking_violation_codes
6parking_violations_2023
7ref_model
8silver_parking_violation_codes
9silver_parking_violations
10silver_violation_tickets
11silver_violation_vehicles
\n", + "
" + ], + "text/plain": [ + " name\n", + "0 bronze_parking_violation_codes\n", + "1 bronze_parking_violations\n", + "2 first_model\n", + "3 gold_ticket_metrics\n", + "4 gold_vehicles_metrics\n", + "5 parking_violation_codes\n", + "6 parking_violations_2023\n", + "7 ref_model\n", + "8 silver_parking_violation_codes\n", + "9 silver_parking_violations\n", + "10 silver_violation_tickets\n", + "11 silver_violation_vehicles" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "sql_query = '''\n", "show tables\n", "'''\n", - "\n", - "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", + "with duckdb.connect('data/prod_nyc_parking_violations.db') as con:\n", " display(con.sql(sql_query).df())" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": {