diff --git a/.github/workflows/run-dbt-prod.yml b/.github/workflows/run-dbt-prod.yml new file mode 100644 index 000000000..8fc42d164 --- /dev/null +++ b/.github/workflows/run-dbt-prod.yml @@ -0,0 +1,38 @@ +# .github/workflows/run-dbt-prod.yml +name: run_dbt_prod + +on: + push: + branches: [ "main" ] + pull_request: + branches: [ "main" ] + # schedule: + # - cron: '0 8 * * *' + +env: + DBT_PROFILES_DIR: ./nyc_parking_violations + DBT_PROJECT_DIR: ./nyc_parking_violations + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - name: Set up Python 3.10 + uses: actions/setup-python@v3 + with: + python-version: "3.10" + - name: Install dependencies + run: | + python -m pip install --upgrade pip + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Run dbt Prod + run: | + dbt debug + dbt compile --target prod + dbt run --target prod + - name: Test dbt Prod + run: | + dbt test --target prod \ No newline at end of file diff --git a/.vscode/settings.json b/.vscode/settings.json index 236981053..ca20f9585 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,7 +17,6 @@ "files.autoSave": "afterDelay", "screencastMode.onlyKeyboardShortcuts": true, "terminal.integrated.fontSize": 18, - "workbench.activityBar.visible": true, "workbench.colorTheme": "Visual Studio Dark", "workbench.fontAliasing": "antialiased", "workbench.statusBar.visible": true diff --git a/NOTICE b/NOTICE index 547595f90..4f2d213b2 100644 --- a/NOTICE +++ b/NOTICE @@ -1,11 +1,17 @@ -Copyright 2022 LinkedIn Corporation +Copyright 2024 LinkedIn Corporation All Rights Reserved. Licensed under the LinkedIn Learning Exercise File License (the "License"). See LICENSE in the project root for license information. ATTRIBUTIONS: -[PLEASE PROVIDE ATTRIBUTIONS OR DELETE THIS AND THE ABOVE LINE “ATTRIBUTIONS”] + +Pandas +https://github.com/pandas-dev/pandas +Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team +Copyright (c) 2011-2021, Open source contributors. +License: BSD 3-Clause +https://opensource.org/licenses/BSD-3-Clause Please note, this project may automatically load third party code from external repositories (for example, NPM modules, Composer packages, or other dependencies). @@ -13,3 +19,38 @@ If so, such third party code may be subject to other license terms than as set forth above. In addition, such third party code may also depend on and load multiple tiers of dependencies. Please review the applicable licenses of the additional dependencies. + +=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + +PANDAS +BSD 3-Clause License + +Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team +All rights reserved. + +Copyright (c) 2011-2023, Open source contributors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index 422e0932c..06667c942 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ -# COURSENAME -This is the repository for the LinkedIn Learning course `course-name`. The full course is available from [LinkedIn Learning][lil-course-url]. +# Data Engineering with Data Build Tool (DBT) +This is the repository for the LinkedIn Learning course `Data Engineering with Data Build Tool (DBT)`. The full course is available from [LinkedIn Learning][lil-course-url]. ![course-name-alt-text][lil-thumbnail-url] +Data Build Tool (dbt) has quickly become an essential tool in many data stacks ranging from startups to big tech for managing data transformations. In this course, data engineer Mark Freeman helps you get started with setting up, running, and managing a dbt project via the open-source offering dbt Core. Learn how to install dbt Core, configure an environment for dbt, create and manage a dbt project, and deploy a dbt project in production. If you’re a data professional tasked with implementing dbt within your organization, recently joined a team utilizing dbt and need to upskill, or just want to learn about dbt to increase your competitiveness within the data job market, check out this course. + _See the readme file in the main branch for updated instructions and information._ ## Instructions This repository has branches for each of the videos in the course. You can use the branch pop up menu in github to switch to a specific branch and take a look at the course at that stage, or you can add `/tree/BRANCH_NAME` to the URL to go to the branch you want to access. @@ -22,15 +24,9 @@ To resolve this issue: Add changes to git using this command: git add . Commit changes using this command: git commit -m "some message" -## Installing -1. To use these exercise files, you must have the following installed: - - [list of requirements for course] -2. Clone this repository into your local machine using the terminal (Mac), CMD (Windows), or a GUI tool like SourceTree. -3. [Course-specific instructions] - [0]: # (Replace these placeholder URLs with actual course URLs) -[lil-course-url]: https://www.linkedin.com/learning/ -[lil-thumbnail-url]: http:// +[lil-course-url]: https://www.linkedin.com/learning/data-engineering-with-dbt +[lil-thumbnail-url]: https://media.licdn.com/dms/image/D560DAQHnrbGtKKAabg/learning-public-crop_675_1200/0/1703199806902?e=2147483647&v=beta&t=Q6jKFTslSvhl3bYTkd0PuIEVnIx26lCa-XWVE5hCmm0 diff --git a/data/nyc_parking_violations.db b/data/nyc_parking_violations.db new file mode 100644 index 000000000..8c30b948a Binary files /dev/null and b/data/nyc_parking_violations.db differ diff --git a/data/prod_nyc_parking_violations.db b/data/prod_nyc_parking_violations.db new file mode 100644 index 000000000..e3bcadd2d Binary files /dev/null and b/data/prod_nyc_parking_violations.db differ diff --git a/logs/dbt.log b/logs/dbt.log new file mode 100644 index 000000000..3ffdc257b --- /dev/null +++ b/logs/dbt.log @@ -0,0 +1,67 @@ +19:55:38.151755 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [, , ]} + + +============================== 19:55:38.190124 | 4ae9eb80-7a61-493f-b7d8-e2223201dba5 ============================== +19:55:38.190124 [info ] [MainThread]: Running with dbt=1.6.1 +19:55:38.190813 [debug] [MainThread]: running dbt with arguments {'quiet': 'False', 'introspect': 'True', 'partial_parse': 'True', 'use_colors': 'True', 'write_json': 'True', 'warn_error': 'None', 'log_path': 'logs', 'send_anonymous_usage_stats': 'True', 'printer_width': '80', 'target_path': 'None', 'debug': 'False', 'invocation_command': 'dbt init', 'static_parser': 'True', 'version_check': 'True', 'cache_selected_only': 'False', 'warn_error_options': 'WarnErrorOptions(include=[], exclude=[])', 'use_experimental_parser': 'False', 'log_cache_events': 'False', 'fail_fast': 'False', 'profiles_dir': '/home/vscode/.dbt', 'log_format': 'default', 'no_print': 'None', 'indirect_selection': 'eager'} +19:55:38.191400 [warn ] [MainThread]: [ConfigFolderDirectory]: Unable to parse dict {'dir': PosixPath('/home/vscode/.dbt')} +19:55:38.191862 [info ] [MainThread]: Creating dbt configuration folder at +19:56:18.260747 [debug] [MainThread]: Starter project path: /home/vscode/.local/lib/python3.10/site-packages/dbt/include/starter_project +19:56:18.270153 [info ] [MainThread]: +Your new dbt project "nyc_parking_violations" was created! + +For more information on how to configure the profiles.yml file, +please consult the dbt documentation here: + + https://docs.getdbt.com/docs/configure-your-profile + +One more thing: + +Need help? Don't hesitate to reach out to us via GitHub issues or on Slack: + + https://community.getdbt.com/ + +Happy modeling! + +19:56:18.270776 [info ] [MainThread]: Setting up your profile. +19:56:57.348418 [info ] [MainThread]: No sample profile found for duckdb. +19:56:57.349268 [debug] [MainThread]: Command `dbt init` succeeded at 19:56:57.349088 after 79.20 seconds +19:56:57.349734 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [, , ]} +19:56:57.350208 [debug] [MainThread]: Flushing usage events +14:30:50.361209 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'start', 'context': [, , ]} + + +============================== 14:30:50.366278 | f00b8a90-b999-4239-9ebb-d05c745067ce ============================== +14:30:50.366278 [info ] [MainThread]: Running with dbt=1.6.1 +14:30:50.366949 [debug] [MainThread]: running dbt with arguments {'log_format': 'default', 'no_print': 'None', 'write_json': 'True', 'send_anonymous_usage_stats': 'True', 'log_cache_events': 'False', 'use_colors': 'True', 'warn_error': 'None', 'fail_fast': 'False', 'use_experimental_parser': 'False', 'introspect': 'True', 'quiet': 'False', 'debug': 'False', 'warn_error_options': 'WarnErrorOptions(include=[], exclude=[])', 'target_path': 'None', 'cache_selected_only': 'False', 'indirect_selection': 'eager', 'log_path': 'logs', 'static_parser': 'True', 'profiles_dir': '/home/vscode/.dbt', 'invocation_command': 'dbt debug', 'printer_width': '80', 'version_check': 'True', 'partial_parse': 'True'} +14:30:50.367574 [info ] [MainThread]: dbt version: 1.6.1 +14:30:50.368018 [info ] [MainThread]: python version: 3.10.12 +14:30:50.368513 [info ] [MainThread]: python path: /usr/local/bin/python +14:30:50.368967 [info ] [MainThread]: os info: Linux-6.8.0-1027-azure-x86_64-with-glibc2.31 +14:30:50.369470 [info ] [MainThread]: Using profiles dir at /home/vscode/.dbt +14:30:50.369922 [info ] [MainThread]: Using profiles.yml file at /home/vscode/.dbt/profiles.yml +14:30:50.370396 [info ] [MainThread]: Using dbt_project.yml file at /workspaces/data-engineering-with-data-build-tool-dbt-4458303/dbt_project.yml +14:30:50.370867 [info ] [MainThread]: Configuration: +14:30:50.371364 [info ] [MainThread]: profiles.yml file [ERROR not found] +14:30:50.371890 [info ] [MainThread]: dbt_project.yml file [ERROR not found] +14:30:50.372384 [info ] [MainThread]: Required dependencies: +14:30:50.372956 [debug] [MainThread]: Executing "git --help" +14:30:50.376109 [debug] [MainThread]: STDOUT: "b"usage: git [-v | --version] [-h | --help] [-C ] [-c =]\n [--exec-path[=]] [--html-path] [--man-path] [--info-path]\n [-p | --paginate | -P | --no-pager] [--no-replace-objects] [--bare]\n [--git-dir=] [--work-tree=] [--namespace=]\n [--config-env==] []\n\nThese are common Git commands used in various situations:\n\nstart a working area (see also: git help tutorial)\n clone Clone a repository into a new directory\n init Create an empty Git repository or reinitialize an existing one\n\nwork on the current change (see also: git help everyday)\n add Add file contents to the index\n mv Move or rename a file, a directory, or a symlink\n restore Restore working tree files\n rm Remove files from the working tree and from the index\n\nexamine the history and state (see also: git help revisions)\n bisect Use binary search to find the commit that introduced a bug\n diff Show changes between commits, commit and working tree, etc\n grep Print lines matching a pattern\n log Show commit logs\n show Show various types of objects\n status Show the working tree status\n\ngrow, mark and tweak your common history\n branch List, create, or delete branches\n commit Record changes to the repository\n merge Join two or more development histories together\n rebase Reapply commits on top of another base tip\n reset Reset current HEAD to the specified state\n switch Switch branches\n tag Create, list, delete or verify a tag object signed with GPG\n\ncollaborate (see also: git help workflows)\n fetch Download objects and refs from another repository\n pull Fetch from and integrate with another repository or a local branch\n push Update remote refs along with associated objects\n\n'git help -a' and 'git help -g' list available subcommands and some\nconcept guides. See 'git help ' or 'git help '\nto read about a specific subcommand or concept.\nSee 'git help git' for an overview of the system.\n"" +14:30:50.376742 [debug] [MainThread]: STDERR: "b''" +14:30:50.377235 [info ] [MainThread]: - git [OK found] + +14:30:50.377734 [info ] [MainThread]: Connection test skipped since no profile was found +14:30:50.378387 [info ] [MainThread]: 2 checks failed: +14:30:50.378976 [info ] [MainThread]: dbt looked for a profiles.yml file in /home/vscode/.dbt/profiles.yml, but did +not find one. For more information on configuring your profile, consult the +documentation: + +https://docs.getdbt.com/docs/configure-your-profile + + +14:30:50.379621 [info ] [MainThread]: Project loading failed for the following reason: + project path not found + +14:30:50.380444 [debug] [MainThread]: Command `dbt debug` failed at 14:30:50.380303 after 0.02 seconds +14:30:50.381042 [debug] [MainThread]: Sending event: {'category': 'dbt', 'action': 'invocation', 'label': 'end', 'context': [, , ]} +14:30:50.381726 [debug] [MainThread]: Flushing usage events diff --git a/nyc_parking_violations/.gitignore b/nyc_parking_violations/.gitignore new file mode 100644 index 000000000..49f147cb9 --- /dev/null +++ b/nyc_parking_violations/.gitignore @@ -0,0 +1,4 @@ + +target/ +dbt_packages/ +logs/ diff --git a/nyc_parking_violations/.user.yml b/nyc_parking_violations/.user.yml new file mode 100644 index 000000000..17f31ac45 --- /dev/null +++ b/nyc_parking_violations/.user.yml @@ -0,0 +1 @@ +id: 70761850-c03a-4f73-809a-02729b3f9420 diff --git a/nyc_parking_violations/README.md b/nyc_parking_violations/README.md new file mode 100644 index 000000000..7874ac842 --- /dev/null +++ b/nyc_parking_violations/README.md @@ -0,0 +1,15 @@ +Welcome to your new dbt project! + +### Using the starter project + +Try running the following commands: +- dbt run +- dbt test + + +### Resources: +- Learn more about dbt [in the docs](https://docs.getdbt.com/docs/introduction) +- Check out [Discourse](https://discourse.getdbt.com/) for commonly asked questions and answers +- Join the [chat](https://community.getdbt.com/) on Slack for live discussions and support +- Find [dbt events](https://events.getdbt.com) near you +- Check out [the blog](https://blog.getdbt.com/) for the latest news on dbt's development and best practices diff --git a/nyc_parking_violations/analyses/.gitkeep b/nyc_parking_violations/analyses/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/nyc_parking_violations/dbt_project.yml b/nyc_parking_violations/dbt_project.yml new file mode 100644 index 000000000..ec48605d3 --- /dev/null +++ b/nyc_parking_violations/dbt_project.yml @@ -0,0 +1,52 @@ + +# Name your project! Project names should contain only lowercase characters +# and underscores. A good package name should reflect your organization's +# name or the intended use of these models +name: 'nyc_parking_violations' +version: '1.0.0' +config-version: 2 + +# This setting configures which "profile" dbt uses for this project. +profile: 'nyc_parking_violations' + +# These configurations specify where dbt should look for different types of files. +# The `model-paths` config, for example, states that models in this project can be +# found in the "models/" directory. You probably won't need to change these! +model-paths: ["models"] +analysis-paths: ["analyses"] +test-paths: ["tests"] +seed-paths: ["seeds"] +macro-paths: ["macros"] +snapshot-paths: ["snapshots"] + +clean-targets: # directories to be removed by `dbt clean` + - "target" + - "dbt_packages" + + +# Configuring models +# Full documentation: https://docs.getdbt.com/docs/configuring-models + +# In this example config, we tell dbt to build all models in the example/ +# directory as views. These settings can be overridden in the individual model +# files using the `{{ config(...) }}` macro. +models: + nyc_parking_violations: + # Config indicated by + and applies to all files under models/example/ + example: + +materialized: ephemeral + bronze: + +materialized: view + silver: + silver_parking_violation_codes: + +materialized: ephemeral + silver_parking_violations: + +materialized: ephemeral + silver_violation_tickets: + +materialized: view + silver_violation_vehicles: + +materialized: view + gold: + +materialized: table +tests: + +store_failures: true diff --git a/nyc_parking_violations/macros/.gitkeep b/nyc_parking_violations/macros/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql b/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql new file mode 100644 index 000000000..217a5f263 --- /dev/null +++ b/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql @@ -0,0 +1,7 @@ +SELECT + code AS violation_code, + definition, + manhattan_96th_st_below, + all_other_areas +FROM + parking_violation_codes diff --git a/nyc_parking_violations/models/bronze/bronze_parking_violations.sql b/nyc_parking_violations/models/bronze/bronze_parking_violations.sql new file mode 100644 index 000000000..46525da2a --- /dev/null +++ b/nyc_parking_violations/models/bronze/bronze_parking_violations.sql @@ -0,0 +1,23 @@ +SELECT + summons_number, + registration_state, + plate_type, + issue_date, + violation_code, + vehicle_body_type, + vehicle_make, + issuing_agency, + vehicle_expiration_date, + violation_location, + violation_precinct, + issuer_precinct, + issuer_code, + issuer_command, + issuer_squad, + violation_time, + violation_county, + violation_legal_code, + vehicle_color, + vehicle_year, +FROM + parking_violations_2023 \ No newline at end of file diff --git a/nyc_parking_violations/models/docs/docs_block.md b/nyc_parking_violations/models/docs/docs_block.md new file mode 100644 index 000000000..fff56dafc --- /dev/null +++ b/nyc_parking_violations/models/docs/docs_block.md @@ -0,0 +1,53 @@ +{% docs violation_code %} Code representing the specific parking violation. {% enddocs %} + +{% docs definition %} Description of the violation for a respective code. {% enddocs %} + +{% docs manhattan_96th_st_below %} The fee in $USD for a violation on or below Manhattan 96th Street. {% enddocs %} + +{% docs all_other_areas %} The fee in $USD for a violation not on or below Manhattan 96th Street. {% enddocs %} + +{% docs summons_number %} Unique identifier for each summons issued for a parking violation. {% enddocs %} + +{% docs registration_state %} The state where the vehicle is registered. {% enddocs %} + +{% docs plate_type %} The type of license plate. {% enddocs %} + +{% docs issue_date %} The date when the summons was issued. {% enddocs %} + +{% docs vehicle_body_type %} The body type of the vehicle involved in the violation. {% enddocs %} + +{% docs vehicle_make %} The make or brand of the vehicle. {% enddocs %} + +{% docs issuing_agency %} The agency that issued the summons. {% enddocs %} + +{% docs vehicle_expiration_date %} The date when the vehicle's registration expires. {% enddocs %} + +{% docs violation_location %} General location where the violation occurred. {% enddocs %} + +{% docs violation_precinct %} Precinct where the violation was identified. {% enddocs %} + +{% docs issuer_precinct %} Precinct of the officer or official who issued the summons. {% enddocs %} + +{% docs issuer_code %} Unique code identifying the issuer. {% enddocs %} + +{% docs issuer_command %} Command or unit of the issuer. {% enddocs %} + +{% docs issuer_squad %} Squad detail for the issuer. {% enddocs %} + +{% docs violation_time %} Time when the violation occurred. {% enddocs %} + +{% docs violation_county %} County where the violation took place. {% enddocs %} + +{% docs violation_legal_code %} Legal code associated with the violation. {% enddocs %} + +{% docs vehicle_color %} Color of the vehicle involved in the violation. {% enddocs %} + +{% docs vehicle_year %} Manufacturing year of the vehicle. {% enddocs %} + +{% docs fee_usd %} The fee charged for a parking violation, specified in USD. This fee varies depending on the location of the violation. {% enddocs %} + +{% docs is_manhattan_96th_st_below %} A boolean value indicating whether the violation occurred in Manhattan on or below 96th Street. {% enddocs %} + +{% docs ticket_count %} The total number of tickets issued for a specific violation code. {% enddocs %} + +{% docs total_revenue_usd %} The total revenue accumulated from tickets, based on the violation code. This sum is represented in USD. {% enddocs %} \ No newline at end of file diff --git a/nyc_parking_violations/models/docs/schema.yml b/nyc_parking_violations/models/docs/schema.yml new file mode 100644 index 000000000..0524962a6 --- /dev/null +++ b/nyc_parking_violations/models/docs/schema.yml @@ -0,0 +1,188 @@ +models: + - name: bronze_parking_violation_codes + description: Raw data representing the violation codes and their fees. + columns: + - name: violation_code + description: '{{ doc("violation_code") }}' + - name: definition + description: '{{ doc("definition") }}' + - name: manhattan_96th_st_below + description: '{{ doc("manhattan_96th_st_below") }}' + - name: all_other_areas + description: '{{ doc("all_other_areas") }}' + + - name: bronze_parking_violations + description: Raw data related to parking violations in 2023, encompassing various details about each violation. + columns: + - name: summons_number + description: '{{ doc("summons_number") }}' + tests: + - unique + - not_null + - generic_not_null + - name: registration_state + description: '{{ doc("registration_state") }}' + - name: plate_type + description: '{{ doc("plate_type") }}' + - name: issue_date + description: '{{ doc("issue_date") }}' + - name: violation_code + description: '{{ doc("violation_code") }}' + - name: vehicle_body_type + description: '{{ doc("vehicle_body_type") }}' + - name: vehicle_make + description: '{{ doc("vehicle_make") }}' + - name: issuing_agency + description: '{{ doc("issuing_agency") }}' + - name: vehicle_expiration_date + description: '{{ doc("vehicle_expiration_date") }}' + - name: violation_location + description: '{{ doc("violation_location") }}' + - name: violation_precinct + description: '{{ doc("violation_precinct") }}' + - name: issuer_precinct + description: '{{ doc("issuer_precinct") }}' + - name: issuer_code + description: '{{ doc("issuer_code") }}' + - name: issuer_command + description: '{{ doc("issuer_command") }}' + - name: issuer_squad + description: '{{ doc("issuer_squad") }}' + - name: violation_time + description: '{{ doc("violation_time") }}' + - name: violation_county + description: '{{ doc("violation_county") }}' + - name: violation_legal_code + description: '{{ doc("violation_legal_code") }}' + - name: vehicle_color + description: '{{ doc("vehicle_color") }}' + - name: vehicle_year + description: '{{ doc("vehicle_year") }}' + + - name: silver_parking_violation_codes + description: "This model unifies violation codes, providing a comprehensive view of violations, indicating whether they occurred on/below 96th St in Manhattan or in other areas, along with the respective fees in USD." + columns: + - name: violation_code + description: '{{ doc("violation_code") }}' + - name: definition + description: '{{ doc("definition") }}' + - name: is_manhattan_96th_st_below + description: '{{ doc("is_manhattan_96th_st_below") }}' + - name: fee_usd + description: '{{ doc("fee_usd") }}' + + - name: silver_parking_violations + description: "Enhanced view of parking violations, enriched with details and specific indicators such as the flag for violations in Manhattan on or below 96th Street." + columns: + - name: summons_number + description: '{{ doc("summons_number") }}' + - name: registration_state + description: '{{ doc("registration_state") }}' + - name: plate_type + description: '{{ doc("plate_type") }}' + - name: issue_date + description: '{{ doc("issue_date") }}' + - name: violation_code + description: '{{ doc("violation_code") }}' + - name: vehicle_body_type + description: '{{ doc("vehicle_body_type") }}' + - name: vehicle_make + description: '{{ doc("vehicle_make") }}' + - name: issuing_agency + description: '{{ doc("issuing_agency") }}' + - name: vehicle_expiration_date + description: '{{ doc("vehicle_expiration_date") }}' + - name: violation_location + description: '{{ doc("violation_location") }}' + - name: violation_precinct + description: '{{ doc("violation_precinct") }}' + - name: issuer_precinct + description: '{{ doc("issuer_precinct") }}' + - name: issuer_code + description: '{{ doc("issuer_code") }}' + - name: issuer_command + description: '{{ doc("issuer_command") }}' + - name: issuer_squad + description: '{{ doc("issuer_squad") }}' + - name: violation_time + description: '{{ doc("violation_time") }}' + - name: violation_county + description: '{{ doc("violation_county") }}' + - name: violation_legal_code + description: '{{ doc("violation_legal_code") }}' + - name: vehicle_color + description: '{{ doc("vehicle_color") }}' + - name: vehicle_year + description: '{{ doc("vehicle_year") }}' + - name: is_manhattan_96th_st_below + description: '{{ doc("is_manhattan_96th_st_below") }}' + + - name: silver_violation_tickets + description: "Consolidated information on parking violations, enriched with associated fee details." + columns: + - name: summons_number + description: '{{ doc("summons_number") }}' + - name: issue_date + description: '{{ doc("issue_date") }}' + - name: violation_code + description: '{{ doc("violation_code") }}' + - name: is_manhattan_96th_st_below + description: '{{ doc("is_manhattan_96th_st_below") }}' + - name: issuing_agency + description: '{{ doc("issuing_agency") }}' + - name: violation_location + description: '{{ doc("violation_location") }}' + - name: violation_precinct + description: '{{ doc("violation_precinct") }}' + - name: issuer_precinct + description: '{{ doc("issuer_precinct") }}' + - name: issuer_code + description: '{{ doc("issuer_code") }}' + - name: issuer_command + description: '{{ doc("issuer_command") }}' + - name: issuer_squad + description: '{{ doc("issuer_squad") }}' + - name: violation_time + description: '{{ doc("violation_time") }}' + - name: violation_county + description: '{{ doc("violation_county") }}' + - name: violation_legal_code + description: '{{ doc("violation_legal_code") }}' + + - name: silver_violation_vehicles + description: "Details of the vehicles involved in parking violations." + columns: + - name: summons_number + description: '{{ doc("summons_number") }}' + - name: registration_state + description: '{{ doc("registration_state") }}' + - name: plate_type + description: '{{ doc("plate_type") }}' + - name: vehicle_body_type + description: '{{ doc("vehicle_body_type") }}' + - name: vehicle_make + description: '{{ doc("vehicle_make") }}' + - name: vehicle_expiration_date + description: '{{ doc("vehicle_expiration_date") }}' + - name: vehicle_color + description: '{{ doc("vehicle_color") }}' + - name: vehicle_year + description: '{{ doc("vehicle_year") }}' + + - name: gold_ticket_metrics + description: "Aggregated metrics representing the total tickets and revenue by violation code." + columns: + - name: violation_code + description: '{{ doc("violation_code") }}' + - name: ticket_count + description: '{{ doc("ticket_count") }}' + - name: total_revenue_usd + description: '{{ doc("total_revenue_usd") }}' + + - name: gold_vehicles_metrics + description: "Aggregated metrics detailing the number of tickets per vehicle, identified by the plate ID." + columns: + - name: registration_state + description: '{{ doc("registration_state") }}' + - name: ticket_count + description: '{{ doc("ticket_count") }}' \ No newline at end of file diff --git a/nyc_parking_violations/models/example/first_model.sql b/nyc_parking_violations/models/example/first_model.sql new file mode 100644 index 000000000..942e472df --- /dev/null +++ b/nyc_parking_violations/models/example/first_model.sql @@ -0,0 +1 @@ +SELECT * FROM parking_violation_codes \ No newline at end of file diff --git a/nyc_parking_violations/models/example/ref_model.sql b/nyc_parking_violations/models/example/ref_model.sql new file mode 100644 index 000000000..753ae2e29 --- /dev/null +++ b/nyc_parking_violations/models/example/ref_model.sql @@ -0,0 +1,4 @@ +select + count(*) + from + {{ref('first_model')}} \ No newline at end of file diff --git a/nyc_parking_violations/models/gold/gold_ticket_metrics.sql b/nyc_parking_violations/models/gold/gold_ticket_metrics.sql new file mode 100644 index 000000000..861de4d61 --- /dev/null +++ b/nyc_parking_violations/models/gold/gold_ticket_metrics.sql @@ -0,0 +1,10 @@ +SELECT + violation_code, + COUNT(summons_number) AS ticket_count, + SUM(fee_usd) AS total_revenue_usd +FROM + {{ref('silver_violation_tickets')}} +GROUP BY + violation_code +ORDER BY + total_revenue_usd DESC \ No newline at end of file diff --git a/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql b/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql new file mode 100644 index 000000000..af796bfa9 --- /dev/null +++ b/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql @@ -0,0 +1,11 @@ +SELECT + registration_state, + COUNT(summons_number) AS ticket_count, +FROM + {{ref('silver_violation_vehicles')}} +WHERE + registration_state != 'NY' +GROUP BY + registration_state +ORDER BY + ticket_count DESC \ No newline at end of file diff --git a/nyc_parking_violations/models/silver/silver_parking_violation_codes.sql b/nyc_parking_violations/models/silver/silver_parking_violation_codes.sql new file mode 100644 index 000000000..44144d9dd --- /dev/null +++ b/nyc_parking_violations/models/silver/silver_parking_violation_codes.sql @@ -0,0 +1,24 @@ +WITH manhattan_violation_codes AS ( + SELECT + violation_code, + definition, + TRUE AS is_manhattan_96th_st_below, + manhattan_96th_st_below AS fee_usd, + FROM + {{ref('bronze_parking_violation_codes')}} +), + +all_other_violation_codes AS ( + SELECT + violation_code, + definition, + FALSE AS is_manhattan_96th_st_below, + all_other_areas AS fee_usd, + FROM + {{ref('bronze_parking_violation_codes')}} +) + +SELECT * FROM manhattan_violation_codes +UNION ALL +SELECT * FROM all_other_violation_codes +ORDER BY violation_code ASC \ No newline at end of file diff --git a/nyc_parking_violations/models/silver/silver_parking_violations.sql b/nyc_parking_violations/models/silver/silver_parking_violations.sql new file mode 100644 index 000000000..3a6e112cb --- /dev/null +++ b/nyc_parking_violations/models/silver/silver_parking_violations.sql @@ -0,0 +1,28 @@ +SELECT + summons_number, + registration_state, + plate_type, + issue_date, + violation_code, + vehicle_body_type, + vehicle_make, + issuing_agency, + vehicle_expiration_date, + violation_location, + violation_precinct, + issuer_precinct, + issuer_code, + issuer_command, + issuer_squad, + violation_time, + violation_county, + violation_legal_code, + vehicle_color, + vehicle_year, + CASE WHEN + violation_county == 'MN' + THEN TRUE + ELSE FALSE + END AS is_manhattan_96th_st_below +FROM + {{ref('bronze_parking_violations')}} \ No newline at end of file diff --git a/nyc_parking_violations/models/silver/silver_violation_tickets.sql b/nyc_parking_violations/models/silver/silver_violation_tickets.sql new file mode 100644 index 000000000..66b5ce0d4 --- /dev/null +++ b/nyc_parking_violations/models/silver/silver_violation_tickets.sql @@ -0,0 +1,22 @@ +SELECT + violations.summons_number, + violations.issue_date, + violations.violation_code, + violations.is_manhattan_96th_st_below, + violations.issuing_agency, + violations.violation_location, + violations.violation_precinct, + violations.issuer_precinct, + violations.issuer_code, + violations.issuer_command, + violations.issuer_squad, + violations.violation_time, + violations.violation_county, + violations.violation_legal_code, + codes.fee_usd +FROM + {{ref('silver_parking_violations')}} AS violations +LEFT JOIN + {{ref('silver_parking_violation_codes')}} AS codes ON + violations.violation_code = codes.violation_code AND + violations.is_manhattan_96th_st_below = codes.is_manhattan_96th_st_below \ No newline at end of file diff --git a/nyc_parking_violations/models/silver/silver_violation_vehicles.sql b/nyc_parking_violations/models/silver/silver_violation_vehicles.sql new file mode 100644 index 000000000..e2b8537a2 --- /dev/null +++ b/nyc_parking_violations/models/silver/silver_violation_vehicles.sql @@ -0,0 +1,11 @@ +SELECT + summons_number, + registration_state, + plate_type, + vehicle_body_type, + vehicle_make, + vehicle_expiration_date, + vehicle_color, + vehicle_year +FROM + {{ref('silver_parking_violations')}} \ No newline at end of file diff --git a/nyc_parking_violations/profiles.yml b/nyc_parking_violations/profiles.yml new file mode 100644 index 000000000..3d89d2f1e --- /dev/null +++ b/nyc_parking_violations/profiles.yml @@ -0,0 +1,12 @@ +nyc_parking_violations: + outputs: + dev: + type: duckdb + path: '../data/nyc_parking_violations.db' + prod: + type: duckdb + # note that path is slightly different as GitHub actions + # start in the root directory and not in the + # nyc_parking_violations directory + path: '../data/prod_nyc_parking_violations.db' + target: dev \ No newline at end of file diff --git a/nyc_parking_violations/seeds/.gitkeep b/nyc_parking_violations/seeds/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/nyc_parking_violations/snapshots/.gitkeep b/nyc_parking_violations/snapshots/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/nyc_parking_violations/tests/.gitkeep b/nyc_parking_violations/tests/.gitkeep new file mode 100644 index 000000000..e69de29bb diff --git a/nyc_parking_violations/tests/generic/generic_not_null.sql b/nyc_parking_violations/tests/generic/generic_not_null.sql new file mode 100644 index 000000000..e49c2b087 --- /dev/null +++ b/nyc_parking_violations/tests/generic/generic_not_null.sql @@ -0,0 +1,8 @@ +-- source: https://docs.getdbt.com/guides/best-practices/writing-custom-generic-tests#generic-tests-with-default-config-values +{% test generic_not_null(model, column_name) %} + + select * + from {{ model }} + where {{ column_name }} is null + +{% endtest %} \ No newline at end of file diff --git a/nyc_parking_violations/tests/violation_codes_revenue.sql b/nyc_parking_violations/tests/violation_codes_revenue.sql new file mode 100644 index 000000000..9426a0ba9 --- /dev/null +++ b/nyc_parking_violations/tests/violation_codes_revenue.sql @@ -0,0 +1,11 @@ +{{ config(severity = 'warn') }} + +SELECT + violation_code, + SUM(fee_usd) AS total_revenue_usd +FROM + {{ref('silver_parking_violation_codes')}} +GROUP BY + violation_code +HAVING + NOT(total_revenue_usd >= 1) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2fb00bfb3..36e1da1dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ requests>=2.28 dbt-core==1.6.1 dbt-duckdb==1.6.0 duckdb==0.9.0 +pandas==2.1.1 +numpy==1.26.4 diff --git a/run_sql_queries_here.ipynb b/run_sql_queries_here.ipynb index c8212b866..8fb8a2f28 100644 --- a/run_sql_queries_here.ipynb +++ b/run_sql_queries_here.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -12,17 +12,210 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
violation_codetotal_revenue_usd
0410.0
\n", + "
" + ], + "text/plain": [ + " violation_code total_revenue_usd\n", + "0 41 0.0" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "sql_query = '''\n", - "show tables\n", + " select * from \"nyc_parking_violations\".\"main_dbt_test__audit\".\"violation_codes_revenue\"\n", "'''\n", "\n", "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", " display(con.sql(sql_query).df())" ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "sql_query_import_1= '''\n", + "CREATE OR REPLACE TABLE parking_violation_codes AS\n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/dof_parking_violation_codes.csv',\n", + " normalize_names = True\n", + ")\n", + "'''\n", + "\n", + "sql_query_import_2 = '''\n", + "CREATE OR REPLACE TABLE parking_violations_2023 AS\n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/parking_violations_issued_fiscal_year_2023_sample.csv',\n", + " normalize_names = True\n", + " )\n", + "'''\n", + "\n", + "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", + " con.sql(sql_query_import_1)\n", + " con.sql(sql_query_import_2)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "sql_query_import_1= '''\n", + "CREATE OR REPLACE TABLE parking_violation_codes AS\n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/dof_parking_violation_codes.csv',\n", + " normalize_names = True\n", + ")\n", + "'''\n", + "\n", + "sql_query_import_2 = '''\n", + "CREATE OR REPLACE TABLE parking_violations_2023 AS\n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/parking_violations_issued_fiscal_year_2023_sample.csv',\n", + " normalize_names = True\n", + " )\n", + "'''\n", + "\n", + "with duckdb.connect('data/prod_nyc_parking_violations.db') as con:\n", + " con.sql(sql_query_import_1)\n", + " con.sql(sql_query_import_2)" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
name
0bronze_parking_violation_codes
1bronze_parking_violations
2gold_ticket_metrics
3gold_vehicles_metrics
4parking_violation_codes
5parking_violations_2023
6silver_violation_tickets
7silver_violation_vehicles
\n", + "
" + ], + "text/plain": [ + " name\n", + "0 bronze_parking_violation_codes\n", + "1 bronze_parking_violations\n", + "2 gold_ticket_metrics\n", + "3 gold_vehicles_metrics\n", + "4 parking_violation_codes\n", + "5 parking_violations_2023\n", + "6 silver_violation_tickets\n", + "7 silver_violation_vehicles" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sql_query = '''\n", + " show tables\n", + "'''\n", + "\n", + "with duckdb.connect('data/prod_nyc_parking_violations.db') as con:\n", + " display(con.sql(sql_query).df())" + ] } ], "metadata": { @@ -32,8 +225,16 @@ "name": "python3" }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", - "version": "3.9.6 (default, May 7 2023, 23:32:44) \n[Clang 14.0.3 (clang-1403.0.22.14.1)]" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" }, "orig_nbformat": 4, "vscode": {