diff --git a/NOTICE b/NOTICE index 547595f90..4f2d213b2 100644 --- a/NOTICE +++ b/NOTICE @@ -1,11 +1,17 @@ -Copyright 2022 LinkedIn Corporation +Copyright 2024 LinkedIn Corporation All Rights Reserved. Licensed under the LinkedIn Learning Exercise File License (the "License"). See LICENSE in the project root for license information. ATTRIBUTIONS: -[PLEASE PROVIDE ATTRIBUTIONS OR DELETE THIS AND THE ABOVE LINE “ATTRIBUTIONS”] + +Pandas +https://github.com/pandas-dev/pandas +Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team +Copyright (c) 2011-2021, Open source contributors. +License: BSD 3-Clause +https://opensource.org/licenses/BSD-3-Clause Please note, this project may automatically load third party code from external repositories (for example, NPM modules, Composer packages, or other dependencies). @@ -13,3 +19,38 @@ If so, such third party code may be subject to other license terms than as set forth above. In addition, such third party code may also depend on and load multiple tiers of dependencies. Please review the applicable licenses of the additional dependencies. + +=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=-=- + +PANDAS +BSD 3-Clause License + +Copyright (c) 2008-2011, AQR Capital Management, LLC, Lambda Foundry, Inc. and PyData Development Team +All rights reserved. + +Copyright (c) 2011-2023, Open source contributors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ""AS IS"" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/README.md b/README.md index 422e0932c..06667c942 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ -# COURSENAME -This is the repository for the LinkedIn Learning course `course-name`. The full course is available from [LinkedIn Learning][lil-course-url]. +# Data Engineering with Data Build Tool (DBT) +This is the repository for the LinkedIn Learning course `Data Engineering with Data Build Tool (DBT)`. The full course is available from [LinkedIn Learning][lil-course-url]. ![course-name-alt-text][lil-thumbnail-url] +Data Build Tool (dbt) has quickly become an essential tool in many data stacks ranging from startups to big tech for managing data transformations. In this course, data engineer Mark Freeman helps you get started with setting up, running, and managing a dbt project via the open-source offering dbt Core. Learn how to install dbt Core, configure an environment for dbt, create and manage a dbt project, and deploy a dbt project in production. If you’re a data professional tasked with implementing dbt within your organization, recently joined a team utilizing dbt and need to upskill, or just want to learn about dbt to increase your competitiveness within the data job market, check out this course. + _See the readme file in the main branch for updated instructions and information._ ## Instructions This repository has branches for each of the videos in the course. You can use the branch pop up menu in github to switch to a specific branch and take a look at the course at that stage, or you can add `/tree/BRANCH_NAME` to the URL to go to the branch you want to access. @@ -22,15 +24,9 @@ To resolve this issue: Add changes to git using this command: git add . Commit changes using this command: git commit -m "some message" -## Installing -1. To use these exercise files, you must have the following installed: - - [list of requirements for course] -2. Clone this repository into your local machine using the terminal (Mac), CMD (Windows), or a GUI tool like SourceTree. -3. [Course-specific instructions] - [0]: # (Replace these placeholder URLs with actual course URLs) -[lil-course-url]: https://www.linkedin.com/learning/ -[lil-thumbnail-url]: http:// +[lil-course-url]: https://www.linkedin.com/learning/data-engineering-with-dbt +[lil-thumbnail-url]: https://media.licdn.com/dms/image/D560DAQHnrbGtKKAabg/learning-public-crop_675_1200/0/1703199806902?e=2147483647&v=beta&t=Q6jKFTslSvhl3bYTkd0PuIEVnIx26lCa-XWVE5hCmm0 diff --git a/data/nyc_parking_violations.db b/data/nyc_parking_violations.db new file mode 100644 index 000000000..7d1800188 Binary files /dev/null and b/data/nyc_parking_violations.db differ diff --git a/requirements.txt b/requirements.txt index 2fb00bfb3..36e1da1dd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,5 @@ requests>=2.28 dbt-core==1.6.1 dbt-duckdb==1.6.0 duckdb==0.9.0 +pandas==2.1.1 +numpy==1.26.4 diff --git a/run_sql_queries_here.ipynb b/run_sql_queries_here.ipynb index c8212b866..f2a1f944d 100644 --- a/run_sql_queries_here.ipynb +++ b/run_sql_queries_here.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -12,9 +12,48 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
name
\n", + "
" + ], + "text/plain": [ + "Empty DataFrame\n", + "Columns: [name]\n", + "Index: []" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "sql_query = '''\n", "show tables\n", @@ -23,6 +62,105 @@ "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", " display(con.sql(sql_query).df())" ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "sql_query_import_1 = '''\n", + "CREATE OR REPLACE TABLE parking_violation_codes AS\n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/dof_parking_violation_codes.csv',\n", + " normalize_names=True\n", + " )\n", + "'''\n", + "\n", + "sql_query_import_2 = '''\n", + "CREATE OR REPLACE TABLE parking_violations_2023 AS\n", + "SELECT *\n", + "FROM read_csv_auto(\n", + " 'data/parking_violations_issued_fiscal_year_2023_sample.csv',\n", + " normalize_names=True\n", + " )\n", + "'''\n", + "\n", + "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", + " con.sql(sql_query_import_1)\n", + " con.sql(sql_query_import_2)" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
name
0parking_violation_codes
1parking_violations_2023
\n", + "
" + ], + "text/plain": [ + " name\n", + "0 parking_violation_codes\n", + "1 parking_violations_2023" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "sql_query = '''\n", + "show tables\n", + "'''\n", + "\n", + "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", + " display(con.sql(sql_query).df())\n", + "\n", + "# output\n", + "# ┌─────────────────────────┐\n", + "# │ name │\n", + "# │ varchar │\n", + "# ├─────────────────────────┤\n", + "# │ parking_violation_codes │\n", + "# │ parking_violations_2023 │\n", + "# └─────────────────────────┘" + ] } ], "metadata": { @@ -32,8 +170,16 @@ "name": "python3" }, "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", "name": "python", - "version": "3.9.6 (default, May 7 2023, 23:32:44) \n[Clang 14.0.3 (clang-1403.0.22.14.1)]" + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" }, "orig_nbformat": 4, "vscode": {