test_1

.vscode/settings.json

@@ -17,7 +17,6 @@
   "files.autoSave": "afterDelay",
   "screencastMode.onlyKeyboardShortcuts": true,
   "terminal.integrated.fontSize": 18,
-  "workbench.activityBar.visible": true,
   "workbench.colorTheme": "Visual Studio Dark",
   "workbench.fontAliasing": "antialiased",
   "workbench.statusBar.visible": true

nyc_parking_violations/.gitignore

@@ -1,4 +1,4 @@
-.user.yml
+
 target/
 dbt_packages/
 logs/

nyc_parking_violations/.user.yml

@@ -0,0 +1 @@
+id: f536095e-1e73-4858-b08e-1ceabadf31b1

nyc_parking_violations/dbt_project.yml

@@ -41,12 +41,12 @@ models:
       silver_parking_violation_codes:
         +materialized: ephemeral
       silver_parking_violations:
-        +materialized: ephemeral
+        +materialized: ephemeral   
       silver_violation_tickets:
-        +materialized: view
+        +materialized: view                
       silver_violation_vehicles:
         +materialized: view
     gold:
-      +materialized: table
+      +materialized: table                
 tests:
-  +store_failures: true
+  +store_failures: true

nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql

@@ -1,7 +1,7 @@
-SELECT
-    code AS violation_code,
-    definition,
-    manhattan_96th_st_below,
-    all_other_areas
-FROM
-    parking_violation_codes
+SELECT 
+  code as violation_code
+ ,definition
+ ,manhattan_96th_st_below
+ ,all_other_areas
+FROM 
+  parking_violation_codes

nyc_parking_violations/models/bronze/bronze_parking_violations.sql

@@ -20,4 +20,4 @@ SELECT
     vehicle_color,
     vehicle_year,
 FROM
-    parking_violations_2023
+    parking_violation_2023

nyc_parking_violations/models/docs/docs_blocks.md → nyc_parking_violations/models/docs/docs_block.md

@@ -104,4 +104,4 @@ The total number of tickets issued for a specific violation code.
 
 {% docs total_revenue_usd %}
 The total revenue accumulated from tickets, based on the violation code. This sum is represented in USD.
-{% enddocs %}
+{% enddocs %}

nyc_parking_violations/models/docs/schema.yml

@@ -179,10 +179,10 @@ models:
       - name: total_revenue_usd
         description: '{{ doc("total_revenue_usd") }}'
 
-  - name: gold_vehicles_metrics
+  - name: gold_vehicle_metrics
     description: "Aggregated metrics detailing the number of tickets per vehicle, identified by the plate ID."
     columns:
       - name: registration_state
         description: '{{ doc("registration_state") }}'
       - name: ticket_count
-        description: '{{ doc("ticket_count") }}'
+        description: '{{ doc("ticket_count") }}'

nyc_parking_violations/models/example/ref_model.sql

@@ -1,4 +1,2 @@
-SELECT
-  COUNT(*)
-FROM
-  {{ref('first_model')}}
+SELECT COUNT(*)
+FROM {{ref('first_model')}}

nyc_parking_violations/profiles.yml

@@ -8,5 +8,5 @@ nyc_parking_violations:
      # note that path is slightly different as GitHub actions
      # start in the root directory and not in the
      # nyc_parking_violations directory
-     path: './data/prod_nyc_parking_violations.db'  
+     path: '../data/prod_nyc_parking_violations.db'  
   target: dev

nyc_parking_violations/tests/violation_codes_revenue.sql

@@ -1,11 +1,9 @@
-{{ config(severity = 'warn') }}
-
-SELECT
-    violation_code,
-    SUM(fee_usd) AS total_revenue_usd
-FROM
-    {{ref('silver_parking_violation_codes')}}
-GROUP BY
-    violation_code
-HAVING
-    NOT(total_revenue_usd >= 1)
+{{config(severity = 'warn')}}
+SELECT violation_code
+      ,sum(fee_usd) as total_revenue_usd
+FROM 
+   {{ref('silver_parking_violation_codes')}}
+GROUP BY 
+   violation_code
+HAVING 
+   NOT((total_revenue_usd >=1))

run_sql_queries_here.ipynb

@@ -2,7 +2,7 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -12,17 +12,224 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 3,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>violation_code</th>\n",
+       "      <th>total_revenue_usd</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>41</td>\n",
+       "      <td>0.0</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "   violation_code  total_revenue_usd\n",
+       "0              41                0.0"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
    "source": [
     "sql_query = '''\n",
-    "show tables\n",
+    "  select * from \"nyc_parking_violations\".\"main_dbt_test__audit\".\"violation_codes_revenue\"\n",
+    " \n",
     "'''\n",
     "\n",
     "with duckdb.connect('data/nyc_parking_violations.db') as con:\n",
     "    display(con.sql(sql_query).df())"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sql_query_import_1= '''\n",
+    "CREATE OR REPLACE TABLE parking_violation_codes AS\n",
+    "SELECT * \n",
+    "FROM read_csv_auto(\n",
+    "  'data/dof_parking_violation_codes.csv',\n",
+    "  normalize_names = True\n",
+    ")\n",
+    "'''\n",
+    "\n",
+    "sql_query_import_2= '''\n",
+    "CREATE OR REPLACE TABLE parking_violation_2023 AS\n",
+    "SELECT * \n",
+    "FROM read_csv_auto(\n",
+    "  'data/parking_violations_issued_fiscal_year_2023_sample.csv',\n",
+    "  normalize_names = True\n",
+    ")\n",
+    "'''\n",
+    "\n",
+    "with duckdb.connect('data/nyc_parking_violations.db') as con:\n",
+    "  con.sql(sql_query_import_1)\n",
+    "  con.sql(sql_query_import_2)\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "sql_query_import_1= '''\n",
+    "CREATE OR REPLACE TABLE parking_violation_codes AS\n",
+    "SELECT * \n",
+    "FROM read_csv_auto(\n",
+    "  'data/dof_parking_violation_codes.csv',\n",
+    "  normalize_names = True\n",
+    ")\n",
+    "'''\n",
+    "\n",
+    "sql_query_import_2= '''\n",
+    "CREATE OR REPLACE TABLE parking_violation_2023 AS\n",
+    "SELECT * \n",
+    "FROM read_csv_auto(\n",
+    "  'data/parking_violations_issued_fiscal_year_2023_sample.csv',\n",
+    "  normalize_names = True\n",
+    ")\n",
+    "'''\n",
+    "\n",
+    "with duckdb.connect('data/prod_nyc_parking_violations.db') as con:\n",
+    "  con.sql(sql_query_import_1)\n",
+    "  con.sql(sql_query_import_2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div>\n",
+       "<style scoped>\n",
+       "    .dataframe tbody tr th:only-of-type {\n",
+       "        vertical-align: middle;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe tbody tr th {\n",
+       "        vertical-align: top;\n",
+       "    }\n",
+       "\n",
+       "    .dataframe thead th {\n",
+       "        text-align: right;\n",
+       "    }\n",
+       "</style>\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>name</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>bronze_parking_violation_codes</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>bronze_parking_violations</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>gold_ticket_metrics</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>gold_vehicles_metrics</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>parking_violation_2023</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>parking_violation_codes</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>parking_violations_2023</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>silver_violation_tickets</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>silver_violation_vehicles</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "                             name\n",
+       "0  bronze_parking_violation_codes\n",
+       "1       bronze_parking_violations\n",
+       "2             gold_ticket_metrics\n",
+       "3           gold_vehicles_metrics\n",
+       "4          parking_violation_2023\n",
+       "5         parking_violation_codes\n",
+       "6         parking_violations_2023\n",
+       "7        silver_violation_tickets\n",
+       "8       silver_violation_vehicles"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "sql_query = '''\n",
+    "  show tables\n",
+    " \n",
+    "'''\n",
+    "\n",
+    "with duckdb.connect('data/prod_nyc_parking_violations.db') as con:\n",
+    "    display(con.sql(sql_query).df())"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {