diff --git a/.vscode/settings.json b/.vscode/settings.json
index 236981053..ca20f9585 100644
--- a/.vscode/settings.json
+++ b/.vscode/settings.json
@@ -17,7 +17,6 @@
"files.autoSave": "afterDelay",
"screencastMode.onlyKeyboardShortcuts": true,
"terminal.integrated.fontSize": 18,
- "workbench.activityBar.visible": true,
"workbench.colorTheme": "Visual Studio Dark",
"workbench.fontAliasing": "antialiased",
"workbench.statusBar.visible": true
diff --git a/data/nyc_parking_violations.db b/data/nyc_parking_violations.db
index 19e587528..47376e57f 100644
Binary files a/data/nyc_parking_violations.db and b/data/nyc_parking_violations.db differ
diff --git a/data/nyc_parking_violations2.db b/data/nyc_parking_violations2.db
new file mode 100644
index 000000000..4965164bb
Binary files /dev/null and b/data/nyc_parking_violations2.db differ
diff --git a/data/prod_nyc_parking_violations.db b/data/prod_nyc_parking_violations.db
index 9a2bd4ad7..e9a88dd87 100644
Binary files a/data/prod_nyc_parking_violations.db and b/data/prod_nyc_parking_violations.db differ
diff --git a/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql b/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql
deleted file mode 100644
index 217a5f263..000000000
--- a/nyc_parking_violations/models/bronze/bronze_parking_violation_codes.sql
+++ /dev/null
@@ -1,7 +0,0 @@
-SELECT
- code AS violation_code,
- definition,
- manhattan_96th_st_below,
- all_other_areas
-FROM
- parking_violation_codes
diff --git a/nyc_parking_violations/models/bronze/bronze_parking_violations.sql b/nyc_parking_violations/models/bronze/bronze_parking_violations.sql
deleted file mode 100644
index 46525da2a..000000000
--- a/nyc_parking_violations/models/bronze/bronze_parking_violations.sql
+++ /dev/null
@@ -1,23 +0,0 @@
-SELECT
- summons_number,
- registration_state,
- plate_type,
- issue_date,
- violation_code,
- vehicle_body_type,
- vehicle_make,
- issuing_agency,
- vehicle_expiration_date,
- violation_location,
- violation_precinct,
- issuer_precinct,
- issuer_code,
- issuer_command,
- issuer_squad,
- violation_time,
- violation_county,
- violation_legal_code,
- vehicle_color,
- vehicle_year,
-FROM
- parking_violations_2023
\ No newline at end of file
diff --git a/nyc_parking_violations/models/example/ref_model.sql b/nyc_parking_violations/models/example/ref_model.sql
deleted file mode 100644
index 9bb4f51c6..000000000
--- a/nyc_parking_violations/models/example/ref_model.sql
+++ /dev/null
@@ -1,4 +0,0 @@
-SELECT
- COUNT(*)
-FROM
- {{ref('first_model')}}
diff --git a/nyc_parking_violations/.gitignore b/nyc_parking_violations2/.gitignore
similarity index 73%
rename from nyc_parking_violations/.gitignore
rename to nyc_parking_violations2/.gitignore
index 9a7943d2d..49f147cb9 100644
--- a/nyc_parking_violations/.gitignore
+++ b/nyc_parking_violations2/.gitignore
@@ -1,4 +1,4 @@
-.user.yml
+
target/
dbt_packages/
logs/
diff --git a/nyc_parking_violations2/.user.yml b/nyc_parking_violations2/.user.yml
new file mode 100644
index 000000000..f127973aa
--- /dev/null
+++ b/nyc_parking_violations2/.user.yml
@@ -0,0 +1 @@
+id: 23096e09-b919-420a-854b-199a5ab84b4f
diff --git a/nyc_parking_violations/README.md b/nyc_parking_violations2/README.md
similarity index 100%
rename from nyc_parking_violations/README.md
rename to nyc_parking_violations2/README.md
diff --git a/nyc_parking_violations/analyses/.gitkeep b/nyc_parking_violations2/analyses/.gitkeep
similarity index 100%
rename from nyc_parking_violations/analyses/.gitkeep
rename to nyc_parking_violations2/analyses/.gitkeep
diff --git a/nyc_parking_violations/dbt_project.yml b/nyc_parking_violations2/dbt_project.yml
similarity index 93%
rename from nyc_parking_violations/dbt_project.yml
rename to nyc_parking_violations2/dbt_project.yml
index ec48605d3..79b6cc6ae 100644
--- a/nyc_parking_violations/dbt_project.yml
+++ b/nyc_parking_violations2/dbt_project.yml
@@ -2,12 +2,12 @@
# Name your project! Project names should contain only lowercase characters
# and underscores. A good package name should reflect your organization's
# name or the intended use of these models
-name: 'nyc_parking_violations'
+name: 'nyc_parking_violations2'
version: '1.0.0'
config-version: 2
# This setting configures which "profile" dbt uses for this project.
-profile: 'nyc_parking_violations'
+profile: 'nyc_parking_violations2'
# These configurations specify where dbt should look for different types of files.
# The `model-paths` config, for example, states that models in this project can be
@@ -31,7 +31,7 @@ clean-targets: # directories to be removed by `dbt clean`
# directory as views. These settings can be overridden in the individual model
# files using the `{{ config(...) }}` macro.
models:
- nyc_parking_violations:
+ nyc_parking_violations2:
# Config indicated by + and applies to all files under models/example/
example:
+materialized: ephemeral
@@ -49,4 +49,4 @@ models:
gold:
+materialized: table
tests:
- +store_failures: true
+ +store_failures: true
\ No newline at end of file
diff --git a/nyc_parking_violations/macros/.gitkeep b/nyc_parking_violations2/macros/.gitkeep
similarity index 100%
rename from nyc_parking_violations/macros/.gitkeep
rename to nyc_parking_violations2/macros/.gitkeep
diff --git a/nyc_parking_violations2/models/bronze/bronze_parking_violation_codes.sql b/nyc_parking_violations2/models/bronze/bronze_parking_violation_codes.sql
new file mode 100644
index 000000000..9e054fc8d
--- /dev/null
+++ b/nyc_parking_violations2/models/bronze/bronze_parking_violation_codes.sql
@@ -0,0 +1,7 @@
+SELECT
+ code AS violation_code,
+ definition,
+ manhattan_96th_st_below,
+ all_other_areas
+FROM
+ parking_violation_codes
\ No newline at end of file
diff --git a/nyc_parking_violations2/models/bronze/bronze_parking_violations.sql b/nyc_parking_violations2/models/bronze/bronze_parking_violations.sql
new file mode 100644
index 000000000..12ec574d5
--- /dev/null
+++ b/nyc_parking_violations2/models/bronze/bronze_parking_violations.sql
@@ -0,0 +1,23 @@
+SELECT
+ summons_number,
+ registration_state,
+ plate_type,
+ issue_date,
+ violation_code,
+ vehicle_body_type,
+ vehicle_make,
+ issuing_agency,
+ vehicle_expiration_date,
+ violation_location,
+ violation_precinct,
+ issuer_precinct,
+ issuer_code,
+ issuer_command,
+ issuer_squad,
+ violation_time,
+ violation_county,
+ violation_legal_code,
+ vehicle_color,
+ vehicle_year,
+FROM
+ parking_violations_2023
\ No newline at end of file
diff --git a/nyc_parking_violations/models/docs/docs_blocks.md b/nyc_parking_violations2/models/docs/docs_block.md
similarity index 99%
rename from nyc_parking_violations/models/docs/docs_blocks.md
rename to nyc_parking_violations2/models/docs/docs_block.md
index cd410941a..aa403dff5 100644
--- a/nyc_parking_violations/models/docs/docs_blocks.md
+++ b/nyc_parking_violations2/models/docs/docs_block.md
@@ -104,4 +104,4 @@ The total number of tickets issued for a specific violation code.
{% docs total_revenue_usd %}
The total revenue accumulated from tickets, based on the violation code. This sum is represented in USD.
-{% enddocs %}
+{% enddocs %}
\ No newline at end of file
diff --git a/nyc_parking_violations/models/docs/schema.yml b/nyc_parking_violations2/models/docs/schema.yml
similarity index 99%
rename from nyc_parking_violations/models/docs/schema.yml
rename to nyc_parking_violations2/models/docs/schema.yml
index c1dfcb9fc..0524962a6 100644
--- a/nyc_parking_violations/models/docs/schema.yml
+++ b/nyc_parking_violations2/models/docs/schema.yml
@@ -185,4 +185,4 @@ models:
- name: registration_state
description: '{{ doc("registration_state") }}'
- name: ticket_count
- description: '{{ doc("ticket_count") }}'
+ description: '{{ doc("ticket_count") }}'
\ No newline at end of file
diff --git a/nyc_parking_violations/models/example/first_model.sql b/nyc_parking_violations2/models/example/first_model.sql
similarity index 100%
rename from nyc_parking_violations/models/example/first_model.sql
rename to nyc_parking_violations2/models/example/first_model.sql
diff --git a/nyc_parking_violations2/models/example/ref_model.sql b/nyc_parking_violations2/models/example/ref_model.sql
new file mode 100644
index 000000000..5624b1b7a
--- /dev/null
+++ b/nyc_parking_violations2/models/example/ref_model.sql
@@ -0,0 +1,4 @@
+SELECT
+ COUNT(*)
+FROM
+ {{ref('first_model')}}
\ No newline at end of file
diff --git a/nyc_parking_violations/models/gold/gold_ticket_metrics.sql b/nyc_parking_violations2/models/gold/gold_ticket_metrics.sql
similarity index 100%
rename from nyc_parking_violations/models/gold/gold_ticket_metrics.sql
rename to nyc_parking_violations2/models/gold/gold_ticket_metrics.sql
diff --git a/nyc_parking_violations/models/gold/gold_vehicles_metrics.sql b/nyc_parking_violations2/models/gold/gold_vehicle_metrics.sql
similarity index 100%
rename from nyc_parking_violations/models/gold/gold_vehicles_metrics.sql
rename to nyc_parking_violations2/models/gold/gold_vehicle_metrics.sql
diff --git a/nyc_parking_violations/models/silver/silver_parking_violation_codes.sql b/nyc_parking_violations2/models/silver/silver_parking_violation_codes.sql
similarity index 100%
rename from nyc_parking_violations/models/silver/silver_parking_violation_codes.sql
rename to nyc_parking_violations2/models/silver/silver_parking_violation_codes.sql
diff --git a/nyc_parking_violations/models/silver/silver_parking_violations.sql b/nyc_parking_violations2/models/silver/silver_parking_violations.sql
similarity index 100%
rename from nyc_parking_violations/models/silver/silver_parking_violations.sql
rename to nyc_parking_violations2/models/silver/silver_parking_violations.sql
diff --git a/nyc_parking_violations/models/silver/silver_violation_tickets.sql b/nyc_parking_violations2/models/silver/silver_violation_tickets.sql
similarity index 100%
rename from nyc_parking_violations/models/silver/silver_violation_tickets.sql
rename to nyc_parking_violations2/models/silver/silver_violation_tickets.sql
diff --git a/nyc_parking_violations/models/silver/silver_violation_vehicles.sql b/nyc_parking_violations2/models/silver/silver_violation_vehicles.sql
similarity index 100%
rename from nyc_parking_violations/models/silver/silver_violation_vehicles.sql
rename to nyc_parking_violations2/models/silver/silver_violation_vehicles.sql
diff --git a/nyc_parking_violations/profiles.yml b/nyc_parking_violations2/profiles.yml
similarity index 77%
rename from nyc_parking_violations/profiles.yml
rename to nyc_parking_violations2/profiles.yml
index cabe18c54..57c76bcf9 100644
--- a/nyc_parking_violations/profiles.yml
+++ b/nyc_parking_violations2/profiles.yml
@@ -1,4 +1,4 @@
-nyc_parking_violations:
+nyc_parking_violations2:
outputs:
dev:
type: duckdb
@@ -8,5 +8,5 @@ nyc_parking_violations:
# note that path is slightly different as GitHub actions
# start in the root directory and not in the
# nyc_parking_violations directory
- path: './data/prod_nyc_parking_violations.db'
+ path: '../data/prod_nyc_parking_violations.db'
target: dev
\ No newline at end of file
diff --git a/nyc_parking_violations/seeds/.gitkeep b/nyc_parking_violations2/seeds/.gitkeep
similarity index 100%
rename from nyc_parking_violations/seeds/.gitkeep
rename to nyc_parking_violations2/seeds/.gitkeep
diff --git a/nyc_parking_violations/snapshots/.gitkeep b/nyc_parking_violations2/snapshots/.gitkeep
similarity index 100%
rename from nyc_parking_violations/snapshots/.gitkeep
rename to nyc_parking_violations2/snapshots/.gitkeep
diff --git a/nyc_parking_violations/tests/.gitkeep b/nyc_parking_violations2/tests/.gitkeep
similarity index 100%
rename from nyc_parking_violations/tests/.gitkeep
rename to nyc_parking_violations2/tests/.gitkeep
diff --git a/nyc_parking_violations/tests/generic/generic_not_null.sql b/nyc_parking_violations2/tests/generic/generic_not_null.sql
similarity index 100%
rename from nyc_parking_violations/tests/generic/generic_not_null.sql
rename to nyc_parking_violations2/tests/generic/generic_not_null.sql
diff --git a/nyc_parking_violations/tests/violation_codes_revenue.sql b/nyc_parking_violations2/tests/violation_codes_revenue.sql
similarity index 85%
rename from nyc_parking_violations/tests/violation_codes_revenue.sql
rename to nyc_parking_violations2/tests/violation_codes_revenue.sql
index 091ddfdd4..9426a0ba9 100644
--- a/nyc_parking_violations/tests/violation_codes_revenue.sql
+++ b/nyc_parking_violations2/tests/violation_codes_revenue.sql
@@ -8,4 +8,4 @@ FROM
GROUP BY
violation_code
HAVING
- NOT(total_revenue_usd >= 1)
+ NOT(total_revenue_usd >= 1)
\ No newline at end of file
diff --git a/run_sql_queries_here.ipynb b/run_sql_queries_here.ipynb
index f7c54e6c9..48b53cfcd 100644
--- a/run_sql_queries_here.ipynb
+++ b/run_sql_queries_here.ipynb
@@ -2,7 +2,7 @@
"cells": [
{
"cell_type": "code",
- "execution_count": null,
+ "execution_count": 2,
"metadata": {},
"outputs": [],
"source": [
@@ -10,19 +10,195 @@
"import pandas as pd"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " violation_code | \n",
+ " total_revenue_usd | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " 41 | \n",
+ " 0.0 | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " violation_code total_revenue_usd\n",
+ "0 41 0.0"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "sql_query = '''\n",
+ "select * from \"nyc_parking_violations\".\"main_dbt_test__audit\".\"violation_codes_revenue\"\n",
+ "'''\n",
+ "\n",
+ "with duckdb.connect('data/nyc_parking_violations.db') as con:\n",
+ " display(con.sql(sql_query).df())"
+ ]
+ },
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
+ "source": [
+ "sql_query_import_1 = '''\n",
+ "CREATE OR REPLACE TABLE parking_violations_code AS \n",
+ "SELECT *\n",
+ "FROM read_csv_auto(\n",
+ " 'data/dof_parking_violation_codes.csv',\n",
+ " normalize_names=True\n",
+ " )\n",
+ "'''\n",
+ "\n",
+ "sql_query_import_2 = '''\n",
+ "CREATE OR REPLACE TABLE parking_violations_2023 AS \n",
+ "SELECT *\n",
+ "FROM read_csv_auto(\n",
+ " 'data/parking_violations_issued_fiscal_year_2023_sample.csv',\n",
+ " normalize_names=True\n",
+ " )\n",
+ "'''\n",
+ "\n",
+ "with duckdb.connect('data/prod_nyc_parking_violations2.db') as con:\n",
+ " con.sql(sql_query_import_1)\n",
+ " con.sql(sql_query_import_2)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " name | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " | 0 | \n",
+ " bronze_parking_violation_codes | \n",
+ "
\n",
+ " \n",
+ " | 1 | \n",
+ " bronze_parking_violations | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
+ " gold_ticket_metrics | \n",
+ "
\n",
+ " \n",
+ " | 3 | \n",
+ " gold_vehicles_metrics | \n",
+ "
\n",
+ " \n",
+ " | 4 | \n",
+ " parking_violation_codes | \n",
+ "
\n",
+ " \n",
+ " | 5 | \n",
+ " parking_violations_2023 | \n",
+ "
\n",
+ " \n",
+ " | 6 | \n",
+ " parking_violations_code | \n",
+ "
\n",
+ " \n",
+ " | 7 | \n",
+ " silver_violation_tickets | \n",
+ "
\n",
+ " \n",
+ " | 8 | \n",
+ " silver_violation_vehicles | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " name\n",
+ "0 bronze_parking_violation_codes\n",
+ "1 bronze_parking_violations\n",
+ "2 gold_ticket_metrics\n",
+ "3 gold_vehicles_metrics\n",
+ "4 parking_violation_codes\n",
+ "5 parking_violations_2023\n",
+ "6 parking_violations_code\n",
+ "7 silver_violation_tickets\n",
+ "8 silver_violation_vehicles"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
"source": [
"sql_query = '''\n",
"show tables\n",
"'''\n",
"\n",
- "with duckdb.connect('data/nyc_parking_violations.db') as con:\n",
+ "with duckdb.connect('data/prod_nyc_parking_violations.db') as con:\n",
" display(con.sql(sql_query).df())"
]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
}
],
"metadata": {