From 66ab5138d2a75fed6920b6dadf99bd47083137ac Mon Sep 17 00:00:00 2001 From: arjun-dn <150115327+arjun-dn@users.noreply.github.com> Date: Wed, 16 Apr 2025 10:23:46 +0000 Subject: [PATCH 1/8] created workflow file. --- .github/workflows/run-dbt-prd.yml | 30 ++++++++ run_sql_queries_here.ipynb | 110 +++++++++++++++++++++++++++--- 2 files changed, 131 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/run-dbt-prd.yml diff --git a/.github/workflows/run-dbt-prd.yml b/.github/workflows/run-dbt-prd.yml new file mode 100644 index 000000000..82143f31d --- /dev/null +++ b/.github/workflows/run-dbt-prd.yml @@ -0,0 +1,30 @@ +# .github/workflows/run-dbt-prd.yml +name: run_dbt_prd + +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + dbt_setup: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: '3.9' + + - name: Install dependencies + run: pip install -r requirements.txt + + - name: Check dbt version + run: | + dbt --version + dbt debug diff --git a/run_sql_queries_here.ipynb b/run_sql_queries_here.ipynb index f7c54e6c9..e6b44fb77 100644 --- a/run_sql_queries_here.ipynb +++ b/run_sql_queries_here.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, "outputs": [], "source": [ @@ -12,9 +12,106 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
name
0bronze_parking_violation_codes
1bronze_parking_violations
2first_model
3gold_ticket_metrics
4gold_vehicles_metrics
5parking_violation_codes
6parking_violations_2023
7ref_model
8silver_parking_violation_codes
9silver_parking_violations
10silver_violation_tickets
11silver_violation_vehicles
\n", + "
" + ], + "text/plain": [ + " name\n", + "0 bronze_parking_violation_codes\n", + "1 bronze_parking_violations\n", + "2 first_model\n", + "3 gold_ticket_metrics\n", + "4 gold_vehicles_metrics\n", + "5 parking_violation_codes\n", + "6 parking_violations_2023\n", + "7 ref_model\n", + "8 silver_parking_violation_codes\n", + "9 silver_parking_violations\n", + "10 silver_violation_tickets\n", + "11 silver_violation_vehicles" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "sql_query = '''\n", "show tables\n", @@ -43,12 +140,7 @@ "pygments_lexer": "ipython3", "version": "3.10.12" }, - "orig_nbformat": 4, - "vscode": { - "interpreter": { - "hash": "31f2aee4e71d21fbe5cf8b01ff0e069b9275f58929596ceb00d14d90e3e16cd6" - } - } + "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 From e8b8165948957c77e093755bd7554c0d1649e5c6 Mon Sep 17 00:00:00 2001 From: arjun-dn <150115327+arjun-dn@users.noreply.github.com> Date: Wed, 16 Apr 2025 18:57:51 +0530 Subject: [PATCH 2/8] Update run-dbt-prd.yml --- .github/workflows/run-dbt-prd.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/run-dbt-prd.yml b/.github/workflows/run-dbt-prd.yml index 82143f31d..a5644a8b8 100644 --- a/.github/workflows/run-dbt-prd.yml +++ b/.github/workflows/run-dbt-prd.yml @@ -8,6 +8,9 @@ on: pull_request: branches: - main +env: + DBT_PROFILES_DIR: ./nyc_parking_violations + DBT_PROJECT_DIR: ./nyc_parking_violations jobs: dbt_setup: From 270037551f0900e59b4e3bdc54cbdaaf483620a2 Mon Sep 17 00:00:00 2001 From: arjun-dn <150115327+arjun-dn@users.noreply.github.com> Date: Wed, 16 Apr 2025 19:11:39 +0530 Subject: [PATCH 3/8] Update run-dbt-prd.yml --- .github/workflows/run-dbt-prd.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/run-dbt-prd.yml b/.github/workflows/run-dbt-prd.yml index a5644a8b8..1cc4b4f3d 100644 --- a/.github/workflows/run-dbt-prd.yml +++ b/.github/workflows/run-dbt-prd.yml @@ -31,3 +31,4 @@ jobs: run: | dbt --version dbt debug + dbt --help From 8ac1cd14ffc676346e34eab974dbb1d82274ecbe Mon Sep 17 00:00:00 2001 From: arjun-dn <150115327+arjun-dn@users.noreply.github.com> Date: Wed, 16 Apr 2025 20:21:40 +0530 Subject: [PATCH 4/8] parent path added ../ --- nyc_parking_violations/profiles.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nyc_parking_violations/profiles.yml b/nyc_parking_violations/profiles.yml index cabe18c54..3d89d2f1e 100644 --- a/nyc_parking_violations/profiles.yml +++ b/nyc_parking_violations/profiles.yml @@ -8,5 +8,5 @@ nyc_parking_violations: # note that path is slightly different as GitHub actions # start in the root directory and not in the # nyc_parking_violations directory - path: './data/prod_nyc_parking_violations.db' + path: '../data/prod_nyc_parking_violations.db' target: dev \ No newline at end of file From 8b6c8f9ed2b12d9ae18d455c281ce411136549b5 Mon Sep 17 00:00:00 2001 From: arjun-dn <150115327+arjun-dn@users.noreply.github.com> Date: Wed, 16 Apr 2025 20:21:57 +0530 Subject: [PATCH 5/8] - --- .vscode/settings.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 236981053..ca20f9585 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -17,7 +17,6 @@ "files.autoSave": "afterDelay", "screencastMode.onlyKeyboardShortcuts": true, "terminal.integrated.fontSize": 18, - "workbench.activityBar.visible": true, "workbench.colorTheme": "Visual Studio Dark", "workbench.fontAliasing": "antialiased", "workbench.statusBar.visible": true From 04510687a469d5650fc914de62f158b982fbd3e7 Mon Sep 17 00:00:00 2001 From: arjun-dn <150115327+arjun-dn@users.noreply.github.com> Date: Thu, 17 Apr 2025 16:32:13 +0530 Subject: [PATCH 6/8] added space. --- run_sql_queries_here.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/run_sql_queries_here.ipynb b/run_sql_queries_here.ipynb index e6b44fb77..51d0854a3 100644 --- a/run_sql_queries_here.ipynb +++ b/run_sql_queries_here.ipynb @@ -12,7 +12,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [ { @@ -115,6 +115,7 @@ "source": [ "sql_query = '''\n", "show tables\n", + "\n", "'''\n", "\n", "with duckdb.connect('data/nyc_parking_violations.db') as con:\n", From e5a7861b5e45600116da707de68e581e4364c082 Mon Sep 17 00:00:00 2001 From: arjun-dn <150115327+arjun-dn@users.noreply.github.com> Date: Thu, 17 Apr 2025 16:38:53 +0530 Subject: [PATCH 7/8] Update run-dbt-prd.yml --- .github/workflows/run-dbt-prd.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/run-dbt-prd.yml b/.github/workflows/run-dbt-prd.yml index 1cc4b4f3d..3d34121ad 100644 --- a/.github/workflows/run-dbt-prd.yml +++ b/.github/workflows/run-dbt-prd.yml @@ -31,4 +31,9 @@ jobs: run: | dbt --version dbt debug - dbt --help + + - name: Check compile & test + run: | + dbt compile + dbt run + dbt test From 70fe170db6883f9cd70bc53c5c74ae3733beb785 Mon Sep 17 00:00:00 2001 From: arjun-dn <150115327+arjun-dn@users.noreply.github.com> Date: Thu, 17 Apr 2025 16:46:24 +0530 Subject: [PATCH 8/8] Update profiles.yml --- nyc_parking_violations/profiles.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nyc_parking_violations/profiles.yml b/nyc_parking_violations/profiles.yml index 3d89d2f1e..1e7b98b40 100644 --- a/nyc_parking_violations/profiles.yml +++ b/nyc_parking_violations/profiles.yml @@ -2,11 +2,11 @@ nyc_parking_violations: outputs: dev: type: duckdb - path: '../data/nyc_parking_violations.db' + path: './data/nyc_parking_violations.db' prod: type: duckdb # note that path is slightly different as GitHub actions # start in the root directory and not in the # nyc_parking_violations directory path: '../data/prod_nyc_parking_violations.db' - target: dev \ No newline at end of file + target: dev