Skip to content

Commit eeb8042

Browse files
wangxiaoyingwangxiaoying
authored
Merge pull request #615 from sfu-db/py312
support python 3.12 A pandas destination dispatcher is added to release the GIL before parallel data fetching. It may need some refractor to be able to reuse the code in original connectorx::dispatcher
2 parents e75197a + d06bd52 commit eeb8042

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

51 files changed

+2877
-2218
lines changed

‎.github/workflows/ci.yml‎

Lines changed: 106 additions & 141 deletions
Original file line numberDiff line numberDiff line change
@@ -10,131 +10,109 @@ on:
1010
- main
1111

1212
jobs:
13-
# rust:
14-
# runs-on: ubuntu-latest
15-
# container: ubuntu:20.04
16-
# services:
17-
# # Label used to access the service container
18-
# postgres:
19-
# # Docker Hub image
20-
# image: postgres
21-
# env:
22-
# POSTGRES_PASSWORD: postgres
23-
# # Set health checks to wait until postgres has started
24-
# options: >-
25-
# --health-cmd pg_isready
26-
# --health-interval 10s
27-
# --health-timeout 5s
28-
# --health-retries 5
29-
# # mysql
30-
# mysql:
31-
# image: ghcr.io/wangxiaoying/mysql:latest
32-
# env:
33-
# MYSQL_DATABASE: mysql
34-
# MYSQL_ROOT_PASSWORD: mysql
35-
# LANG: C.UTF-8
36-
# ports:
37-
# - 3306:3306
38-
# options: >-
39-
# --health-cmd "mysqladmin ping"
40-
# --health-interval 10s
41-
# --health-timeout 10s
42-
# --health-retries 5
43-
# mssql:
44-
# image: mcr.microsoft.com/mssql/server:2019-latest
45-
# env:
46-
# ACCEPT_EULA: y
47-
# SA_PASSWORD: mssql!Password
48-
# ports:
49-
# - 1433:1433
50-
# options: >-
51-
# --health-cmd "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P \"$SA_PASSWORD\" -Q 'SELECT 1' || exit 1"
52-
# --health-interval 10s
53-
# --health-timeout 5s
54-
# --health-retries 20
55-
# steps:
56-
# - uses: actions/checkout@v4
13+
rust:
14+
runs-on: ubuntu-latest
15+
container: ubuntu:22.04
16+
services:
17+
postgres:
18+
image: postgres
19+
env:
20+
POSTGRES_PASSWORD: postgres
21+
# Set health checks to wait until postgres has started
22+
options: >-
23+
--health-cmd pg_isready
24+
--health-interval 10s
25+
--health-timeout 5s
26+
--health-retries 5
27+
mysql:
28+
image: ghcr.io/wangxiaoying/mysql:latest
29+
env:
30+
MYSQL_DATABASE: mysql
31+
MYSQL_ROOT_PASSWORD: mysql
32+
LANG: C.UTF-8
33+
ports:
34+
- 3306:3306
35+
options: >-
36+
--health-cmd "mysqladmin ping"
37+
--health-interval 10s
38+
--health-timeout 10s
39+
--health-retries 5
5740
58-
# - name: Install tools
59-
# run: |
60-
# apt-get update
61-
# apt-get install -y curl postgresql-client build-essential pkg-config libssl-dev git sqlite3 libsqlite3-dev mysql-client python3 python3-pip libicu66 libkrb5-dev libclang-dev
62-
# pip3 install mssql-cli
63-
# pip3 install cli-helpers==2.2.0
64-
# ln -s /usr/bin/python3 /usr/bin/python
65-
# echo "Cache Version ${{ secrets.CACHE_VERSION }}"
66-
# env:
67-
# DEBIAN_FRONTEND: noninteractive
41+
steps:
42+
- uses: actions/checkout@v4
6843

69-
# - name: Install Rust
70-
# uses: actions-rs/toolchain@v1
71-
# with:
72-
# toolchain: stable
73-
# components: rustfmt
74-
# default: true
44+
- name: Install tools
45+
run: |
46+
apt-get update
47+
apt-get install -y curl postgresql-client build-essential pkg-config libssl-dev git sqlite3 libsqlite3-dev mysql-client python3 python3-pip libkrb5-dev libclang-dev
48+
echo "Cache Version ${{ secrets.CACHE_VERSION }}"
49+
env:
50+
DEBIAN_FRONTEND: noninteractive
7551

76-
# - uses: actions/cache@v4
77-
# with:
78-
# path: |
79-
# ~/.cargo/bin/
80-
# ~/.cargo/registry/index/
81-
# ~/.cargo/registry/cache/
82-
# ~/.cargo/git/db/
83-
# target/
84-
# key: ${{ runner.os }}-cargo-${{ secrets.CACHE_VERSION }}-${{ hashFiles('**/Cargo.lock') }}
52+
- name: Install Rust
53+
uses: actions-rs/toolchain@v1
54+
with:
55+
toolchain: stable
56+
components: rustfmt
57+
default: true
8558

86-
# - name: Install other dependent tools
87-
# run: |
88-
# if [ ! -f "$HOME/.cargo/bin/just" ]; then curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to ~/.cargo/bin; fi
59+
- uses: actions/cache@v4
60+
with:
61+
path: |
62+
~/.cargo/bin/
63+
~/.cargo/registry/index/
64+
~/.cargo/registry/cache/
65+
~/.cargo/git/db/
66+
target/
67+
key: ${{ runner.os }}-cargo-${{ secrets.CACHE_VERSION }}-${{ hashFiles('**/Cargo.lock') }}
8968

90-
# - name: Seed the database
91-
# run: just seed-db
92-
# env:
93-
# POSTGRES_URL: "postgresql://postgres:postgres@postgres:5432/postgres"
94-
# SQLITE_URL: "sqlite:///tmp/test.db"
95-
# MYSQL_HOST: mysql
96-
# MYSQL_PORT: 3306
97-
# MYSQL_DB: mysql
98-
# MYSQL_USER: root
99-
# MYSQL_PASSWORD: mysql
100-
# MSSQL_HOST: mssql
101-
# MSSQL_PORT: 1433
102-
# MSSQL_DB: tempdb
103-
# MSSQL_USER: sa
104-
# MSSQL_PASSWORD: mssql!Password
69+
- uses: extractions/setup-just@v2
70+
71+
- name: Seed the database
72+
run: just seed-db
73+
env:
74+
POSTGRES_URL: "postgresql://postgres:postgres@postgres:5432/postgres"
75+
SQLITE_URL: "sqlite:///tmp/test.db"
76+
MYSQL_HOST: mysql
77+
MYSQL_PORT: 3306
78+
MYSQL_DB: mysql
79+
MYSQL_USER: root
80+
MYSQL_PASSWORD: mysql
10581

106-
# - name: Is the code formatted?
107-
# uses: actions-rs/cargo@v1
108-
# with:
109-
# command: fmt
110-
# args: --all -- --check -q
82+
- name: Is the code formatted?
83+
uses: actions-rs/cargo@v1
84+
with:
85+
command: fmt
86+
args: --all -- --check -q
11187

112-
# - name: Clippy linting
113-
# uses: actions-rs/cargo@v1
114-
# with:
115-
# command: clippy
116-
# args: --features all
88+
- name: Clippy linting
89+
uses: actions-rs/cargo@v1
90+
with:
91+
command: clippy
92+
args: --features all
11793

118-
# - name: Check each feature gate
119-
# run: just test-feature-gate
94+
# - name: Check each feature gate
95+
# run: just test-feature-gate
12096

121-
# - name: Run tests
122-
# run: cargo clean && just test
123-
# env:
124-
# POSTGRES_URL: "postgresql://postgres:postgres@postgres:5432/postgres"
125-
# SQLITE_URL: "sqlite:///tmp/test.db"
126-
# MYSQL_URL: "mysql://root:mysql@mysql:3306/mysql"
127-
# MSSQL_URL: "mssql://sa:mssql!Password@mssql:1433/tempdb"
97+
- name: Run tests
98+
run: cargo clean && just test-ci
99+
env:
100+
POSTGRES_URL: "postgresql://postgres:postgres@postgres:5432/postgres"
101+
SQLITE_URL: "sqlite:///tmp/test.db"
102+
MYSQL_URL: "mysql://root:mysql@mysql:3306/mysql"
128103

129-
# - name: Test build docs
130-
# uses: actions-rs/cargo@v1
131-
# with:
132-
# command: doc
133-
# args: --no-deps --features all
104+
- name: Test build docs
105+
uses: actions-rs/cargo@v1
106+
with:
107+
command: doc
108+
args: --no-deps --features all
134109

135110
python:
136111
runs-on: ubuntu-latest
137-
container: ubuntu:20.04
112+
container: ubuntu:22.04
113+
strategy:
114+
matrix:
115+
python-version: ["3.12"]
138116
services:
139117
# Label used to access the service container
140118
postgres:
@@ -162,32 +140,30 @@ jobs:
162140
--health-interval 10s
163141
--health-timeout 10s
164142
--health-retries 5
165-
mssql:
166-
image: mcr.microsoft.com/mssql/server:2019-latest
167-
env:
168-
ACCEPT_EULA: y
169-
SA_PASSWORD: mssql!Password
170-
ports:
171-
- 1433:1433
172-
options: >-
173-
--health-cmd "/opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P \"$SA_PASSWORD\" -Q 'SELECT 1' || exit 1"
174-
--health-interval 10s
175-
--health-timeout 5s
176-
--health-retries 20
177143
178144
steps:
179145
- uses: actions/checkout@v4
180146

147+
- uses: actions/setup-python@v5
148+
with:
149+
python-version: ${{ matrix.python-version }}
150+
151+
- name: Setup python env
152+
run: |
153+
echo "$Python3_ROOT_DIR/bin" >> $GITHUB_PATH
154+
echo $GITHUB_PATH
155+
181156
- name: Install tools
182157
run: |
183158
apt-get update
184-
apt-get install -y curl postgresql-client build-essential python3-dev python3-pip pkg-config libssl-dev git sqlite3 libsqlite3-dev mysql-client libmysqlclient-dev python3 python3-pip libicu66 libkrb5-dev libclang-dev
185-
pip3 install mssql-cli
186-
pip3 install cli-helpers==2.2.0
187-
ln -s /usr/bin/python3 /usr/bin/python
159+
apt-get install -y wget curl postgresql-client build-essential pkg-config libssl-dev git sqlite3 libsqlite3-dev mysql-client libmysqlclient-dev libkrb5-dev libclang-dev
188160
env:
189161
DEBIAN_FRONTEND: noninteractive
190162

163+
- uses: Gr1N/setup-poetry@v9
164+
165+
- uses: extractions/setup-just@v2
166+
191167
- name: Install Rust
192168
uses: actions-rs/toolchain@v1
193169
with:
@@ -208,12 +184,7 @@ jobs:
208184
~/.cargo/registry/cache/
209185
~/.cargo/git/db/
210186
target/
211-
key: ${{ runner.os }}-cargo-python-${{ secrets.CACHE_VERSION }}-${{ hashFiles('**/Cargo.lock') }}
212-
213-
- name: Install other dependent tools
214-
run: |
215-
pip3 install poetry
216-
if [ ! -f "$HOME/.cargo/bin/just" ]; then curl --proto '=https' --tlsv1.2 -sSf https://just.systems/install.sh | bash -s -- --to ~/.cargo/bin; fi
187+
key: ${{ runner.os }}-cargo-python-${{ matrix.python-version }}-${{ secrets.CACHE_VERSION }}-${{ hashFiles('**/Cargo.lock') }}
217188

218189
- name: Seed the database
219190
run: just seed-db
@@ -225,11 +196,6 @@ jobs:
225196
MYSQL_DB: mysql
226197
MYSQL_USER: root
227198
MYSQL_PASSWORD: mysql
228-
MSSQL_HOST: mssql
229-
MSSQL_PORT: 1433
230-
MSSQL_DB: tempdb
231-
MSSQL_USER: sa
232-
MSSQL_PASSWORD: mssql!Password
233199

234200
- name: Clippy linting
235201
uses: actions-rs/cargo@v1
@@ -241,7 +207,7 @@ jobs:
241207
uses: actions/cache@v4
242208
with:
243209
path: ~/.cache/pypoetry/virtualenvs
244-
key: ${{ runner.os }}-venv-${{ secrets.CACHE_VERSION }}-${{ hashFiles('connectorx-python/poetry.lock') }}
210+
key: ${{ runner.os }}-${{ matrix.python-version }}-venv-${{ secrets.CACHE_VERSION }}-${{ hashFiles('connectorx-python/poetry.lock') }}
245211

246212
- name: Install python dependencies
247213
run: just bootstrap-python
@@ -252,7 +218,6 @@ jobs:
252218
POSTGRES_URL: "postgresql://postgres:postgres@postgres:5432/postgres"
253219
SQLITE_URL: "sqlite:///tmp/test.db"
254220
MYSQL_URL: "mysql://root:mysql@mysql:3306/mysql"
255-
MSSQL_URL: "mssql://sa:mssql!Password@mssql:1433/tempdb"
256221
DB1: "postgresql://postgres:postgres@postgres:5432/postgres"
257222
DB2: "postgresql://postgres:postgres@postgres:5432/postgres"
258223
FED_CONFIG_PATH: ${{ github.workspace }}/.github/config

‎.github/workflows/release.yml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ jobs:
1212
container: quay.io/pypa/manylinux_2_28_x86_64
1313
strategy:
1414
matrix:
15-
python-version: [[38, "3.8"], [39, "3.9"], [310, "3.10"], [311, "3.11"]]
15+
python-version: [[39, "3.9"], [310, "3.10"], [311, "3.11"], [312, "3.12"]]
1616
steps:
1717
- uses: actions/checkout@v4
1818

@@ -162,7 +162,7 @@ jobs:
162162
strategy:
163163
matrix:
164164
os: ["windows-latest", "macos-latest"]
165-
python-version: ["3.8", "3.9", "3.10", "3.11"]
165+
python-version: ["3.9", "3.10", "3.11", "3.12"]
166166
include:
167167
- os: "macos-latest"
168168
features: "--features integrated-auth-gssapi"
@@ -226,7 +226,7 @@ jobs:
226226
runs-on: macos-latest
227227
strategy:
228228
matrix:
229-
python-version: ["3.8", "3.9", "3.10", "3.11"]
229+
python-version: ["3.9", "3.10", "3.11", "3.12"]
230230
steps:
231231
- uses: actions/checkout@v4
232232

@@ -290,7 +290,7 @@ jobs:
290290
needs: [win-and-mac, linux, apple-arm]
291291
strategy:
292292
matrix:
293-
python-version: ["3.8", "3.9", "3.10", "3.11"]
293+
python-version: ["3.9", "3.10", "3.11", "3.12"]
294294
os: [macos-latest, ubuntu-latest, windows-latest]
295295
steps:
296296
- uses: actions/checkout@v4

‎Justfile‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@ build-cpp-release +ARGS="":
1515
test +ARGS="":
1616
cargo test --features all {{ARGS}} -- --nocapture
1717

18+
test-ci:
19+
cargo test --features all --test test_postgres --test test_polars --test test_arrow --test test_mysql
20+
1821
test-feature-gate:
1922
cargo c --features src_postgres
2023
cargo c --features src_mysql
@@ -54,10 +57,10 @@ seed-db:
5457
psql $POSTGRES_URL -f scripts/postgres.sql
5558
sqlite3 ${SQLITE_URL#sqlite://} < scripts/sqlite.sql
5659
mysql --protocol tcp -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD $MYSQL_DB < scripts/mysql.sql
57-
mssql-cli -S$MSSQL_HOST -U$MSSQL_USER -P$MSSQL_PASSWORD -d$MSSQL_DB -i scripts/mssql.sql
5860

5961
# dbs not included in ci
6062
seed-db-more:
63+
mssql-cli -S$MSSQL_HOST -U$MSSQL_USER -P$MSSQL_PASSWORD -d$MSSQL_DB -i scripts/mssql.sql
6164
mysql --protocol tcp -h$CLICKHOUSE_HOST -P$CLICKHOUSE_PORT -u$CLICKHOUSE_USER -p$CLICKHOUSE_PASSWORD $CLICKHOUSE_DB < scripts/clickhouse.sql
6265
psql $REDSHIFT_URL -f scripts/redshift.sql
6366
ORACLE_URL_SCRIPT=`echo ${ORACLE_URL#oracle://} | sed "s/:/\//"`

0 commit comments

Comments
 (0)