From 680111ce606862ef144fc18d825e0da41082633a Mon Sep 17 00:00:00 2001 From: Ross Masters Date: Wed, 27 Dec 2023 01:46:53 +0000 Subject: [PATCH 01/11] Documents using different column types, including string length --- docs/advanced/column-types.md | 152 ++++++++++++++++++ docs/tutorial/create-db-and-table.md | 2 +- docs_src/advanced/column_types/__init__.py | 0 docs_src/advanced/column_types/tutorial001.py | 80 +++++++++ docs_src/advanced/column_types/tutorial002.py | 76 +++++++++ docs_src/advanced/column_types/tutorial003.py | 21 +++ mkdocs.yml | 1 + pyproject.toml | 3 + .../test_column_types/__init__.py | 0 .../test_column_types/test_tutorial001.py | 44 +++++ .../test_column_types/test_tutorial002.py | 24 +++ 11 files changed, 402 insertions(+), 1 deletion(-) create mode 100644 docs/advanced/column-types.md create mode 100644 docs_src/advanced/column_types/__init__.py create mode 100644 docs_src/advanced/column_types/tutorial001.py create mode 100644 docs_src/advanced/column_types/tutorial002.py create mode 100644 docs_src/advanced/column_types/tutorial003.py create mode 100644 tests/test_advanced/test_column_types/__init__.py create mode 100644 tests/test_advanced/test_column_types/test_tutorial001.py create mode 100644 tests/test_advanced/test_column_types/test_tutorial002.py diff --git a/docs/advanced/column-types.md b/docs/advanced/column-types.md new file mode 100644 index 0000000000..5bec456184 --- /dev/null +++ b/docs/advanced/column-types.md @@ -0,0 +1,152 @@ +# Column Types + +In the tutorial, we stored scalar data types in our tables, like strings, numbers and timestamps. In practice, we often +work with more complicated types that need to be converted to a data type our database supports. + +## Customising String Field Lengths + +As we discussed in [`TEXT` or `VARCHAR`](../tutorial/create-db-and-table.md#text-or-varchar), a `str` field type will be +created as a `VARCHAR`, which has varying maximum-lengths depending on the database engine you are using. + +For cases where you know you only need to store a certain length of text, string field maximum length can be reduced +using the `max_length` validation argument to `Field()`: + +```Python hl_lines="11" +{!./docs_src/advanced/column_types/tutorial001.py[ln:1-12]!} +``` + +/// details | 👀 Full file preview + +```Python +{!./docs_src/advanced/column_types/tutorial001.py!} +``` + +/// + +/// warning + +Database engines behave differently when you attempt to store longer text than the character length of the `VARCHAR` +column. Notably: + +* SQLite does not enforce the length of a `VARCHAR`. It will happily store up to 500-million characters of text. +* MySQL will emit a warning, but will also truncate your text to fit the size of the `VARCHAR`. +* PostgreSQL will respond with an error code, and your query will not be executed. + +/// + +However if you need to store much longer strings than `VARCHAR` can allow, databases provide `TEXT` or `CLOB` +(**c**haracter **l**arge **ob**ject) column types. We can use these by specifying an SQLAlchemy column type to the field +with the `sa_type` keyword argument: + +```Python hl_lines="12" +{!./docs_src/advanced/column_types/tutorial001.py[ln:5-45]!} +``` + +/// tip + +`Text` also accepts a character length argument, which databases use to optimise the storage of a particular field. +Some databases support `TINYTEXT`, `SMALLTEXT`, `MEDIUMTEXT` and `LONGTEXT` column types - ranging from 255 bytes to +4 gigabytes. If you know the maximum length of data, specifying it like `Text(1000)` will automatically select the +best-suited, supported type for your database engine. + +/// + + +With this approach, we can use [any kind of SQLAlchemy type](https://docs.sqlalchemy.org/en/20/core/type_basics.html). +For example, if we were building a mapping application, we could store spatial information: + +```Python +{!./docs_src/advanced/column_types/tutorial002.py!} +``` + +## Supported Types + +Python types are mapped to column types as so: + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Python typeSQLAlchemy typeDatabase column types
strStringVARCHAR
intIntegerINTEGER
floatFloatFLOAT, REAL, DOUBLE
boolBooleanBOOL or TINYINT
datetime.datetimeDateTimeDATETIME, TIMESTAMP, DATE
datetime.dateDateDATE
datetime.timedeltaIntervalINTERVAL, INT
datetime.timeTimeTIME, DATETIME
bytesLargeBinaryBLOB, BYTEA
DecimalNumericDECIMAL, FLOAT
enum.EnumEnumENUM, VARCHAR
uuid.UUIDGUIDUUID, CHAR(32)
+ +In addition, the following types are stored as `VARCHAR`: + +* ipaddress.IPv4Address +* ipaddress.IPv4Network +* ipaddress.IPv6Address +* ipaddress.IPv6Network +* pathlib.Path + +### IP Addresses + +IP Addresses from the Python `ipaddress` module are stored as text. + +```Python hl_lines="1 11" +{!./docs_src/advanced/column_types/tutorial003.py[ln:1-13]!} +``` + +### Filesystem Paths + +Paths to files and directories using the Python `pathlib` module are stored as text. + +```Python hl_lines="5 11" +{!./docs_src/advanced/column_types/tutorial003.py[ln:1-15]!} +``` + +/// tip + +The stored value of a Path is the basic string value: `str(Path('../path/to/file'))`. If you need to store the full path +ensure you call `absolute()` on the path before setting it in your model. + +/// + +### UUIDs + +UUIDs from the Python `uuid` +module are stored as `UUID` types in supported databases (just PostgreSQL at the moment), otherwise as a `CHAR(32)`. + +```Python hl_lines="3 10" +{!./docs_src/advanced/column_types/tutorial003.py[ln:1-15]!} +``` + +## Custom Pydantic types + +As SQLModel is built on Pydantic, you can use any custom type as long as it would work in a Pydantic model. However, if +the type is not a subclass of [a type from the table above](#supported-types), you will need to specify an SQLAlchemy +type to use. diff --git a/docs/tutorial/create-db-and-table.md b/docs/tutorial/create-db-and-table.md index 0d8a9a21ce..5dfb757ae1 100644 --- a/docs/tutorial/create-db-and-table.md +++ b/docs/tutorial/create-db-and-table.md @@ -500,7 +500,7 @@ To make it easier to start using **SQLModel** right away independent of the data /// tip -You will learn how to change the maximum length of string columns later in the Advanced Tutorial - User Guide. +You can learn how to change the maximum length of string columns later in the [Advanced Tutorial - User Guide](../advanced/column-types.md){.internal-link target=_blank}. /// diff --git a/docs_src/advanced/column_types/__init__.py b/docs_src/advanced/column_types/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docs_src/advanced/column_types/tutorial001.py b/docs_src/advanced/column_types/tutorial001.py new file mode 100644 index 0000000000..a5270959db --- /dev/null +++ b/docs_src/advanced/column_types/tutorial001.py @@ -0,0 +1,80 @@ +from typing import Optional + +from sqlalchemy import Text +from sqlmodel import Field, Session, SQLModel, create_engine, select +from wonderwords import RandomWord + + +class Villian(SQLModel, table=True): + id: Optional[int] = Field(default=None, primary_key=True) + name: str = Field(index=True) + country_code: str = Field(max_length=2) + backstory: str = Field(sa_type=Text()) + + +sqlite_file_name = "database.db" +sqlite_url = f"sqlite:///{sqlite_file_name}" + +engine = create_engine(sqlite_url, echo=True) + + +def create_db_and_tables(): + SQLModel.metadata.create_all(engine) + + +def generate_backstory(words: int) -> str: + return " ".join(RandomWord().random_words(words, regex=r"\S+")) + + +def create_villains(): + villian_1 = Villian( + name="Green Gobbler", country_code="US", backstory=generate_backstory(500) + ) + villian_2 = Villian( + name="Arnim Zozza", country_code="DE", backstory=generate_backstory(500) + ) + villian_3 = Villian( + name="Low-key", country_code="AS", backstory=generate_backstory(500) + ) + + with Session(engine) as session: + session.add(villian_1) + session.add(villian_2) + session.add(villian_3) + + session.commit() + + +def count_words(sentence: str) -> int: + return sentence.count(" ") + 1 + + +def select_villians(): + with Session(engine) as session: + statement = select(Villian).where(Villian.name == "Green Gobbler") + results = session.exec(statement) + villian_1 = results.one() + print( + "Villian 1:", + {"name": villian_1.name, "country_code": villian_1.country_code}, + count_words(villian_1.backstory), + ) + + statement = select(Villian).where(Villian.name == "Low-key") + results = session.exec(statement) + villian_2 = results.one() + print( + "Villian 2:", + {"name": villian_2.name, "country_code": villian_2.country_code}, + count_words(villian_1.backstory), + ) + + +def main(): + create_db_and_tables() + create_villains() + select_villians() + + +if __name__ == "__main__": + main() diff --git a/docs_src/advanced/column_types/tutorial002.py b/docs_src/advanced/column_types/tutorial002.py new file mode 100644 index 0000000000..64cc2e6fdc --- /dev/null +++ b/docs_src/advanced/column_types/tutorial002.py @@ -0,0 +1,76 @@ +from datetime import UTC, datetime +from typing import TypedDict + +from sqlalchemy import PickleType +from sqlmodel import Field, Session, SQLModel, create_engine, select + + +class ModelOutput(TypedDict): + model_checkpoint: datetime + score: float + + +class ModelResult(SQLModel, table=True): + id: int = Field(default=..., primary_key=True) + output: ModelOutput = Field(sa_type=PickleType()) + + +sqlite_file_name = "database.db" +sqlite_url = f"sqlite:///{sqlite_file_name}" + +engine = create_engine(sqlite_url, echo=True) + + +def create_db_and_tables(): + SQLModel.metadata.create_all(engine) + + +def create_model_results(): + checkpoint = datetime.now(tz=UTC) + + result_1 = ModelResult( + output={ + "model_checkpoint": checkpoint, + "score": 0.9123, + } + ) + result_2 = ModelResult( + output={ + "model_checkpoint": checkpoint, + "score": 0.1294, + } + ) + result_3 = ModelResult( + output={ + "model_checkpoint": checkpoint, + "score": 0.4821, + } + ) + + with Session(engine) as session: + session.add(result_1) + session.add(result_2) + session.add(result_3) + + session.commit() + + +def get_average_score(): + with Session(engine) as session: + statement = select(ModelResult) + result = session.exec(statement) + model_results = result.all() + + scores = [model_result.output["score"] for model_result in model_results] + + print("Average score:", sum(scores) / len(scores)) + + +def main(): + create_db_and_tables() + create_model_results() + get_average_score() + + +if __name__ == "__main__": + main() diff --git a/docs_src/advanced/column_types/tutorial003.py b/docs_src/advanced/column_types/tutorial003.py new file mode 100644 index 0000000000..e20305a17e --- /dev/null +++ b/docs_src/advanced/column_types/tutorial003.py @@ -0,0 +1,21 @@ +import ipaddress +from datetime import UTC, datetime +from pathlib import Path +from uuid import UUID, uuid4 + +from sqlmodel import Field, SQLModel, create_engine + + +class Avatar(SQLModel, table=True): + id: UUID = Field(default_factory=uuid4, primary_key=True) + source_ip_address: ipaddress.IPv4Address + upload_location: Path + uploaded_at: datetime = Field(default=datetime.now(tz=UTC)) + + +sqlite_file_name = "database.db" +sqlite_url = f"sqlite:///{sqlite_file_name}" + +engine = create_engine(sqlite_url, echo=True) + +SQLModel.metadata.create_all(engine) diff --git a/mkdocs.yml b/mkdocs.yml index ce98f1524e..22a82b1657 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -98,6 +98,7 @@ nav: - Advanced User Guide: - advanced/index.md - advanced/decimal.md + - advanced/column-types.md - alternatives.md - help.md - contributing.md diff --git a/pyproject.toml b/pyproject.toml index 10d73793d2..ed17a3ffe8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -54,6 +54,9 @@ httpx = "0.24.1" dirty-equals = "^0.6.0" typer-cli = "^0.0.13" mkdocs-markdownextradata-plugin = ">=0.1.7,<0.3.0" +# For column type tests +wonderwords = "^2.2.0" +geoalchemy2 = "^0.14.3" [build-system] requires = ["poetry-core"] diff --git a/tests/test_advanced/test_column_types/__init__.py b/tests/test_advanced/test_column_types/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/test_advanced/test_column_types/test_tutorial001.py b/tests/test_advanced/test_column_types/test_tutorial001.py new file mode 100644 index 0000000000..e3439feee2 --- /dev/null +++ b/tests/test_advanced/test_column_types/test_tutorial001.py @@ -0,0 +1,44 @@ +from unittest.mock import patch + +from sqlmodel import create_engine + +from ...conftest import get_testing_print_function + +expected_calls = [ + [ + "Villian 1:", + { + "name": "Green Gobbler", + "country_code": "US", + }, + 500, + ], + [ + "Villian 2:", + { + "name": "Low-key", + "country_code": "AS", + }, + 500, + ], +] + + +def test_tutorial(clear_sqlmodel): + """ + Unfortunately, SQLite does not enforce varchar lengths, so we can't test an oversize case without spinning up a + database engine. + + """ + + from docs_src.advanced.column_types import tutorial001 as mod + + mod.sqlite_url = "sqlite://" + mod.engine = create_engine(mod.sqlite_url) + calls = [] + + new_print = get_testing_print_function(calls) + + with patch("builtins.print", new=new_print): + mod.main() + assert calls == expected_calls diff --git a/tests/test_advanced/test_column_types/test_tutorial002.py b/tests/test_advanced/test_column_types/test_tutorial002.py new file mode 100644 index 0000000000..eb19304088 --- /dev/null +++ b/tests/test_advanced/test_column_types/test_tutorial002.py @@ -0,0 +1,24 @@ +from unittest.mock import patch + +import pytest +from sqlmodel import create_engine + +from ...conftest import get_testing_print_function + +expected_calls = [ + ["Average score:", pytest.approx(0.5079, abs=0.0001)], +] + + +def test_tutorial(clear_sqlmodel): + from docs_src.advanced.column_types import tutorial002 as mod + + mod.sqlite_url = "sqlite://" + mod.engine = create_engine(mod.sqlite_url) + calls = [] + + new_print = get_testing_print_function(calls) + + with patch("builtins.print", new=new_print): + mod.main() + assert calls == expected_calls From 7432323d48ca2bc666c57f80621df3ddada411c2 Mon Sep 17 00:00:00 2001 From: Ross Masters Date: Wed, 27 Dec 2023 01:51:43 +0000 Subject: [PATCH 02/11] Add support for pydantic.EmailStr --- docs/advanced/column-types.md | 9 +++++---- docs_src/advanced/column_types/tutorial003.py | 2 ++ sqlmodel/main.py | 4 +++- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/docs/advanced/column-types.md b/docs/advanced/column-types.md index 5bec456184..f25526e81a 100644 --- a/docs/advanced/column-types.md +++ b/docs/advanced/column-types.md @@ -112,20 +112,21 @@ In addition, the following types are stored as `VARCHAR`: * ipaddress.IPv6Address * ipaddress.IPv6Network * pathlib.Path +* pydantic.EmailStr ### IP Addresses IP Addresses from the Python `ipaddress` module are stored as text. -```Python hl_lines="1 11" -{!./docs_src/advanced/column_types/tutorial003.py[ln:1-13]!} +```Python hl_lines="1 12" +{!./docs_src/advanced/column_types/tutorial003.py[ln:1-15]!} ``` ### Filesystem Paths Paths to files and directories using the Python `pathlib` module are stored as text. -```Python hl_lines="5 11" +```Python hl_lines="3 13" {!./docs_src/advanced/column_types/tutorial003.py[ln:1-15]!} ``` @@ -141,7 +142,7 @@ ensure you call `absolute()` on the path before setting it in your model. UUIDs from the Python `uuid` module are stored as `UUID` types in supported databases (just PostgreSQL at the moment), otherwise as a `CHAR(32)`. -```Python hl_lines="3 10" +```Python hl_lines="4 11" {!./docs_src/advanced/column_types/tutorial003.py[ln:1-15]!} ``` diff --git a/docs_src/advanced/column_types/tutorial003.py b/docs_src/advanced/column_types/tutorial003.py index e20305a17e..23b21dd738 100644 --- a/docs_src/advanced/column_types/tutorial003.py +++ b/docs_src/advanced/column_types/tutorial003.py @@ -3,6 +3,7 @@ from pathlib import Path from uuid import UUID, uuid4 +from pydantic import EmailStr from sqlmodel import Field, SQLModel, create_engine @@ -11,6 +12,7 @@ class Avatar(SQLModel, table=True): source_ip_address: ipaddress.IPv4Address upload_location: Path uploaded_at: datetime = Field(default=datetime.now(tz=UTC)) + author_email: EmailStr sqlite_file_name = "database.db" diff --git a/sqlmodel/main.py b/sqlmodel/main.py index 10064c7116..5e06b8630a 100644 --- a/sqlmodel/main.py +++ b/sqlmodel/main.py @@ -24,7 +24,7 @@ overload, ) -from pydantic import BaseModel +from pydantic import BaseModel, EmailStr from pydantic.fields import FieldInfo as PydanticFieldInfo from sqlalchemy import ( Boolean, @@ -602,6 +602,8 @@ def get_sqlalchemy_type(field: Any) -> Any: return AutoString if issubclass(type_, Path): return AutoString + if issubclass(type_, EmailStr): + return AutoString if issubclass(type_, uuid.UUID): return GUID raise ValueError(f"{type_} has no matching SQLAlchemy type") From 1ef61ce1ff78a9d7fb4a52b801b3a1ec57410056 Mon Sep 17 00:00:00 2001 From: Ross Masters Date: Wed, 27 Dec 2023 13:25:29 +0000 Subject: [PATCH 03/11] Add AutoString support for Pydantic network types --- docs/advanced/column-types.md | 5 ++ docs_src/advanced/column_types/tutorial003.py | 60 +++++++++++++++++-- pyproject.toml | 2 +- sqlmodel/main.py | 9 ++- .../test_column_types/test_tutorial003.py | 33 ++++++++++ 5 files changed, 102 insertions(+), 7 deletions(-) create mode 100644 tests/test_advanced/test_column_types/test_tutorial003.py diff --git a/docs/advanced/column-types.md b/docs/advanced/column-types.md index f25526e81a..7f969595fb 100644 --- a/docs/advanced/column-types.md +++ b/docs/advanced/column-types.md @@ -112,8 +112,13 @@ In addition, the following types are stored as `VARCHAR`: * ipaddress.IPv6Address * ipaddress.IPv6Network * pathlib.Path +* pydantic.networks.IPvAnyAddress +* pydantic.networks.IPvAnyInterface +* pydantic.networks.IPvAnyNetwork * pydantic.EmailStr +Note that while the column types for these are `VARCHAR`, values are not converted to and from strings. + ### IP Addresses IP Addresses from the Python `ipaddress` module are stored as text. diff --git a/docs_src/advanced/column_types/tutorial003.py b/docs_src/advanced/column_types/tutorial003.py index 23b21dd738..3d810e25cb 100644 --- a/docs_src/advanced/column_types/tutorial003.py +++ b/docs_src/advanced/column_types/tutorial003.py @@ -1,15 +1,14 @@ -import ipaddress from datetime import UTC, datetime from pathlib import Path from uuid import UUID, uuid4 -from pydantic import EmailStr -from sqlmodel import Field, SQLModel, create_engine +from pydantic import EmailStr, IPvAnyAddress +from sqlmodel import Field, Session, SQLModel, create_engine, select class Avatar(SQLModel, table=True): id: UUID = Field(default_factory=uuid4, primary_key=True) - source_ip_address: ipaddress.IPv4Address + source_ip_address: IPvAnyAddress upload_location: Path uploaded_at: datetime = Field(default=datetime.now(tz=UTC)) author_email: EmailStr @@ -20,4 +19,55 @@ class Avatar(SQLModel, table=True): engine = create_engine(sqlite_url, echo=True) -SQLModel.metadata.create_all(engine) + +def create_db_and_tables(): + SQLModel.metadata.create_all(engine) + + +def create_avatars(): + avatar_1 = Avatar( + source_ip_address="127.0.0.1", + upload_location="/uploads/1/123456789.jpg", + author_email="tiangolo@example.com", + ) + + avatar_2 = Avatar( + source_ip_address="192.168.0.1", + upload_location="/uploads/9/987654321.png", + author_email="rmasters@example.com", + ) + + with Session(engine) as session: + session.add(avatar_1) + session.add(avatar_2) + + session.commit() + + +def read_avatars(): + with Session(engine) as session: + statement = select(Avatar).where(Avatar.author_email == "tiangolo@example.com") + result = session.exec(statement) + avatar_1: Avatar = result.one() + + print( + "Avatar 1:", + { + "email": avatar_1.author_email, + "email_type": type(avatar_1.author_email), + "ip_address": avatar_1.source_ip_address, + "ip_address_type": type(avatar_1.source_ip_address), + "upload_location": avatar_1.upload_location, + "upload_location_type": type(avatar_1.upload_location), + }, + ) + + +def main(): + create_db_and_tables() + create_avatars() + read_avatars() + + +if __name__ == "__main__": + main() diff --git a/pyproject.toml b/pyproject.toml index ed17a3ffe8..98159cd908 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ typer-cli = "^0.0.13" mkdocs-markdownextradata-plugin = ">=0.1.7,<0.3.0" # For column type tests wonderwords = "^2.2.0" -geoalchemy2 = "^0.14.3" +pydantic = {extras = ["email"], version = ">=1.10.13,<3.0.0"} [build-system] requires = ["poetry-core"] diff --git a/sqlmodel/main.py b/sqlmodel/main.py index 5e06b8630a..4f86a239d4 100644 --- a/sqlmodel/main.py +++ b/sqlmodel/main.py @@ -24,8 +24,9 @@ overload, ) -from pydantic import BaseModel, EmailStr +from pydantic import BaseModel from pydantic.fields import FieldInfo as PydanticFieldInfo +from pydantic.networks import EmailStr, IPvAnyAddress, IPvAnyInterface, IPvAnyNetwork from sqlalchemy import ( Boolean, Column, @@ -600,6 +601,12 @@ def get_sqlalchemy_type(field: Any) -> Any: return AutoString if issubclass(type_, ipaddress.IPv6Network): return AutoString + if issubclass(type_, IPvAnyAddress): + return AutoString + if issubclass(type_, IPvAnyInterface): + return AutoString + if issubclass(type_, IPvAnyNetwork): + return AutoString if issubclass(type_, Path): return AutoString if issubclass(type_, EmailStr): diff --git a/tests/test_advanced/test_column_types/test_tutorial003.py b/tests/test_advanced/test_column_types/test_tutorial003.py new file mode 100644 index 0000000000..a9c02c142b --- /dev/null +++ b/tests/test_advanced/test_column_types/test_tutorial003.py @@ -0,0 +1,33 @@ +from unittest.mock import patch + +from sqlmodel import create_engine + +from ...conftest import get_testing_print_function + +expected_calls = [ + [ + "Avatar 1:", + { + "email": "tiangolo@example.com", + "email_type": str, + "ip_address": "127.0.0.1", + "ip_address_type": str, + "upload_location": "/uploads/1/123456789.jpg", + "upload_location_type": str, + }, + ], +] + + +def test_tutorial(clear_sqlmodel): + from docs_src.advanced.column_types import tutorial003 as mod + + mod.sqlite_url = "sqlite://" + mod.engine = create_engine(mod.sqlite_url) + calls = [] + + new_print = get_testing_print_function(calls) + + with patch("builtins.print", new=new_print): + mod.main() + assert calls == expected_calls From 46faa67ca0b3aff7c50cf01b7803554bd7b8426d Mon Sep 17 00:00:00 2001 From: Ross Masters Date: Wed, 27 Dec 2023 13:37:21 +0000 Subject: [PATCH 04/11] Fix code sample highlight lines --- docs/advanced/column-types.md | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/docs/advanced/column-types.md b/docs/advanced/column-types.md index 7f969595fb..94e33a6af3 100644 --- a/docs/advanced/column-types.md +++ b/docs/advanced/column-types.md @@ -123,7 +123,7 @@ Note that while the column types for these are `VARCHAR`, values are not convert IP Addresses from the Python `ipaddress` module are stored as text. -```Python hl_lines="1 12" +```Python hl_lines="5 11" {!./docs_src/advanced/column_types/tutorial003.py[ln:1-15]!} ``` @@ -131,7 +131,7 @@ IP Addresses from the Python `pathlib` module are stored as text. -```Python hl_lines="3 13" +```Python hl_lines="2 12" {!./docs_src/advanced/column_types/tutorial003.py[ln:1-15]!} ``` @@ -147,10 +147,20 @@ ensure you call `absolute()` on the path before setting it in your model. UUIDs from the Python `uuid` module are stored as `UUID` types in supported databases (just PostgreSQL at the moment), otherwise as a `CHAR(32)`. -```Python hl_lines="4 11" +```Python hl_lines="3 10" {!./docs_src/advanced/column_types/tutorial003.py[ln:1-15]!} ``` +### Email Addresses + +Email addresses using Pydantic's `EmailStr` type +are stored as strings. + +```Python hl_lines="5 14" +{!./docs_src/advanced/column_types/tutorial003.py[ln:1-15]!} +``` + + ## Custom Pydantic types As SQLModel is built on Pydantic, you can use any custom type as long as it would work in a Pydantic model. However, if From 9964b07c1b809dac4087e614e89015bf3bab51bc Mon Sep 17 00:00:00 2001 From: Ross Masters Date: Wed, 27 Dec 2023 13:44:33 +0000 Subject: [PATCH 05/11] Use templates for links, stop manually word wrapping --- docs/advanced/column-types.md | 41 +++++++++++------------------------ 1 file changed, 13 insertions(+), 28 deletions(-) diff --git a/docs/advanced/column-types.md b/docs/advanced/column-types.md index 94e33a6af3..718ce822fe 100644 --- a/docs/advanced/column-types.md +++ b/docs/advanced/column-types.md @@ -1,15 +1,12 @@ # Column Types -In the tutorial, we stored scalar data types in our tables, like strings, numbers and timestamps. In practice, we often -work with more complicated types that need to be converted to a data type our database supports. +In the tutorial, we stored scalar data types in our tables, like strings, numbers and timestamps. In practice, we often work with more complicated types that need to be converted to a data type our database supports. ## Customising String Field Lengths -As we discussed in [`TEXT` or `VARCHAR`](../tutorial/create-db-and-table.md#text-or-varchar), a `str` field type will be -created as a `VARCHAR`, which has varying maximum-lengths depending on the database engine you are using. +As we discussed in [`TEXT` or `VARCHAR`](../tutorial/create-db-and-table.md#text-or-varchar), a `str` field type will be created as a `VARCHAR`, which has varying maximum-lengths depending on the database engine you are using. -For cases where you know you only need to store a certain length of text, string field maximum length can be reduced -using the `max_length` validation argument to `Field()`: +For cases where you know you only need to store a certain length of text, string field maximum length can be reduced using the `max_length` validation argument to `Field()`: ```Python hl_lines="11" {!./docs_src/advanced/column_types/tutorial001.py[ln:1-12]!} @@ -25,8 +22,7 @@ using the `max_length` validation argument to `Field()`: /// warning -Database engines behave differently when you attempt to store longer text than the character length of the `VARCHAR` -column. Notably: +Database engines behave differently when you attempt to store longer text than the character length of the `VARCHAR` column. Notably: * SQLite does not enforce the length of a `VARCHAR`. It will happily store up to 500-million characters of text. * MySQL will emit a warning, but will also truncate your text to fit the size of the `VARCHAR`. @@ -34,9 +30,7 @@ column. Notably: /// -However if you need to store much longer strings than `VARCHAR` can allow, databases provide `TEXT` or `CLOB` -(**c**haracter **l**arge **ob**ject) column types. We can use these by specifying an SQLAlchemy column type to the field -with the `sa_type` keyword argument: +However if you need to store much longer strings than `VARCHAR` can allow, databases provide `TEXT` or `CLOB` (**c**haracter **l**arge **ob**ject) column types. We can use these by specifying an SQLAlchemy column type to the field with the `sa_type` keyword argument: ```Python hl_lines="12" {!./docs_src/advanced/column_types/tutorial001.py[ln:5-45]!} @@ -44,16 +38,12 @@ with the `sa_type` keyword argument: /// tip -`Text` also accepts a character length argument, which databases use to optimise the storage of a particular field. -Some databases support `TINYTEXT`, `SMALLTEXT`, `MEDIUMTEXT` and `LONGTEXT` column types - ranging from 255 bytes to -4 gigabytes. If you know the maximum length of data, specifying it like `Text(1000)` will automatically select the -best-suited, supported type for your database engine. +`Text` also accepts a character length argument, which databases use to optimise the storage of a particular field. Some databases support `TINYTEXT`, `SMALLTEXT`, `MEDIUMTEXT` and `LONGTEXT` column types - ranging from 255 bytes to 4 gigabytes. If you know the maximum length of data, specifying it like `Text(1000)` will automatically select the best-suited, supported type for your database engine. /// -With this approach, we can use [any kind of SQLAlchemy type](https://docs.sqlalchemy.org/en/20/core/type_basics.html). -For example, if we were building a mapping application, we could store spatial information: +With this approach, we can use [any kind of SQLAlchemy type](https://docs.sqlalchemy.org/en/20/core/type_basics.html). For example, we can store pickled objects in the database: ```Python {!./docs_src/advanced/column_types/tutorial002.py!} @@ -121,7 +111,7 @@ Note that while the column types for these are `VARCHAR`, values are not convert ### IP Addresses -IP Addresses from the Python `ipaddress` module are stored as text. +IP Addresses from the [Python `ipaddress` module](https://docs.python.org/3/library/ipaddress.html){.external-link target=_blank} are stored as text. ```Python hl_lines="5 11" {!./docs_src/advanced/column_types/tutorial003.py[ln:1-15]!} @@ -129,7 +119,7 @@ IP Addresses from the Python `pathlib` module are stored as text. +Paths to files and directories using the [Python `pathlib` module](https://docs.python.org/3/library/pathlib.html){.external-link target=_blank} are stored as text. ```Python hl_lines="2 12" {!./docs_src/advanced/column_types/tutorial003.py[ln:1-15]!} @@ -137,15 +127,13 @@ Paths to files and directories using the Python `uuid` -module are stored as `UUID` types in supported databases (just PostgreSQL at the moment), otherwise as a `CHAR(32)`. +UUIDs from the [Python `uuid` module](https://docs.python.org/3/library/uuid.html){.external-link target=_blank} are stored as native `UUID` types in supported databases (just PostgreSQL at the moment), otherwise as a `CHAR(32)`. ```Python hl_lines="3 10" {!./docs_src/advanced/column_types/tutorial003.py[ln:1-15]!} @@ -153,8 +141,7 @@ module are stored as `UUID` types in supported databases (just PostgreSQL at ### Email Addresses -Email addresses using Pydantic's `EmailStr` type -are stored as strings. +Email addresses using [Pydantic's `EmailStr` type](https://docs.pydantic.dev/latest/api/networks/#pydantic.networks.EmailStr){.external-link target=_blank} are stored as strings. ```Python hl_lines="5 14" {!./docs_src/advanced/column_types/tutorial003.py[ln:1-15]!} @@ -163,6 +150,4 @@ are stored as strings. ## Custom Pydantic types -As SQLModel is built on Pydantic, you can use any custom type as long as it would work in a Pydantic model. However, if -the type is not a subclass of [a type from the table above](#supported-types), you will need to specify an SQLAlchemy -type to use. +As SQLModel is built on Pydantic, you can use any custom type as long as it would work in a Pydantic model. However, if the type is not a subclass of [a type from the table above](#supported-types), you will need to specify an SQLAlchemy type to use. From 7aead7fc0b802dee0c5e9329c5f8551917857a18 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 27 Dec 2023 13:44:42 +0000 Subject: [PATCH 06/11] =?UTF-8?q?=F0=9F=8E=A8=20[pre-commit.ci]=20Auto=20f?= =?UTF-8?q?ormat=20from=20pre-commit.com=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/advanced/column-types.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced/column-types.md b/docs/advanced/column-types.md index 718ce822fe..46650e6683 100644 --- a/docs/advanced/column-types.md +++ b/docs/advanced/column-types.md @@ -107,7 +107,7 @@ In addition, the following types are stored as `VARCHAR`: * pydantic.networks.IPvAnyNetwork * pydantic.EmailStr -Note that while the column types for these are `VARCHAR`, values are not converted to and from strings. +Note that while the column types for these are `VARCHAR`, values are not converted to and from strings. ### IP Addresses From 94aff5c6f77392109e74fe36e094d3074d8f1441 Mon Sep 17 00:00:00 2001 From: Ross Masters Date: Sun, 31 Dec 2023 14:59:36 +0000 Subject: [PATCH 07/11] python 3.7 compatible datetime.UTC --- docs_src/advanced/column_types/tutorial002.py | 6 +++++- docs_src/advanced/column_types/tutorial003.py | 6 +++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docs_src/advanced/column_types/tutorial002.py b/docs_src/advanced/column_types/tutorial002.py index 64cc2e6fdc..b1ebbf5825 100644 --- a/docs_src/advanced/column_types/tutorial002.py +++ b/docs_src/advanced/column_types/tutorial002.py @@ -1,4 +1,8 @@ -from datetime import UTC, datetime +from datetime import datetime +try: + from datetime import UTC +except ImportError: + UTC = None from typing import TypedDict from sqlalchemy import PickleType diff --git a/docs_src/advanced/column_types/tutorial003.py b/docs_src/advanced/column_types/tutorial003.py index 3d810e25cb..0ae92aea13 100644 --- a/docs_src/advanced/column_types/tutorial003.py +++ b/docs_src/advanced/column_types/tutorial003.py @@ -1,4 +1,8 @@ -from datetime import UTC, datetime +from datetime import datetime +try: + from datetime import UTC +except ImportError: + UTC = None from pathlib import Path from uuid import UUID, uuid4 From bae753d33fc1265987691a0202331978b7d5d3e8 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Sun, 31 Dec 2023 14:59:46 +0000 Subject: [PATCH 08/11] =?UTF-8?q?=F0=9F=8E=A8=20[pre-commit.ci]=20Auto=20f?= =?UTF-8?q?ormat=20from=20pre-commit.com=20hooks?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs_src/advanced/column_types/tutorial002.py | 1 + docs_src/advanced/column_types/tutorial003.py | 1 + 2 files changed, 2 insertions(+) diff --git a/docs_src/advanced/column_types/tutorial002.py b/docs_src/advanced/column_types/tutorial002.py index b1ebbf5825..1438a9c273 100644 --- a/docs_src/advanced/column_types/tutorial002.py +++ b/docs_src/advanced/column_types/tutorial002.py @@ -1,4 +1,5 @@ from datetime import datetime + try: from datetime import UTC except ImportError: diff --git a/docs_src/advanced/column_types/tutorial003.py b/docs_src/advanced/column_types/tutorial003.py index 0ae92aea13..2e94d3270c 100644 --- a/docs_src/advanced/column_types/tutorial003.py +++ b/docs_src/advanced/column_types/tutorial003.py @@ -1,4 +1,5 @@ from datetime import datetime + try: from datetime import UTC except ImportError: From 8cd7c17b4c8aebff76c1bd5b75bca4707d0d2929 Mon Sep 17 00:00:00 2001 From: Ross Masters Date: Sun, 31 Dec 2023 15:31:20 +0000 Subject: [PATCH 09/11] Fix TypedDict on py3.7 --- docs_src/advanced/column_types/tutorial002.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs_src/advanced/column_types/tutorial002.py b/docs_src/advanced/column_types/tutorial002.py index 1438a9c273..bfc60b947b 100644 --- a/docs_src/advanced/column_types/tutorial002.py +++ b/docs_src/advanced/column_types/tutorial002.py @@ -4,10 +4,9 @@ from datetime import UTC except ImportError: UTC = None -from typing import TypedDict - from sqlalchemy import PickleType from sqlmodel import Field, Session, SQLModel, create_engine, select +from typing_extensions import TypedDict class ModelOutput(TypedDict): From 8c45261ee97d5a710506c3592d819a234c867fed Mon Sep 17 00:00:00 2001 From: Ross Masters Date: Sun, 31 Dec 2023 15:36:26 +0000 Subject: [PATCH 10/11] Fix Elipsis type isn't a NoneType --- docs_src/advanced/column_types/tutorial002.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs_src/advanced/column_types/tutorial002.py b/docs_src/advanced/column_types/tutorial002.py index bfc60b947b..2971ac3955 100644 --- a/docs_src/advanced/column_types/tutorial002.py +++ b/docs_src/advanced/column_types/tutorial002.py @@ -15,7 +15,7 @@ class ModelOutput(TypedDict): class ModelResult(SQLModel, table=True): - id: int = Field(default=..., primary_key=True) + id: int = Field(default=None, primary_key=True) output: ModelOutput = Field(sa_type=PickleType()) From 9a0c172c14da93d884d876653cd886b43048fd4b Mon Sep 17 00:00:00 2001 From: Ross Masters Date: Sun, 31 Dec 2023 16:48:19 +0000 Subject: [PATCH 11/11] Fix pydantic v2 tests --- tests/test_advanced/test_column_types/test_tutorial002.py | 3 ++- tests/test_advanced/test_column_types/test_tutorial003.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_advanced/test_column_types/test_tutorial002.py b/tests/test_advanced/test_column_types/test_tutorial002.py index eb19304088..90e90c707b 100644 --- a/tests/test_advanced/test_column_types/test_tutorial002.py +++ b/tests/test_advanced/test_column_types/test_tutorial002.py @@ -3,13 +3,14 @@ import pytest from sqlmodel import create_engine -from ...conftest import get_testing_print_function +from ...conftest import get_testing_print_function, needs_pydanticv2 expected_calls = [ ["Average score:", pytest.approx(0.5079, abs=0.0001)], ] +@needs_pydanticv2 def test_tutorial(clear_sqlmodel): from docs_src.advanced.column_types import tutorial002 as mod diff --git a/tests/test_advanced/test_column_types/test_tutorial003.py b/tests/test_advanced/test_column_types/test_tutorial003.py index a9c02c142b..c778946fce 100644 --- a/tests/test_advanced/test_column_types/test_tutorial003.py +++ b/tests/test_advanced/test_column_types/test_tutorial003.py @@ -2,7 +2,7 @@ from sqlmodel import create_engine -from ...conftest import get_testing_print_function +from ...conftest import get_testing_print_function, needs_pydanticv2 expected_calls = [ [ @@ -19,6 +19,7 @@ ] +@needs_pydanticv2 def test_tutorial(clear_sqlmodel): from docs_src.advanced.column_types import tutorial003 as mod