Initial work on DuckDB

.github/workflows/std-libs-labels.yml

@@ -89,6 +89,31 @@ jobs:
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           labels: -libs-API-change-Database
+  stdlib-api-check-DuckDB-linux-amd64:
+    name: DuckDB-change-labels
+    runs-on:
+      - ubuntu-latest
+    steps:
+      - name: Checking out the repository
+        uses: actions/checkout@v4
+        with:
+          clean: false
+          fetch-depth: 2
+      - id: DuckDB-changed-files
+        name: DuckDB-changed-files
+        uses: step-security/changed-files@v45
+        with:
+          files: distribution/lib/Standard/DuckDB/**/docs/api/**/**.md
+      - name: List all changed files in DuckDB
+        run: "\n        if [[ \"${{ steps.DuckDB-changed-files.outputs.any_changed }}\" == \"true\" ]]; then\n            echo \"Files changed:\"\n        fi\n        for file in ${ALL_CHANGED_FILES}; do\n            echo \"$file\"\n        done\n        "
+        env:
+          ALL_CHANGED_FILES: ${{ steps.DuckDB-changed-files.outputs.all_changed_files }}
+      - if: steps.DuckDB-changed-files.outputs.any_changed == 'true'
+        name: Append -libs-API-change-DuckDB label
+        uses: actions-ecosystem/action-add-labels@v1
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          labels: -libs-API-change-DuckDB
   stdlib-api-check-Generic_JDBC-linux-amd64:
     name: Generic_JDBC-change-labels
     runs-on:

.gitignore

@@ -115,6 +115,9 @@ bench-report*.xml
 *.pdb
 *.so
 *.jar
+*.so_linux_amd64
+*.so_osx_universal
+*.so_windows_amd64
 
 
 #########

bazel_scripts/stdlibs.bzl

@@ -6,6 +6,7 @@ STDLIB_NAMES = [
     "AWS",
     "Base",
     "Database",
+    "DuckDB",
     "Examples",
     "Generic_JDBC",
     "Geo",

build.sbt

@@ -146,6 +146,10 @@ GatherLicenses.distributions := Seq(
   makeStdLibDistribution(
     "Saas",
     Distribution.sbtProjects(`std-saas`)
+  ),
+  makeStdLibDistribution(
+    "DuckDB",
+    Distribution.sbtProjects(`std-duckdb`)
   )
 )
 
@@ -400,6 +404,7 @@ lazy val enso = (project in file("."))
     `std-table`,
     `std-tableau`,
     `std-saas`,
+    `std-duckdb`,
     `sqlite-wrapper`,
     `syntax-rust-definition`,
     `tableau-wrapper`,
@@ -2852,6 +2857,7 @@ lazy val runtime = (project in file("engine/runtime"))
       .dependsOn(`std-microsoft` / Compile / packageBin)
       .dependsOn(`std-tableau` / Compile / packageBin)
       .dependsOn(`std-saas` / Compile / packageBin)
+      .dependsOn(`std-duckdb` / Compile / packageBin)
       .value
   )
   .dependsOn(`common-polyglot-core-utils`)
@@ -3931,6 +3937,9 @@ lazy val `engine-runner` = project
             .listFiles("*.jar")
             .map(_.getAbsolutePath()) ++
           `std-tableau-polyglot-root`
+            .listFiles("*.jar")
+            .map(_.getAbsolutePath()) ++
+          `std-duckdb-polyglot-root`
             .listFiles("*.jar")
             .map(_.getAbsolutePath()) ++ (if (
                                             GraalVM.EnsoLauncher.disableMicrosoft
@@ -4063,7 +4072,8 @@ lazy val `engine-runner` = project
               "com.tableau.hyperapi",
               // See https://github.com/HarrDevY/native-register-bouncy-castle
               "org.bouncycastle.jcajce.provider.drbg.DRBG$Default",
-              "org.bouncycastle.jcajce.provider.drbg.DRBG$NonceAndIV"
+              "org.bouncycastle.jcajce.provider.drbg.DRBG$NonceAndIV",
+              "org.duckdb"
             ),
             initializeAtBuildtime = NativeImage.defaultBuildTimeInitClasses ++
               Seq(
@@ -5025,6 +5035,10 @@ val `std-tableau-native-libs` =
   stdLibComponentRoot("Tableau") / "polyglot" / "lib"
 val `std-saas-polyglot-root` =
   stdLibComponentRoot("Saas") / "polyglot" / "java"
+val `std-duckdb-polyglot-root` =
+  stdLibComponentRoot("DuckDB") / "polyglot" / "java"
+val `std-duckdb-native-libs` =
+  stdLibComponentRoot("DuckDB") / "polyglot" / "lib"
 
 lazy val `std-base` = project
   .in(file("std-bits") / "base")
@@ -5518,6 +5532,31 @@ lazy val `sqlite-wrapper` = project
     )
   )
 
+lazy val `duckdb-wrapper` = project
+  .in(file("lib/java/duckdb-wrapper"))
+  .enablePlugins(JarExtractPlugin)
+  .settings(
+    frgaalJavaCompilerSetting,
+    autoScalaLibrary := false,
+    libraryDependencies ++= Seq(
+      "org.duckdb" % "duckdb_jdbc" % duckdbVersion
+    ),
+    inputJar := "org.duckdb" % "duckdb_jdbc" % duckdbVersion,
+    version := "0.1",
+    jarExtractor := JarExtractor(
+      "libduckdb_java.so_linux_amd64"   -> PolyglotLib(LinuxAMD64),
+      "libduckdb_java.so_osx_universal" -> PolyglotLib(MacOSArm64),
+      "libduckdb_java.so_osx_universal" -> PolyglotLib(MacOSAMD64),
+      "libduckdb_java.so_windows_amd64" -> PolyglotLib(WindowsAMD64),
+      "META-INF/**"                     -> CopyToOutputJar,
+      "org/**/*.class"                  -> CopyToOutputJar
+    ),
+    inputJarResolved := assembly.value,
+    assemblyMergeStrategy := { case _ =>
+      MergeStrategy.preferProject
+    }
+  )
+
 lazy val `std-image` = project
   .in(file("std-bits") / "image")
   .settings(
@@ -6045,6 +6084,52 @@ lazy val `std-saas` = project
   .dependsOn(`std-base` % "provided")
   .dependsOn(`std-table` % "provided")
 
+lazy val `std-duckdb` = project
+  .in(file("std-bits") / "duckdb")
+  .settings(
+    frgaalJavaCompilerSetting,
+    autoScalaLibrary := false,
+    Compile / compile / compileInputs := (Compile / compile / compileInputs)
+      .dependsOn(SPIHelpers.ensureSPIConsistency)
+      .value,
+    Compile / packageBin / artifactPath :=
+      `std-duckdb-polyglot-root` / "std-duckdb.jar",
+    libraryDependencies ++= Seq(
+      "org.duckdb" % "duckdb_jdbc" % duckdbVersion % "provided"
+    ),
+    Compile / packageBin := {
+      val stdDuckDBJar      = (Compile / packageBin).value
+      val cacheStoreFactory = streams.value.cacheStoreFactory
+      StdBits
+        .copyDependencies(
+          `std-duckdb-polyglot-root`,
+          Seq("std-duckdb.jar"),
+          ignoreScalaLibrary = true,
+          libraryUpdates     = (Compile / update).value,
+          unmanagedClasspath = (Compile / unmanagedClasspath).value,
+          polyglotLibDir     = Some(`std-duckdb-native-libs`),
+          ignoreDependencies = None,
+          extractedNativeLibsDirs = Seq(
+            (`duckdb-wrapper` / extractedFilesDir).value
+          ),
+          extraJars = Seq(
+            (`duckdb-wrapper` / thinJarOutput).value
+          ),
+          logger            = streams.value.log,
+          cacheStoreFactory = cacheStoreFactory
+        )
+      stdDuckDBJar
+    },
+    clean := Def.task {
+      val _ = clean.value
+      IO.delete(`std-duckdb-polyglot-root`)
+      IO.delete(`std-duckdb-native-libs`)
+    }.value
+  )
+  .dependsOn(`std-base` % "provided")
+  .dependsOn(`std-table` % "provided")
+  .dependsOn(`std-database` % "provided")
+
 lazy val fetchZipToUnmanaged =
   taskKey[Seq[Attributed[File]]](
     "Download zip file from an `unmanagedExternalZip` url and unpack jars to unmanaged libs directory"
@@ -6353,7 +6438,8 @@ val stdBitsProjects =
     "Microsoft",
     "Snowflake",
     "Table",
-    "Saas"
+    "Saas",
+    "DuckDB"
   ) ++ allStdBitsSuffix
 val allStdBits: Parser[String] =
   stdBitsProjects.map(v => v: Parser[String]).reduce(_ | _)
@@ -6434,6 +6520,8 @@ pkgStdLibInternal := Def.inputTask {
       (`std-tableau` / Compile / packageBin).value
     case "Saas" =>
       (`std-saas` / Compile / packageBin).value
+    case "DuckDB" =>
+      (`std-duckdb` / Compile / packageBin).value
     case _ if buildAllCmd =>
       (`std-base` / Compile / packageBin).value
       (`enso-test-java-helpers` / Compile / packageBin).value
@@ -6451,6 +6539,7 @@ pkgStdLibInternal := Def.inputTask {
       (`std-microsoft` / Compile / packageBin).value
       (`std-tableau` / Compile / packageBin).value
       (`std-saas` / Compile / packageBin).value
+      (`std-duckdb` / Compile / packageBin).value
     case _ =>
   }
   val libs =

build_tools/build/src/ci_gen.rs

@@ -955,6 +955,7 @@ fn stdlib_api_change_labels_workflow() -> Result<Workflow> {
         "AWS",
         "Base",
         "Database",
+        "DuckDB",
         "Generic_JDBC",
         "Google",
         "Image",

distribution/lib/Standard/Database/0.0.0-dev/docs/api/Connection/Connection.md

@@ -37,4 +37,5 @@
 - make_schema_selector connection:Standard.Base.Any.Any include_any:Standard.Base.Data.Boolean.Boolean= -> Standard.Base.Any.Any
 - make_structure_creator -> Standard.Base.Any.Any
 - make_table_name_selector connection:Standard.Base.Any.Any -> Standard.Base.Any.Any
+- make_table_name_with_schema_selector connection:Standard.Base.Any.Any schema_black_list:Standard.Base.Any.Any= -> Standard.Base.Any.Any
 - make_table_types_selector connection:Standard.Base.Any.Any -> Standard.Base.Any.Any

distribution/lib/Standard/Database/0.0.0-dev/docs/api/Internal/JDBC_Connection.md

@@ -16,7 +16,7 @@
     - with_metadata self ~action:Standard.Base.Any.Any -> Standard.Base.Any.Any
     - with_prepared_statement self query:Standard.Base.Any.Any statement_setter:Standard.Base.Any.Any action:Standard.Base.Any.Any skip_log:Standard.Base.Any.Any= -> Standard.Base.Any.Any
 - close_connection connection:Standard.Base.Any.Any -> Standard.Base.Any.Any
-- create url:Standard.Base.Any.Any properties:Standard.Base.Any.Any -> Standard.Base.Any.Any
+- create url:Standard.Base.Any.Any properties:Standard.Base.Any.Any database:Standard.Base.Data.Text.Text= schema:Standard.Base.Data.Text.Text= -> Standard.Base.Any.Any
 - from_java java_jdbc_connection:Standard.Base.Any.Any sql_exception:Standard.Base.Any.Any= -> Standard.Base.Any.Any
 - get_pragma_value jdbc_connection:Standard.Base.Any.Any sql:Standard.Base.Any.Any -> Standard.Base.Any.Any
 - handle_sql_errors ~action:Standard.Base.Any.Any related_query:Standard.Base.Any.Any= sql_exception:Standard.Base.Any.Any= -> Standard.Base.Any.Any

distribution/lib/Standard/Database/0.0.0-dev/docs/api/SQL_Query.md

@@ -4,6 +4,12 @@
     - Raw_SQL sql:Standard.Base.Data.Text.Text=
     - Table_Name name:Standard.Base.Data.Text.Text=
     - to_db_table self connection:Standard.Database.Connection.Connection.Connection alias:Standard.Base.Data.Text.Text -> Standard.Base.Any.Any
+- type SQL_Query_With_Schema
+    - Raw_SQL sql:Standard.Base.Data.Text.Text=
+    - Table_Name name:Standard.Base.Data.Text.Text= schema:Standard.Base.Data.Text.Text=
+    - From that:Standard.Database.SQL_Query.SQL_Query -> Standard.Base.Any.Any
+    - to_db_table self connection:Standard.Database.Connection.Connection.Connection alias:Standard.Base.Data.Text.Text -> Standard.Base.Any.Any
 - make_table_for_name connection:Standard.Base.Any.Any name:Standard.Base.Any.Any schema:Standard.Base.Any.Any alias:Standard.Base.Any.Any internal_temporary_keep_alive_reference:Standard.Base.Any.Any= -> Standard.Base.Any.Any
 - make_table_from_query connection:Standard.Base.Any.Any query:(Standard.Base.Data.Text.Text|Standard.Database.SQL.SQL_Statement) alias:Standard.Base.Data.Text.Text -> (Standard.Table.Table.Table&Standard.Database.DB_Table.DB_Table)
 - Standard.Database.SQL_Query.SQL_Query.from that:Standard.Base.Data.Text.Text -> Standard.Database.SQL_Query.SQL_Query
+- Standard.Database.SQL_Query.SQL_Query_With_Schema.from that:Standard.Base.Data.Text.Text -> Standard.Database.SQL_Query.SQL_Query_With_Schema

distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Connection.enso

@@ -593,6 +593,20 @@ make_table_name_selector connection =
     values = table_name_values + [Option '<Table_Name>' '..Table_Name ""', Option '<Raw_SQL>' '..Raw_SQL ""']
     Single_Choice display=Display.Always values=values
 
+## ---
+   private: true
+   ---
+make_table_name_with_schema_selector connection schema_black_list=[] =
+    schema = connection.schema
+    schema_text sch = if sch == schema then "" else sch+"."
+    tables = connection.tables schema="*" . filter "Schema" (..Is_In schema_black_list ..Remove)
+    table_name_values = tables.rows.map r->
+        name = r.at "Name"
+        sch = r.at "Schema"
+        Option ((schema_text sch) + name) "(..Table_Name "+name.pretty+(if sch == schema then "" else " "+sch.pretty)+")"
+    values = table_name_values + [Option '<Table_Name>' '..Table_Name ""', Option '<Raw_SQL>' '..Raw_SQL ""']
+    Single_Choice display=..Always values=values
+
 ## ---
    private: true
    ---

distribution/lib/Standard/Database/0.0.0-dev/src/Connection/Postgres_Connection.enso

@@ -280,7 +280,6 @@ type Postgres_Connection
     execute_query self query (limit : Rows_To_Read = ..First_With_Warning 1000) write_operation:Boolean=True =
         self.connection.execute_query query limit write_operation
 
-
     ## ---
        advanced: true
        aliases: [sql]

distribution/lib/Standard/Database/0.0.0-dev/src/Internal/JDBC_Connection.enso

@@ -292,9 +292,11 @@ type JDBC_Connection
    ## Arguments
     - `url`: The URL to connect to.
     - `properties`: A vector of properties for the connection.
-create : Text -> Vector -> JDBC_Connection
-create url properties = handle_sql_errors <|
-    java_connection = JDBCProxy.getConnection url (properties_as_java_props properties)
+    - `database`: The database name to connect to.
+    - `schema`: The schema name to connect to.
+create : Text -> Vector -> Text -> Text -> JDBC_Connection
+create url properties database:Text="" schema:Text="" = handle_sql_errors <|
+    java_connection = JDBCProxy.getConnectionWithCatalogSchema url (properties_as_java_props properties) database schema
     from_java java_connection
 
 ## ---

distribution/lib/Standard/Database/0.0.0-dev/src/Main.enso

@@ -20,6 +20,7 @@ export project.Extensions.Upload_Table.select_into_database_table
 export project.Extensions.Upload_Table.update_rows
 
 export project.SQL_Query.SQL_Query
+export project.SQL_Query.SQL_Query_With_Schema
 
 export project.Update_Action.Update_Action
 

distribution/lib/Standard/Database/0.0.0-dev/src/SQL_Query.enso

@@ -34,11 +34,43 @@ type SQL_Query
             connection.dialect.ensure_query_has_no_holes connection.jdbc_connection raw_sql . if_not_error <|
                 make_table_from_query connection raw_sql alias
 
+## Enhanced SQL_Query that also carries schema information.
+type SQL_Query_With_Schema
+    ## Query a whole table or view.
+    Table_Name (name : Text = Missing_Argument.throw "name") (schema : Text = "")
+
+    ## Raw SQL query statement.
+    Raw_SQL (sql : Text = Missing_Argument.throw "sql")
+
+    ## ---
+       private: true
+       ---
+    to_db_table self connection:Connection alias:Text = alias.if_not_error <| case self of
+        SQL_Query_With_Schema.Table_Name name schema ->
+            table_naming_helper = connection.base_connection.table_naming_helper
+            table_naming_helper.verify_table_name name <|
+                make_table_for_name connection name schema alias
+        SQL_Query_With_Schema.Raw_SQL raw_sql -> handle_sql_errors <|
+            connection.dialect.ensure_query_has_no_holes connection.jdbc_connection raw_sql . if_not_error <|
+                make_table_from_query connection raw_sql alias
+
 ## ---
    private: true
    ---
 SQL_Query.from (that:Text) = SQL_Query.Table_Name that
 
+## ---
+   private: true
+   ---
+SQL_Query_With_Schema.from (that:Text) = SQL_Query_With_Schema.Table_Name that
+
+## ---
+   private: true
+   ---
+SQL_Query_With_Schema.From (that:SQL_Query) = case that of
+    SQL_Query.Table_Name name -> SQL_Query_With_Schema.Table_Name name ""
+    SQL_Query.Raw_SQL sql -> SQL_Query_With_Schema.Raw_SQL sql
+
 ## ---
    private: true
    ---

distribution/lib/Standard/DuckDB/0.0.0-dev/THIRD-PARTY/NOTICE

@@ -0,0 +1,2 @@
+Enso
+Copyright 2020 - 2025 New Byte Order sp. z o. o.

distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB.md

@@ -0,0 +1,9 @@
+## Enso Signatures 1.0
+## module Standard.DuckDB.DuckDB
+- type DuckDB
+    - From_File location:Standard.Base.System.File.File= schema:Standard.Base.Data.Text.Text= read_only:Standard.Base.Data.Boolean.Boolean=
+    - In_Memory schema:Standard.Base.Data.Text.Text= read_only:Standard.Base.Data.Boolean.Boolean=
+    - connect self options:Standard.Base.Any.Any -> Standard.Base.Any.Any
+    - jdbc_properties self -> Standard.Base.Any.Any
+    - jdbc_url self -> Standard.Base.Any.Any
+    - resolve constructor:Standard.Base.Any.Any -> Standard.Base.Any.Any

distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/DuckDB_Connection.md

@@ -0,0 +1,28 @@
+## Enso Signatures 1.0
+## module Standard.DuckDB.DuckDB_Connection
+- type DuckDB_Connection
+    - base_connection self -> Standard.Base.Any.Any
+    - close self -> Standard.Base.Any.Any
+    - create url:Standard.Base.Any.Any properties:Standard.Base.Any.Any make_new:Standard.Base.Any.Any schema:Standard.Base.Any.Any -> Standard.Base.Any.Any
+    - create_table self table_name:Standard.Base.Data.Text.Text structure:(Standard.Base.Any.Any|Standard.Table.Table.Table) primary_key:(Standard.Base.Any.Any|Standard.Base.Nothing.Nothing)= temporary:Standard.Base.Data.Boolean.Boolean= allow_existing:Standard.Base.Data.Boolean.Boolean= on_problems:Standard.Base.Errors.Problem_Behavior.Problem_Behavior= -> Standard.Base.Any.Any
+    - database self -> Standard.Base.Any.Any
+    - databases self -> Standard.Base.Any.Any
+    - dialect self -> Standard.Base.Any.Any
+    - drop_table self table_name:Standard.Base.Any.Any if_exists:Standard.Base.Any.Any= -> Standard.Base.Any.Any
+    - execute self query:Standard.Base.Any.Any -> Standard.Base.Any.Any
+    - execute_query self query:Standard.Base.Any.Any limit:Standard.Table.Rows_To_Read.Rows_To_Read= write_operation:Standard.Base.Data.Boolean.Boolean= -> Standard.Base.Any.Any
+    - execute_update self query:Standard.Base.Any.Any -> Standard.Base.Any.Any
+    - jdbc_connection self -> Standard.Base.Any.Any
+    - query self query:(Standard.Database.SQL_Query.SQL_Query_With_Schema|Standard.Database.SQL_Query.SQL_Query) alias:Standard.Base.Any.Any= -> Standard.Base.Any.Any
+    - read self query:(Standard.Database.SQL_Query.SQL_Query_With_Schema|Standard.Database.SQL_Query.SQL_Query) limit:Standard.Table.Rows_To_Read.Rows_To_Read= -> Standard.Base.Any.Any
+    - schema self -> Standard.Base.Any.Any
+    - schemas self -> Standard.Base.Any.Any
+    - set_database self database:Standard.Base.Any.Any -> Standard.Base.Any.Any
+    - set_schema self schema:Standard.Base.Any.Any -> Standard.Base.Any.Any
+    - table_types self -> Standard.Base.Any.Any
+    - tables self name_like:Standard.Base.Data.Text.Text= database:Standard.Base.Data.Text.Text= schema:Standard.Base.Data.Text.Text= types:Standard.Base.Any.Any= all_fields:Standard.Base.Any.Any= -> Standard.Base.Any.Any
+    - to_js_object self -> Standard.Base.Any.Any
+    - truncate_table self table_name:Standard.Base.Any.Any -> Standard.Base.Any.Any
+    - version self -> Standard.Base.Data.Text.Text
+- schema_black_list -> Standard.Base.Any.Any
+- Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data.from that:Standard.DuckDB.DuckDB_Connection.DuckDB_Connection -> Standard.Base.Visualization.Table_Viz_Data.Table_Viz_Data

distribution/lib/Standard/DuckDB/0.0.0-dev/docs/api/Internal/DuckDB_Entity_Naming_Properties.md

@@ -0,0 +1,3 @@
+## Enso Signatures 1.0
+## module Standard.DuckDB.Internal.DuckDB_Entity_Naming_Properties
+- new -> Standard.Base.Any.Any