From 49fdf39cfc9e6e8608d03f0dd3abd68eac310851 Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Fri, 17 Dec 2021 14:34:21 +0200 Subject: [PATCH 1/6] Add the local AWS Athena JDBC file --- plugins/nf-sqldb/build.gradle | 7 +++++++ .../src/main/nextflow/sql/config/SqlDataSource.groovy | 1 + .../src/test/nextflow/sql/config/SqlDataSourceTest.groovy | 1 + 3 files changed, 9 insertions(+) diff --git a/plugins/nf-sqldb/build.gradle b/plugins/nf-sqldb/build.gradle index ae8f78ee68..2cb30ca3ea 100644 --- a/plugins/nf-sqldb/build.gradle +++ b/plugins/nf-sqldb/build.gradle @@ -18,6 +18,12 @@ apply plugin: 'java-test-fixtures' apply plugin: 'idea' apply plugin: 'groovy' +repositories { + flatDir { + dirs 'libs' + } +} + sourceSets { main.java.srcDirs = [] main.groovy.srcDirs = ['src/main'] @@ -43,6 +49,7 @@ dependencies { api 'org.postgresql:postgresql:42.2.23' api 'org.xerial:sqlite-jdbc:3.36.0.3' api 'org.duckdb:duckdb_jdbc:0.3.0' + implementation name: 'AthenaJDBC42_2.0.25.1001' testImplementation(testFixtures(project(":nextflow"))) testImplementation project(':nextflow') diff --git a/plugins/nf-sqldb/src/main/nextflow/sql/config/SqlDataSource.groovy b/plugins/nf-sqldb/src/main/nextflow/sql/config/SqlDataSource.groovy index 1270fda4bb..ec25a42598 100644 --- a/plugins/nf-sqldb/src/main/nextflow/sql/config/SqlDataSource.groovy +++ b/plugins/nf-sqldb/src/main/nextflow/sql/config/SqlDataSource.groovy @@ -65,6 +65,7 @@ class SqlDataSource { case 'mariadb': return 'org.mariadb.jdbc.Driver' case 'postgresql': return 'org.postgresql.Driver' case 'duckdb': return 'org.duckdb.DuckDBDriver' + case 'awsathena': return 'com.simba.athena.jdbc.Driver' } return null } diff --git a/plugins/nf-sqldb/src/test/nextflow/sql/config/SqlDataSourceTest.groovy b/plugins/nf-sqldb/src/test/nextflow/sql/config/SqlDataSourceTest.groovy index 92de661a98..6a6d0539cc 100644 --- a/plugins/nf-sqldb/src/test/nextflow/sql/config/SqlDataSourceTest.groovy +++ b/plugins/nf-sqldb/src/test/nextflow/sql/config/SqlDataSourceTest.groovy @@ -69,6 +69,7 @@ class SqlDataSourceTest extends Specification { 'jdbc:mysql:some-host' | 'com.mysql.cj.jdbc.Driver' 'jdbc:mariadb:other-host' | 'org.mariadb.jdbc.Driver' 'jdbc:duckdb:' | 'org.duckdb.DuckDBDriver' + 'jdbc:awsathena:' | 'com.simba.athena.jdbc.Driver' } def 'should get default config' () { From aa61418aed7e4cb371966214a66ac1aafdbc9420 Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Sun, 19 Dec 2021 19:43:01 +0200 Subject: [PATCH 2/6] Add gradle tasks for downloading the Athena JDBC driver --- plugins/nf-sqldb/build.gradle | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/plugins/nf-sqldb/build.gradle b/plugins/nf-sqldb/build.gradle index 2cb30ca3ea..58cc86f71f 100644 --- a/plugins/nf-sqldb/build.gradle +++ b/plugins/nf-sqldb/build.gradle @@ -13,6 +13,40 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + +plugins { + id 'de.undercouch.download' version '4.1.2' +} + +/** + * The following tasks download and confirm the MD5 checksum of the ZIP archive + * for Simba Athena JDBC driver and extract its contents to the build directory + */ +task downloadZipFile(type: Download) { + src 'https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC-2.0.25.1002/SimbaAthenaJDBC-2.0.25.1002.zip' + dest new File(buildDir, 'SimbaAthenaJDBC-2.0.25.1002.zip') +} + +task verifyFile(type: Verify, dependsOn: downloadZipFile) { + src new File(buildDir, 'SimbaAthenaJDBC-2.0.25.1002.zip') + algorithm 'MD5' + checksum '9dfef9e578a709b863b1135a21bb3857' +} + +task downloadAndUnzipFile(dependsOn: [downloadZipFile, verifyFile] , type: Copy) { + from zipTree(downloadZipFile.dest) + into "${projectDir}/libs" +} + +task copyAthenaSimbaJar (dependsOn: downloadAndUnzipFile, type: Copy) { + from "${projectDir}/libs/SimbaAthenaJDBC-2.0.25.1002/AthenaJDBC42_2.0.25.1002.jar" + into "${projectDir}/libs" +} + +task cleanUpLibsFolder (dependsOn: copyAthenaSimbaJar, type: Delete) { + delete "${projectDir}/libs/SimbaAthenaJDBC-2.0.25.1002" +} + apply plugin: 'java' apply plugin: 'java-test-fixtures' apply plugin: 'idea' @@ -50,6 +84,7 @@ dependencies { api 'org.xerial:sqlite-jdbc:3.36.0.3' api 'org.duckdb:duckdb_jdbc:0.3.0' implementation name: 'AthenaJDBC42_2.0.25.1001' + api "org.slf4j:log4j-over-slf4j:1.7.32" testImplementation(testFixtures(project(":nextflow"))) testImplementation project(':nextflow') From d84e6020ef851f23628057d705bd0ebbddbbcd13 Mon Sep 17 00:00:00 2001 From: Abhinav Sharma Date: Sun, 19 Dec 2021 21:47:58 +0200 Subject: [PATCH 3/6] Update the driver - use the AWS included SDK --- plugins/nf-sqldb/build.gradle | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/plugins/nf-sqldb/build.gradle b/plugins/nf-sqldb/build.gradle index 58cc86f71f..1210c15148 100644 --- a/plugins/nf-sqldb/build.gradle +++ b/plugins/nf-sqldb/build.gradle @@ -23,14 +23,14 @@ plugins { * for Simba Athena JDBC driver and extract its contents to the build directory */ task downloadZipFile(type: Download) { - src 'https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC-2.0.25.1002/SimbaAthenaJDBC-2.0.25.1002.zip' - dest new File(buildDir, 'SimbaAthenaJDBC-2.0.25.1002.zip') + src 'https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC-2.0.25.1001/SimbaAthenaJDBC-2.0.25.1001.zip' + dest new File(buildDir, 'SimbaAthenaJDBC-2.0.25.1001.zip') } task verifyFile(type: Verify, dependsOn: downloadZipFile) { - src new File(buildDir, 'SimbaAthenaJDBC-2.0.25.1002.zip') + src new File(buildDir, 'SimbaAthenaJDBC-2.0.25.1001.zip') algorithm 'MD5' - checksum '9dfef9e578a709b863b1135a21bb3857' + checksum '71504d0317656d790978363358d0c068' } task downloadAndUnzipFile(dependsOn: [downloadZipFile, verifyFile] , type: Copy) { @@ -38,15 +38,23 @@ task downloadAndUnzipFile(dependsOn: [downloadZipFile, verifyFile] , type: Copy) into "${projectDir}/libs" } -task copyAthenaSimbaJar (dependsOn: downloadAndUnzipFile, type: Copy) { - from "${projectDir}/libs/SimbaAthenaJDBC-2.0.25.1002/AthenaJDBC42_2.0.25.1002.jar" +task copyAthenaSimbaJar(dependsOn: downloadAndUnzipFile, type: Copy) { + from "${projectDir}/libs/SimbaAthenaJDBC-2.0.25.1001/AthenaJDBC42_2.0.25.1001.jar" into "${projectDir}/libs" } -task cleanUpLibsFolder (dependsOn: copyAthenaSimbaJar, type: Delete) { - delete "${projectDir}/libs/SimbaAthenaJDBC-2.0.25.1002" +task downloadAndCleanUpLibsFolder(dependsOn: copyAthenaSimbaJar, type: Delete) { + delete "${projectDir}/libs/SimbaAthenaJDBC-2.0.25.1001/" } +defaultTasks 'downloadAndCleanUpLibsFolder' + +build.dependsOn downloadAndCleanUpLibsFolder +compileGroovy.dependsOn downloadAndCleanUpLibsFolder +compileTestGroovy.dependsOn downloadAndCleanUpLibsFolder +test.dependsOn downloadAndCleanUpLibsFolder +copyPluginLibs.dependsOn downloadAndCleanUpLibsFolder + apply plugin: 'java' apply plugin: 'java-test-fixtures' apply plugin: 'idea' From b5c7325837a40b97c902e1cbeb60ee4e3f63a082 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 21 Dec 2021 19:38:00 +0100 Subject: [PATCH 4/6] Simplify build for Athena support --- plugins/nf-sqldb/build.gradle | 27 +++++++++------------------ 1 file changed, 9 insertions(+), 18 deletions(-) diff --git a/plugins/nf-sqldb/build.gradle b/plugins/nf-sqldb/build.gradle index 1210c15148..45690670d1 100644 --- a/plugins/nf-sqldb/build.gradle +++ b/plugins/nf-sqldb/build.gradle @@ -24,36 +24,27 @@ plugins { */ task downloadZipFile(type: Download) { src 'https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC-2.0.25.1001/SimbaAthenaJDBC-2.0.25.1001.zip' - dest new File(buildDir, 'SimbaAthenaJDBC-2.0.25.1001.zip') + dest new File(buildDir, 'downloads/SimbaAthenaJDBC-2.0.25.1001.zip') + overwrite false } task verifyFile(type: Verify, dependsOn: downloadZipFile) { - src new File(buildDir, 'SimbaAthenaJDBC-2.0.25.1001.zip') + src new File(buildDir, 'downloads/SimbaAthenaJDBC-2.0.25.1001.zip') algorithm 'MD5' checksum '71504d0317656d790978363358d0c068' } task downloadAndUnzipFile(dependsOn: [downloadZipFile, verifyFile] , type: Copy) { from zipTree(downloadZipFile.dest) - into "${projectDir}/libs" + into "${buildDir}/downloads/unzip" } task copyAthenaSimbaJar(dependsOn: downloadAndUnzipFile, type: Copy) { - from "${projectDir}/libs/SimbaAthenaJDBC-2.0.25.1001/AthenaJDBC42_2.0.25.1001.jar" - into "${projectDir}/libs" + from "${buildDir}/downloads/unzip/SimbaAthenaJDBC-2.0.25.1001/AthenaJDBC42_2.0.25.1001.jar" + into "libs" } -task downloadAndCleanUpLibsFolder(dependsOn: copyAthenaSimbaJar, type: Delete) { - delete "${projectDir}/libs/SimbaAthenaJDBC-2.0.25.1001/" -} - -defaultTasks 'downloadAndCleanUpLibsFolder' - -build.dependsOn downloadAndCleanUpLibsFolder -compileGroovy.dependsOn downloadAndCleanUpLibsFolder -compileTestGroovy.dependsOn downloadAndCleanUpLibsFolder -test.dependsOn downloadAndCleanUpLibsFolder -copyPluginLibs.dependsOn downloadAndCleanUpLibsFolder +compileGroovy.dependsOn copyAthenaSimbaJar apply plugin: 'java' apply plugin: 'java-test-fixtures' @@ -84,6 +75,7 @@ dependencies { compileOnly project(':nextflow') compileOnly 'org.slf4j:slf4j-api:1.7.10' compileOnly 'org.pf4j:pf4j:3.4.1' + compileOnly "org.slf4j:log4j-over-slf4j:1.7.32" api("org.codehaus.groovy:groovy-sql:3.0.9") { transitive = false } api 'com.h2database:h2:1.4.200' api 'mysql:mysql-connector-java:8.0.22' @@ -91,8 +83,7 @@ dependencies { api 'org.postgresql:postgresql:42.2.23' api 'org.xerial:sqlite-jdbc:3.36.0.3' api 'org.duckdb:duckdb_jdbc:0.3.0' - implementation name: 'AthenaJDBC42_2.0.25.1001' - api "org.slf4j:log4j-over-slf4j:1.7.32" + api name: 'AthenaJDBC42_2.0.25.1001' testImplementation(testFixtures(project(":nextflow"))) testImplementation project(':nextflow') From 0ea5f8c79b5d43de063a7aa453156c1f2412472f Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 21 Dec 2021 21:12:06 +0100 Subject: [PATCH 5/6] Fix Athena download tasks order --- plugins/nf-sqldb/build.gradle | 59 ++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/plugins/nf-sqldb/build.gradle b/plugins/nf-sqldb/build.gradle index 45690670d1..7f8fc38acc 100644 --- a/plugins/nf-sqldb/build.gradle +++ b/plugins/nf-sqldb/build.gradle @@ -18,34 +18,6 @@ plugins { id 'de.undercouch.download' version '4.1.2' } -/** - * The following tasks download and confirm the MD5 checksum of the ZIP archive - * for Simba Athena JDBC driver and extract its contents to the build directory - */ -task downloadZipFile(type: Download) { - src 'https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC-2.0.25.1001/SimbaAthenaJDBC-2.0.25.1001.zip' - dest new File(buildDir, 'downloads/SimbaAthenaJDBC-2.0.25.1001.zip') - overwrite false -} - -task verifyFile(type: Verify, dependsOn: downloadZipFile) { - src new File(buildDir, 'downloads/SimbaAthenaJDBC-2.0.25.1001.zip') - algorithm 'MD5' - checksum '71504d0317656d790978363358d0c068' -} - -task downloadAndUnzipFile(dependsOn: [downloadZipFile, verifyFile] , type: Copy) { - from zipTree(downloadZipFile.dest) - into "${buildDir}/downloads/unzip" -} - -task copyAthenaSimbaJar(dependsOn: downloadAndUnzipFile, type: Copy) { - from "${buildDir}/downloads/unzip/SimbaAthenaJDBC-2.0.25.1001/AthenaJDBC42_2.0.25.1001.jar" - into "libs" -} - -compileGroovy.dependsOn copyAthenaSimbaJar - apply plugin: 'java' apply plugin: 'java-test-fixtures' apply plugin: 'idea' @@ -91,3 +63,34 @@ dependencies { testImplementation "org.codehaus.groovy:groovy-nio:3.0.9" } + +/** + * The following tasks download and confirm the MD5 checksum of the ZIP archive + * for Simba Athena JDBC driver and extract its contents to the build directory + */ +tasks.register('downloadZipFile', Download) { + src 'https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC-2.0.25.1001/SimbaAthenaJDBC-2.0.25.1001.zip' + dest new File(buildDir, 'downloads/SimbaAthenaJDBC-2.0.25.1001.zip') + overwrite false +} + +tasks.register('verifyFile', Verify) { + dependsOn 'downloadZipFile' + src new File(buildDir, 'downloads/SimbaAthenaJDBC-2.0.25.1001.zip') + algorithm 'MD5' + checksum '71504d0317656d790978363358d0c068' +} + +tasks.register('downloadAndUnzipFile', Copy) { + dependsOn 'downloadZipFile', 'verifyFile' + from zipTree(downloadZipFile.dest) + into "${buildDir}/downloads/unzip" +} + +tasks.register('copyAthenaSimbaJar', Copy) { + dependsOn 'downloadAndUnzipFile' + from "${buildDir}/downloads/unzip/SimbaAthenaJDBC-2.0.25.1001/AthenaJDBC42_2.0.25.1001.jar" + into "libs" +} + +compileGroovy.dependsOn('copyAthenaSimbaJar') From 4b078ae7d85fc624e9fa35cd152e88b5f3df3437 Mon Sep 17 00:00:00 2001 From: Paolo Di Tommaso Date: Tue, 21 Dec 2021 21:39:29 +0100 Subject: [PATCH 6/6] Add athena tasks --- plugins/nf-sqldb/build.gradle | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/plugins/nf-sqldb/build.gradle b/plugins/nf-sqldb/build.gradle index 7f8fc38acc..2ea371cbc6 100644 --- a/plugins/nf-sqldb/build.gradle +++ b/plugins/nf-sqldb/build.gradle @@ -23,11 +23,6 @@ apply plugin: 'java-test-fixtures' apply plugin: 'idea' apply plugin: 'groovy' -repositories { - flatDir { - dirs 'libs' - } -} sourceSets { main.java.srcDirs = [] @@ -55,7 +50,7 @@ dependencies { api 'org.postgresql:postgresql:42.2.23' api 'org.xerial:sqlite-jdbc:3.36.0.3' api 'org.duckdb:duckdb_jdbc:0.3.0' - api name: 'AthenaJDBC42_2.0.25.1001' + api files('downloads/unzip/AthenaJDBC42_2.0.25.1001.jar') testImplementation(testFixtures(project(":nextflow"))) testImplementation project(':nextflow') @@ -68,29 +63,21 @@ dependencies { * The following tasks download and confirm the MD5 checksum of the ZIP archive * for Simba Athena JDBC driver and extract its contents to the build directory */ -tasks.register('downloadZipFile', Download) { +task downloadAthenDep(type: Download) { src 'https://s3.amazonaws.com/athena-downloads/drivers/JDBC/SimbaAthenaJDBC-2.0.25.1001/SimbaAthenaJDBC-2.0.25.1001.zip' dest new File(buildDir, 'downloads/SimbaAthenaJDBC-2.0.25.1001.zip') overwrite false } -tasks.register('verifyFile', Verify) { - dependsOn 'downloadZipFile' +task verifyAthenDep(type: Verify, dependsOn: downloadAthenDep) { src new File(buildDir, 'downloads/SimbaAthenaJDBC-2.0.25.1001.zip') algorithm 'MD5' checksum '71504d0317656d790978363358d0c068' } -tasks.register('downloadAndUnzipFile', Copy) { - dependsOn 'downloadZipFile', 'verifyFile' - from zipTree(downloadZipFile.dest) +task unzipAthenDep(dependsOn: verifyAthenDep, type: Copy) { + from zipTree(new File(buildDir, 'downloads/SimbaAthenaJDBC-2.0.25.1001.zip')) into "${buildDir}/downloads/unzip" } -tasks.register('copyAthenaSimbaJar', Copy) { - dependsOn 'downloadAndUnzipFile' - from "${buildDir}/downloads/unzip/SimbaAthenaJDBC-2.0.25.1001/AthenaJDBC42_2.0.25.1001.jar" - into "libs" -} - -compileGroovy.dependsOn('copyAthenaSimbaJar') +project.compileGroovy.dependsOn('unzipAthenDep')