From 7e8cb9f28273bb462bdad5d4a73370b2700beb60 Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Tue, 10 Jan 2023 16:05:10 +0100 Subject: [PATCH 01/16] Lock Bundler to v2.3.26 This is the last version that supports Ruby 2.4 and 2.5. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index eaf03ccd..89fc5aac 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -112,7 +112,7 @@ jobs: command: | $Env:PATH = "C:\\Ruby<< parameters.ruby_version >>-x64\\bin;$Env:PATH" ridk install 2 - gem install bundler + gem install bundler -v 2.3.26 - checkout From 7b33fe3b05de8e042046d340eeb003e435a84949 Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Tue, 10 Jan 2023 20:28:07 +0100 Subject: [PATCH 02/16] Remove fix to use rake-compiler-dock on Windows for libiconv The original author states that the code I removed is for using virtualbox on Windows to compile the gem. I read up on this and while rake-compiler-dock still supports using docker-machine to compile gems, the project has been discontinued in favor of Docker Desktop, which nowadays only works with Docker in WSL2. --- tasks/ports/libiconv.rb | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/tasks/ports/libiconv.rb b/tasks/ports/libiconv.rb index e3a3531d..52e0f184 100644 --- a/tasks/ports/libiconv.rb +++ b/tasks/ports/libiconv.rb @@ -8,25 +8,8 @@ def initialize(version) set_patches end - def cook - chdir_for_build do - super - end - self - end - private - # When using rake-compiler-dock on Windows, the underlying Virtualbox shared - # folders don't support symlinks, but libiconv expects it for a build on - # Linux. We work around this limitation by using the temp dir for cooking. - def chdir_for_build - build_dir = ENV['RCD_HOST_RUBY_PLATFORM'].to_s =~ /mingw|mswin|cygwin/ ? '/tmp' : '.' - Dir.chdir(build_dir) do - yield - end - end - def configure_defaults [ "--host=#{@host}", From 4da29c5bcde3ba8255650cf87f519aceddc61c8f Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Tue, 10 Jan 2023 22:52:01 +0100 Subject: [PATCH 03/16] Bump openssl to v1.1.1s 1.1.1d gives an error when compiling against i686-w64-mingw32. Upgrading it seems to work. --- ext/tiny_tds/extconsts.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ext/tiny_tds/extconsts.rb b/ext/tiny_tds/extconsts.rb index 1e1c1bf2..f3c5e249 100644 --- a/ext/tiny_tds/extconsts.rb +++ b/ext/tiny_tds/extconsts.rb @@ -2,7 +2,7 @@ ICONV_VERSION = ENV['TINYTDS_ICONV_VERSION'] || "1.15" ICONV_SOURCE_URI = "http://ftp.gnu.org/pub/gnu/libiconv/libiconv-#{ICONV_VERSION}.tar.gz" -OPENSSL_VERSION = ENV['TINYTDS_OPENSSL_VERSION'] || '1.1.1d' +OPENSSL_VERSION = ENV['TINYTDS_OPENSSL_VERSION'] || '1.1.1s' OPENSSL_SOURCE_URI = "https://www.openssl.org/source/openssl-#{OPENSSL_VERSION}.tar.gz" FREETDS_VERSION = ENV['TINYTDS_FREETDS_VERSION'] || "1.1.24" From fc83d92b46e6a61dbdee9f2b15cccea2ec345b95 Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Tue, 10 Jan 2023 23:37:07 +0100 Subject: [PATCH 04/16] Define task to compile native extensions --- Rakefile | 1 + tasks/compile.rake | 18 ++++++++++++++++++ tasks/native_gem.rake | 2 +- 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 tasks/compile.rake diff --git a/Rakefile b/Rakefile index 3150afdc..588f2598 100644 --- a/Rakefile +++ b/Rakefile @@ -10,6 +10,7 @@ GEM_PLATFORM_HOSTS = { 'x86-mingw32' => 'i686-w64-mingw32', 'x64-mingw32' => 'x86_64-w64-mingw32' } +RUBY_CC_VERSION="2.7.0:2.6.0:2.5.0:2.4.0".freeze # Add our project specific files to clean for a rebuild CLEAN.include FileList["{ext,lib}/**/*.{so,#{RbConfig::CONFIG['DLEXT']},o}"], diff --git a/tasks/compile.rake b/tasks/compile.rake new file mode 100644 index 00000000..aa61b05c --- /dev/null +++ b/tasks/compile.rake @@ -0,0 +1,18 @@ +desc 'Cross-compiles the gem' +task 'compile:cross' do + require 'rake_compiler_dock' + + # make sure to install our bundle + sh "bundle package --all" # Avoid repeated downloads of gems by using gem files from the host. + + # and finally build the native gem + GEM_PLATFORM_HOSTS.each do |gem_platform, host| + commands = [ + "bundle --local", + "rake ports:compile[#{host}] MAKE='make -j`nproc`'", + "RUBY_CC_VERSION=#{RUBY_CC_VERSION} CFLAGS='-Wall' MAKE='make -j`nproc`' rake compile:#{gem_platform}" + ] + + RakeCompilerDock.sh commands.join(" && "), platform: gem_platform + end +end diff --git a/tasks/native_gem.rake b/tasks/native_gem.rake index d19cdb85..31fe80f6 100644 --- a/tasks/native_gem.rake +++ b/tasks/native_gem.rake @@ -8,7 +8,7 @@ task 'gem:windows' => ['ports:cross'] do build = ['bundle'] # and finally build the native gem - build << 'rake cross native gem RUBY_CC_VERSION=2.7.0:2.6.0:2.5.0:2.4.0 CFLAGS="-Wall" MAKE="make -j`nproc`"' + build << "rake cross native gem RUBY_CC_VERSION=#{RUBY_CC_VERSION} CFLAGS=\"-Wall\" MAKE=\"make -j`nproc`\"" RakeCompilerDock.sh build.join(' && ') end From be925795dd934ba1881faf00466688795f30a663 Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Wed, 11 Jan 2023 19:41:38 +0100 Subject: [PATCH 05/16] Skip dead connection test on Windows --- test/client_test.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/client_test.rb b/test/client_test.rb index e85098eb..de988d6e 100644 --- a/test/client_test.rb +++ b/test/client_test.rb @@ -154,6 +154,8 @@ class ClientTest < TinyTds::TestCase end it 'raises TinyTds exception with dead connection network failure' do + skip if ruby_windows? + begin client = new_connection timeout: 2, port: 1234, host: ENV['TOXIPROXY_HOST'] assert_client_works(client) From f7e365611c712d4c2e8af653b629039308ed9cc2 Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Tue, 10 Jan 2023 19:22:01 +0100 Subject: [PATCH 06/16] Extend pipeline to test gem on Windows --- .circleci/config.yml | 129 ++++++++++++++++++++++++++++--------- Gemfile | 7 -- test/bin/install-mssql.ps1 | 31 +++++++++ 3 files changed, 131 insertions(+), 36 deletions(-) create mode 100644 test/bin/install-mssql.ps1 diff --git a/.circleci/config.yml b/.circleci/config.yml index 89fc5aac..f8c15e72 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -2,6 +2,7 @@ version: 2.1 orbs: win: circleci/windows@4.1 + ruby: circleci/ruby@2.0.0 jobs: test_linux: @@ -19,7 +20,7 @@ jobs: - run: name: start docker-compose build environment - command: | + command: | sudo ./test/bin/setup_volume_permissions.sh docker-compose up -d echo "Waiting for containers to start..." @@ -58,7 +59,7 @@ jobs: - run: name: test gem command: | - docker exec cimg_ruby bash -c 'bundle exec rake test' + docker exec cimg_ruby bash -c 'bundle exec rake test' test_windows: parameters: @@ -108,10 +109,9 @@ jobs: rm .\ruby-setup.exe - run: - name: update build env + name: install bundler command: | $Env:PATH = "C:\\Ruby<< parameters.ruby_version >>-x64\\bin;$Env:PATH" - ridk install 2 gem install bundler -v 2.3.26 - checkout @@ -127,7 +127,7 @@ jobs: name: bundle install gems command: | $Env:PATH = "C:\\Ruby<< parameters.ruby_version >>-x64\\bin;$Env:PATH" - bundle install + bundle install --path vendor/bundle - save_cache: name: save gem cache @@ -135,49 +135,120 @@ jobs: - ./vendor/bundle key: v1-bundle-<< parameters.ruby_version >>-{{ .Branch }}-{{ checksum "tiny_tds.gemspec" }} + - restore_cache: + name: restore cross-compiled code + keys: + - compiled-<< pipeline.git.revision >> + - run: - name: build openssl - no_output_timeout: 30m + name: restore cross-compiled code command: | $Env:PATH = "C:\\Ruby<< parameters.ruby_version >>-x64\\bin;$Env:PATH" - bundle exec rake ports:openssl + + $rubyArchitecture = (ruby -e 'puts RUBY_PLATFORM').Trim() + $source = "C:\home\circleci\project\tmp\$rubyArchitecture\stage\lib\tiny_tds" + $destination = ".\lib\tiny_tds" + Get-ChildItem $source -Recurse -Exclude "*.rb" | Copy-Item -Destination {Join-Path $destination $_.FullName.Substring($source.length)} + + - restore_cache: + name: restore mssql installation file + key: downloads-{{ checksum "test/bin/install-mssql.ps1" }} - run: - name: build libiconv - no_output_timeout: 30m + name: setup mssql command: | - $Env:PATH = "C:\\Ruby<< parameters.ruby_version >>-x64\\bin;$Env:PATH" - bundle exec rake ports:libiconv + .\test\bin\install-mssql.ps1 + + - save_cache: + name: save downloads cache + paths: + - C:\Downloads + key: downloads-{{ checksum "test/bin/install-mssql.ps1" }} - run: - name: build freetds - no_output_timeout: 30m + name: install toxiproxy-server command: | - $Env:PATH = "C:\\Ruby<< parameters.ruby_version >>-x64\\bin;$Env:PATH" - bundle exec rake ports:freetds + choco install toxiproxy-server --version=2.5.0 -y + Start-Process toxiproxy-server + + - restore_cache: + name: restore ports cache + keys: + - ports-libiconv115-openssl111s-freetds1124 - run: - name: build gem - no_output_timeout: 30m + name: copy ports cache into project + command: | + Move-Item C:\home\circleci\project\ports . + + - run: + name: test gem command: | $Env:PATH = "C:\\Ruby<< parameters.ruby_version >>-x64\\bin;$Env:PATH" - bundle exec rake ports + bundle exec rake test + environment: + TOXIPROXY_HOST: "localhost" + + cross_compile_ports: + machine: + image: ubuntu-2004:current + + steps: + - ruby/install: + version: '2.7' + - checkout + - restore_cache: + name: restore gem cache + keys: + - v1-bundle-{{ .Branch }}-{{ checksum "tiny_tds.gemspec" }} + - v1-bundle-{{ .Branch }}- + - v1-bundle- + + - run: + name: bundle install gems + command: | + bundle install --path vendor/bundle + + - save_cache: + name: save gem cache + paths: + - ./vendor/bundle + key: v1-bundle-{{ .Branch }}-{{ checksum "tiny_tds.gemspec" }} + + - restore_cache: + name: restore ports cache + keys: + - ports-libiconv115-openssl111s-freetds1124 + + - run: + name: cross compile ports + command: | + bundle exec rake compile:cross + + - save_cache: + name: save ports cache + paths: + - ./ports + key: ports-libiconv115-openssl111s-freetds1124 + + - save_cache: + name: save cross-compiled-code + paths: + - ./tmp + key: compiled-<< pipeline.git.revision >> workflows: test_supported_ruby_versions: jobs: - - test_linux: - matrix: + - cross_compile_ports + - test_windows: + requires: + - cross_compile_ports + matrix: &ruby_versions parameters: ruby_version: - '2.5' - '2.6' - '2.7' - - - test_windows: - matrix: - parameters: - ruby_version: - - '2.5' - - '2.6' - - '2.7' + - test_linux: + matrix: *ruby_versions diff --git a/Gemfile b/Gemfile index 313a64f0..851fabc2 100644 --- a/Gemfile +++ b/Gemfile @@ -1,9 +1,2 @@ source 'https://rubygems.org' gemspec - -group :development do -end - -group :test do - gem 'minitest' -end diff --git a/test/bin/install-mssql.ps1 b/test/bin/install-mssql.ps1 new file mode 100644 index 00000000..bc4c7248 --- /dev/null +++ b/test/bin/install-mssql.ps1 @@ -0,0 +1,31 @@ +$ProgressPreference = 'SilentlyContinue' + +if (-not(Test-path "C:\Downloads")) +{ + mkdir "C:\Downloads" +} + +$sqlInstallationFile = "C:\Downloads\sqlexpress.exe" +if (-not(Test-path $sqlInstallationFile -PathType leaf)) +{ + Write-Host "Downloading SQL Express ..." + Invoke-WebRequest -Uri "https://go.microsoft.com/fwlink/?linkid=829176" -OutFile "C:\Downloads\sqlexpress.exe" +} + +Write-Host "Installing SQL Express ..." +Start-Process -Wait -FilePath "C:\Downloads\sqlexpress.exe" -ArgumentList /qs, /x:"C:\Downloads\setup" +C:\Downloads\setup\setup.exe /q /ACTION=Install /INSTANCENAME=SQLEXPRESS /FEATURES=SQLEngine /UPDATEENABLED=0 /SQLSVCACCOUNT='NT AUTHORITY\System' /SQLSYSADMINACCOUNTS='BUILTIN\ADMINISTRATORS' /TCPENABLED=1 /NPENABLED=0 /IACCEPTSQLSERVERLICENSETERMS + +Write-Host "Configuring SQL Express ..." +stop-service MSSQL`$SQLEXPRESS +set-itemproperty -path 'HKLM:\software\microsoft\microsoft sql server\mssql14.SQLEXPRESS\mssqlserver\supersocketnetlib\tcp\ipall' -name tcpdynamicports -value '' +set-itemproperty -path 'HKLM:\software\microsoft\microsoft sql server\mssql14.SQLEXPRESS\mssqlserver\supersocketnetlib\tcp\ipall' -name tcpport -value 1433 +set-itemproperty -path 'HKLM:\software\microsoft\microsoft sql server\mssql14.SQLEXPRESS\mssqlserver\' -name LoginMode -value 2 + +Write-Host "Starting SQL Express ..." +start-service MSSQL`$SQLEXPRESS + +Write-Host "Configuring MSSQL for TinyTDS ..." +& sqlcmd -Q "CREATE DATABASE [tinytdstest];" +& sqlcmd -Q "CREATE LOGIN [tinytds] WITH PASSWORD = '', CHECK_POLICY = OFF, DEFAULT_DATABASE = [tinytdstest];" +& sqlcmd -Q "USE [tinytdstest]; CREATE USER [tinytds] FOR LOGIN [tinytds]; EXEC sp_addrolemember N'db_owner', N'tinytds';" From 55c342d06082d5aa52952ec9d4c9d4759b376b67 Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Wed, 11 Jan 2023 19:53:33 +0100 Subject: [PATCH 07/16] Package gems on CI before Windows tests --- .circleci/config.yml | 20 ++++++++++++++------ README.md | 2 +- tasks/compile.rake | 18 ------------------ tasks/native_gem.rake | 13 ++++++------- 4 files changed, 21 insertions(+), 32 deletions(-) delete mode 100644 tasks/compile.rake diff --git a/.circleci/config.yml b/.circleci/config.yml index f8c15e72..53cd7a26 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: ruby_version: description: 'version tag for the cimg/ruby container' type: string - + machine: image: ubuntu-2004:current @@ -189,7 +189,7 @@ jobs: environment: TOXIPROXY_HOST: "localhost" - cross_compile_ports: + cross_compile_gem: machine: image: ubuntu-2004:current @@ -221,9 +221,17 @@ jobs: - ports-libiconv115-openssl111s-freetds1124 - run: - name: cross compile ports + name: Build gems command: | - bundle exec rake compile:cross + bundle exec rake gem + bundle exec rake gem:native + + # Move gem files to a separate directory, as CircleCI artifacts don't support wildcards for paths + mkdir pkg_artifacts + mv pkg/*.gem pkg_artifacts + + - store_artifacts: + path: pkg_artifacts - save_cache: name: save ports cache @@ -240,10 +248,10 @@ jobs: workflows: test_supported_ruby_versions: jobs: - - cross_compile_ports + - cross_compile_gem - test_windows: requires: - - cross_compile_ports + - cross_compile_gem matrix: &ruby_versions parameters: ruby_version: diff --git a/README.md b/README.md index 45fdbb35..78240925 100644 --- a/README.md +++ b/README.md @@ -407,7 +407,7 @@ For the convenience of Windows users, TinyTDS ships pre-compiled gems for suppor Run the following rake task to compile the gems for Windows. This will check the availability of [Docker](https://www.docker.com/) (and boot2docker on Windows or OS-X) and will give some advice for download and installation. When docker is running, it will download the docker image (once-only) and start the build: ``` -$ rake gem:windows +$ rake gem:native ``` The compiled gems will exist in `./pkg` directory. diff --git a/tasks/compile.rake b/tasks/compile.rake deleted file mode 100644 index aa61b05c..00000000 --- a/tasks/compile.rake +++ /dev/null @@ -1,18 +0,0 @@ -desc 'Cross-compiles the gem' -task 'compile:cross' do - require 'rake_compiler_dock' - - # make sure to install our bundle - sh "bundle package --all" # Avoid repeated downloads of gems by using gem files from the host. - - # and finally build the native gem - GEM_PLATFORM_HOSTS.each do |gem_platform, host| - commands = [ - "bundle --local", - "rake ports:compile[#{host}] MAKE='make -j`nproc`'", - "RUBY_CC_VERSION=#{RUBY_CC_VERSION} CFLAGS='-Wall' MAKE='make -j`nproc`' rake compile:#{gem_platform}" - ] - - RakeCompilerDock.sh commands.join(" && "), platform: gem_platform - end -end diff --git a/tasks/native_gem.rake b/tasks/native_gem.rake index 31fe80f6..8dbe4768 100644 --- a/tasks/native_gem.rake +++ b/tasks/native_gem.rake @@ -1,14 +1,13 @@ # encoding: UTF-8 -desc 'Build the windows binary gems per rake-compiler-dock' -task 'gem:windows' => ['ports:cross'] do +desc 'Build the native binary gems using rake-compiler-dock' +task 'gem:native' => ['ports:cross'] do require 'rake_compiler_dock' # make sure to install our bundle - build = ['bundle'] + sh "bundle package --all" # Avoid repeated downloads of gems by using gem files from the host. - # and finally build the native gem - build << "rake cross native gem RUBY_CC_VERSION=#{RUBY_CC_VERSION} CFLAGS=\"-Wall\" MAKE=\"make -j`nproc`\"" - - RakeCompilerDock.sh build.join(' && ') + GEM_PLATFORM_HOSTS.keys.each do |plat| + RakeCompilerDock.sh "bundle --local && RUBY_CC_VERSION=#{RUBY_CC_VERSION} rake native:#{plat} gem", platform: plat + end end From dd3fd22e3c0396e783ffff1c8a0b18e45921f385 Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Wed, 11 Jan 2023 20:02:10 +0100 Subject: [PATCH 08/16] Ensure test container uses Ruby version defined in CI matrix --- .circleci/config.yml | 2 ++ docker-compose.yml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 53cd7a26..df583bf4 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -25,6 +25,8 @@ jobs: docker-compose up -d echo "Waiting for containers to start..." sleep 10 + environment: + RUBY_VERSION: << parameters.ruby_version >> - run: name: install sql prereqs diff --git a/docker-compose.yml b/docker-compose.yml index 3a1afd30..17a2a1c0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ services: network_mode: "host" cimgruby: - image: cimg/ruby:2.7.0 + image: "cimg/ruby:${RUBY_VERSION:-2.7}" container_name: cimg_ruby environment: TESTOPTS: '-v' From 00480bc9e532b12b4c22c8613002cc7f1f9a6ec5 Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Wed, 11 Jan 2023 20:08:37 +0100 Subject: [PATCH 09/16] Test against Ruby 2.4 --- .circleci/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index df583bf4..16dcafa9 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -257,6 +257,7 @@ workflows: matrix: &ruby_versions parameters: ruby_version: + - '2.4' - '2.5' - '2.6' - '2.7' From 43d252c7934013214e8a49f8efd5711d762e93f2 Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Wed, 11 Jan 2023 20:19:49 +0100 Subject: [PATCH 10/16] Publish test results to CircleCI --- .circleci/config.yml | 6 ++++++ .gitignore | 1 + tiny_tds.gemspec | 1 + 3 files changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 16dcafa9..18c93b4a 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -63,6 +63,9 @@ jobs: command: | docker exec cimg_ruby bash -c 'bundle exec rake test' + - store_test_results: + path: test/reports + test_windows: parameters: ruby_version: @@ -191,6 +194,9 @@ jobs: environment: TOXIPROXY_HOST: "localhost" + - store_test_results: + path: test/reports + cross_compile_gem: machine: image: ubuntu-2004:current diff --git a/.gitignore b/.gitignore index 6c1b7e4e..8d3986f8 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ misc /exe/* /ports/* !/ports/patches/ +test/reports diff --git a/tiny_tds.gemspec b/tiny_tds.gemspec index fa50d942..e85ede2a 100644 --- a/tiny_tds.gemspec +++ b/tiny_tds.gemspec @@ -25,6 +25,7 @@ Gem::Specification.new do |s| s.add_development_dependency 'rake-compiler', '~> 1.1.0' s.add_development_dependency 'rake-compiler-dock', '~> 1.1.0' s.add_development_dependency 'minitest', '~> 5.14.0' + s.add_development_dependency 'minitest-ci', '~> 3.4.0' s.add_development_dependency 'connection_pool', '~> 2.2.0' s.add_development_dependency 'toxiproxy', '~> 2.0.0' end From 741d1c5f98cc8776bd6d0a39df2320ac9c290707 Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Thu, 12 Jan 2023 17:52:44 +0100 Subject: [PATCH 11/16] Rename artifact for gems --- .circleci/config.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 18c93b4a..90eae24f 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -235,11 +235,11 @@ jobs: bundle exec rake gem:native # Move gem files to a separate directory, as CircleCI artifacts don't support wildcards for paths - mkdir pkg_artifacts - mv pkg/*.gem pkg_artifacts + mkdir gems + mv pkg/*.gem gems - store_artifacts: - path: pkg_artifacts + path: gems - save_cache: name: save ports cache From 9d12f78200da99d88df38b38ccfda52396b36da7 Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Thu, 12 Jan 2023 18:18:00 +0100 Subject: [PATCH 12/16] Restore cross-compiled code from final gem file --- .circleci/config.yml | 52 +++++++++++++++++++++---------------- .gitignore | 1 + tasks/ports.rake | 61 +++++++++++++++++++++++++++----------------- 3 files changed, 69 insertions(+), 45 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 90eae24f..6abf49f9 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -141,19 +141,29 @@ jobs: key: v1-bundle-<< parameters.ruby_version >>-{{ .Branch }}-{{ checksum "tiny_tds.gemspec" }} - restore_cache: - name: restore cross-compiled code + name: restore cross-compiled gems keys: - - compiled-<< pipeline.git.revision >> + - gems-<< pipeline.git.revision >> - run: - name: restore cross-compiled code + name: restore cross-compiled code from gem command: | $Env:PATH = "C:\\Ruby<< parameters.ruby_version >>-x64\\bin;$Env:PATH" + $rubyArchitecture = (ruby -e 'puts RUBY_PLATFORM').Trim() + $gemVersion = (Get-Content VERSION).Trim() - $rubyArchitecture = (ruby -e 'puts RUBY_PLATFORM').Trim() - $source = "C:\home\circleci\project\tmp\$rubyArchitecture\stage\lib\tiny_tds" - $destination = ".\lib\tiny_tds" + # Install appropriate gem + # The project on Windows is checked out at C:/Users/circleci.PACKER-633B1A5A/project + # However, since the cache with the gems was created on Linux, it uses a different path + gem install --local --install-dir=./tmp "C:\home\circleci\project\gems\tiny_tds-$gemVersion-$rubyArchitecture.gem" + + # Restore precompiled code + $source = (Resolve-Path ".\tmp\gems\tiny_tds-$gemVersion-$rubyArchitecture\lib\tiny_tds").Path + $destination = (Resolve-Path ".\lib\tiny_tds").Path Get-ChildItem $source -Recurse -Exclude "*.rb" | Copy-Item -Destination {Join-Path $destination $_.FullName.Substring($source.length)} + + # Restore ports + Copy-Item -Path ".\tmp\gems\tiny_tds-$gemVersion-$rubyArchitecture\ports" -Destination "." -Recurse - restore_cache: name: restore mssql installation file @@ -176,16 +186,6 @@ jobs: choco install toxiproxy-server --version=2.5.0 -y Start-Process toxiproxy-server - - restore_cache: - name: restore ports cache - keys: - - ports-libiconv115-openssl111s-freetds1124 - - - run: - name: copy ports cache into project - command: | - Move-Item C:\home\circleci\project\ports . - - run: name: test gem command: | @@ -223,10 +223,16 @@ jobs: - ./vendor/bundle key: v1-bundle-{{ .Branch }}-{{ checksum "tiny_tds.gemspec" }} + - run: + name: Write used versions for ports into file + command: | + bundle exec rake ports:version_file + - restore_cache: name: restore ports cache keys: - - ports-libiconv115-openssl111s-freetds1124 + - ports-{{ checksum ".ports_versions" }} + - ports- - run: name: Build gems @@ -234,7 +240,9 @@ jobs: bundle exec rake gem bundle exec rake gem:native - # Move gem files to a separate directory, as CircleCI artifacts don't support wildcards for paths + - run: + name: Move gems into separate directory before caching + command: | mkdir gems mv pkg/*.gem gems @@ -245,13 +253,13 @@ jobs: name: save ports cache paths: - ./ports - key: ports-libiconv115-openssl111s-freetds1124 + key: ports-{{ checksum ".ports_versions" }} - save_cache: - name: save cross-compiled-code + name: save gems into cache paths: - - ./tmp - key: compiled-<< pipeline.git.revision >> + - ./gems + key: gems-<< pipeline.git.revision >> workflows: test_supported_ruby_versions: diff --git a/.gitignore b/.gitignore index 8d3986f8..2e1452dd 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ misc /ports/* !/ports/patches/ test/reports +.ports_versions diff --git a/tasks/ports.rake b/tasks/ports.rake index 86ee1e91..c8ec4f75 100644 --- a/tasks/ports.rake +++ b/tasks/ports.rake @@ -7,9 +7,11 @@ require_relative 'ports/freetds' require_relative '../ext/tiny_tds/extconsts' namespace :ports do - openssl = Ports::Openssl.new(OPENSSL_VERSION) - libiconv = Ports::Libiconv.new(ICONV_VERSION) - freetds = Ports::Freetds.new(FREETDS_VERSION) + libraries_to_compile = { + freetds: Ports::Freetds.new(FREETDS_VERSION), + libiconv: Ports::Libiconv.new(ICONV_VERSION), + openssl: Ports::Openssl.new(OPENSSL_VERSION) + } directory "ports" CLEAN.include "ports/*mingw32*" @@ -18,51 +20,51 @@ namespace :ports do task :openssl, [:host] do |task, args| args.with_defaults(host: RbConfig::CONFIG['host']) - openssl.files = [OPENSSL_SOURCE_URI] - openssl.host = args.host - openssl.cook - openssl.activate + libraries_to_compile[:openssl].files = [OPENSSL_SOURCE_URI] + libraries_to_compile[:openssl].host = args.host + libraries_to_compile[:openssl].cook + libraries_to_compile[:openssl].activate end task :libiconv, [:host] do |task, args| args.with_defaults(host: RbConfig::CONFIG['host']) - libiconv.files = [ICONV_SOURCE_URI] - libiconv.host = args.host - libiconv.cook - libiconv.activate + libraries_to_compile[:libiconv].files = [ICONV_SOURCE_URI] + libraries_to_compile[:libiconv].host = args.host + libraries_to_compile[:libiconv].cook + libraries_to_compile[:libiconv].activate end task :freetds, [:host] do |task, args| args.with_defaults(host: RbConfig::CONFIG['host']) - freetds.files = [FREETDS_SOURCE_URI] - freetds.host = args.host + libraries_to_compile[:freetds].files = [FREETDS_SOURCE_URI] + libraries_to_compile[:freetds].host = args.host - if openssl + if libraries_to_compile[:openssl] # freetds doesn't have an option that will provide an rpath # so we do it manually - ENV['OPENSSL_CFLAGS'] = "-Wl,-rpath -Wl,#{openssl.path}/lib" + ENV['OPENSSL_CFLAGS'] = "-Wl,-rpath -Wl,#{libraries_to_compile[:openssl].path}/lib" # Add the pkgconfig file with MSYS2'ish path, to prefer our ports build # over MSYS2 system OpenSSL. - ENV['PKG_CONFIG_PATH'] = "#{openssl.path.gsub(/^(\w):/i){"/"+$1.downcase}}/lib/pkgconfig:#{ENV['PKG_CONFIG_PATH']}" - freetds.configure_options << "--with-openssl=#{openssl.path}" + ENV['PKG_CONFIG_PATH'] = "#{libraries_to_compile[:openssl].path.gsub(/^(\w):/i) { "/" + $1.downcase }}/lib/pkgconfig:#{ENV['PKG_CONFIG_PATH']}" + libraries_to_compile[:freetds].configure_options << "--with-openssl=#{libraries_to_compile[:openssl].path}" end - if libiconv - freetds.configure_options << "--with-libiconv-prefix=#{libiconv.path}" + if libraries_to_compile[:libiconv] + libraries_to_compile[:freetds].configure_options << "--with-libiconv-prefix=#{libraries_to_compile[:libiconv].path}" end - freetds.cook - freetds.activate + libraries_to_compile[:freetds].cook + libraries_to_compile[:freetds].activate end - task :compile, [:host] do |task,args| + task :compile, [:host] do |task, args| args.with_defaults(host: RbConfig::CONFIG['host']) puts "Compiling ports for #{args.host}..." - ['openssl','libiconv','freetds'].each do |lib| + libraries_to_compile.keys.each do |lib| Rake::Task["ports:#{lib}"].invoke(args.host) end end @@ -79,6 +81,19 @@ namespace :ports do RakeCompilerDock.sh build.join(' && '), platform: gem_platform end end + + desc "Notes the actual versions for the compiled ports into a file" + task "version_file" do + ports_version = {} + + libraries_to_compile.each do |library, library_recipe| + ports_version[library] = library_recipe.version + end + + File.open(".ports_versions", "w") do |f| + f.write ports_version + end + end end desc 'Build ports and activate libraries for the current architecture.' From 2a071ad05aabdf1c5813da1849742728910e0cfb Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Thu, 12 Jan 2023 22:47:41 +0100 Subject: [PATCH 13/16] Store precompiled gem into workspace instead of cache this seems to be the more appropriate type of data store according to the CircleCI documentation. --- .circleci/config.yml | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6abf49f9..8f6884bf 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -140,22 +140,17 @@ jobs: - ./vendor/bundle key: v1-bundle-<< parameters.ruby_version >>-{{ .Branch }}-{{ checksum "tiny_tds.gemspec" }} - - restore_cache: - name: restore cross-compiled gems - keys: - - gems-<< pipeline.git.revision >> + - attach_workspace: + at: artifacts - run: - name: restore cross-compiled code from gem + name: install native gem and restore cross-compiled code from gem command: | $Env:PATH = "C:\\Ruby<< parameters.ruby_version >>-x64\\bin;$Env:PATH" $rubyArchitecture = (ruby -e 'puts RUBY_PLATFORM').Trim() $gemVersion = (Get-Content VERSION).Trim() - # Install appropriate gem - # The project on Windows is checked out at C:/Users/circleci.PACKER-633B1A5A/project - # However, since the cache with the gems was created on Linux, it uses a different path - gem install --local --install-dir=./tmp "C:\home\circleci\project\gems\tiny_tds-$gemVersion-$rubyArchitecture.gem" + gem install --local --install-dir=./tmp "artifacts/gems/tiny_tds-$gemVersion-$rubyArchitecture.gem" # Restore precompiled code $source = (Resolve-Path ".\tmp\gems\tiny_tds-$gemVersion-$rubyArchitecture\lib\tiny_tds").Path @@ -243,11 +238,11 @@ jobs: - run: name: Move gems into separate directory before caching command: | - mkdir gems - mv pkg/*.gem gems + mkdir -p artifacts/gems + mv pkg/*.gem artifacts/gems - store_artifacts: - path: gems + path: artifacts/gems - save_cache: name: save ports cache @@ -255,11 +250,11 @@ jobs: - ./ports key: ports-{{ checksum ".ports_versions" }} - - save_cache: - name: save gems into cache + - persist_to_workspace: + name: save gems into workspace + root: artifacts paths: - - ./gems - key: gems-<< pipeline.git.revision >> + - gems workflows: test_supported_ruby_versions: From 05e66f2b1fa19522edcc1132887896a6e5f80b05 Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Fri, 13 Jan 2023 18:20:39 +0100 Subject: [PATCH 14/16] Fix order for compiling ports --- tasks/ports.rake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/ports.rake b/tasks/ports.rake index c8ec4f75..ca63e582 100644 --- a/tasks/ports.rake +++ b/tasks/ports.rake @@ -8,9 +8,9 @@ require_relative '../ext/tiny_tds/extconsts' namespace :ports do libraries_to_compile = { - freetds: Ports::Freetds.new(FREETDS_VERSION), + openssl: Ports::Openssl.new(OPENSSL_VERSION), libiconv: Ports::Libiconv.new(ICONV_VERSION), - openssl: Ports::Openssl.new(OPENSSL_VERSION) + freetds: Ports::Freetds.new(FREETDS_VERSION) } directory "ports" From 1f2bea0b25826b53c602c49916ae56c2aacd094a Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Sun, 2 Apr 2023 11:32:42 +0200 Subject: [PATCH 15/16] Disable progress bar for Invoke-WebRequest --- .circleci/config.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8f6884bf..aba1487d 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -90,6 +90,8 @@ jobs: - run: name: download and install ruby devkit command: | + $ProgressPreference='SilentlyContinue' + $uri = 'https://api.github.com/repos/oneclick/rubyinstaller2/tags?per_page=200' $releases = ((Invoke-WebRequest $uri) | ConvertFrom-Json).name | select-string -Pattern '<< parameters.ruby_version >>' $target_release = (($releases | Sort-Object -Descending)[0] | Out-String).Trim() @@ -99,6 +101,8 @@ jobs: [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 Invoke-WebRequest -UseBasicParsing -uri $download_uri -OutFile ruby-setup.exe + + echo "Download finished, starting installation of $target_version" .\ruby-setup.exe /VERYSILENT /NORESTART /DIR=C:/Ruby<< parameters.ruby_version >>-x64 - run: From bb5c9649cd3fa89a31571a4d1738faac5ff9cfc7 Mon Sep 17 00:00:00 2001 From: Andy Pfister Date: Sun, 2 Apr 2023 11:59:06 +0200 Subject: [PATCH 16/16] Add `/ALLUSERS` option to RubyInstaller Looks like a new option that has been introduced in the most recent 2.7 version and explains the timeout we see with the Ruby installation on CircleCI. https://github.com/oneclick/rubyinstaller2/wiki/FAQ#user-content-install-mode Older versions seem to ignore the parameter and run the installation as usual. --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index aba1487d..88729b5e 100755 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -103,7 +103,7 @@ jobs: Invoke-WebRequest -UseBasicParsing -uri $download_uri -OutFile ruby-setup.exe echo "Download finished, starting installation of $target_version" - .\ruby-setup.exe /VERYSILENT /NORESTART /DIR=C:/Ruby<< parameters.ruby_version >>-x64 + .\ruby-setup.exe /VERYSILENT /NORESTART /ALLUSERS /DIR=C:/Ruby<< parameters.ruby_version >>-x64 - run: name: ruby diagnostics