Merge pull request #223 from lrytz/bootstrap

Allow jenkins jobs to use an external script

Author: lrytz

attributes/default.rb

@@ -8,7 +8,7 @@
 default['repos']['private']['realm']        = "Artifactory Realm"
 default['repos']['private']['host']         = scalaCiHost
 default['repos']['private']['pr-snap']      = "https://#{scalaCiHost}/artifactory/scala-pr-validation-snapshots/"
-default['repos']['private']['release-temp'] = "https://#{scalaCiHost}/artifactory/scala-release-temp/"
+default['repos']['private']['integration']  = "https://#{scalaCiHost}/artifactory/scala-integration/"
 
 default['repos']['caching-proxy']['central']['mirrorOf'] = "central" # TODO: add ",sonatype.release,sonatype.snapshot"
 default['repos']['caching-proxy']['central']['url']      = "https://#{scalaCiHost}/artifactory/central/"

attributes/worker.rb

@@ -1,4 +1,4 @@
-default['sbt']['version'] = "0.13.12"
+default['sbt']['version'] = "0.13.13"
 
 workerJavaOpts = "-Dfile.encoding=UTF-8 -server -XX:+AggressiveOpts -XX:+UseParNewGC -Xmx2G -Xss1M -XX:MaxPermSize=512M -XX:ReservedCodeCacheSize=128M -Dpartest.threads=4"
 
@@ -51,7 +51,7 @@
     default["jenkinsHomes"][jenkinsHome]["env"]['ANT_OPTS']           = "#{workerJavaOpts} #{jvmDirOptions}"
     default["jenkinsHomes"][jenkinsHome]["env"]['MAVEN_OPTS']         = "#{workerJavaOpts} #{jvmDirOptions}"
     default["jenkinsHomes"][jenkinsHome]["env"]['prRepoUrl']          = node['repos']['private']['pr-snap']
-    default["jenkinsHomes"][jenkinsHome]["env"]['releaseTempRepoUrl'] = node['repos']['private']['release-temp']
+    default["jenkinsHomes"][jenkinsHome]["env"]['integrationRepoUrl'] = node['repos']['private']['integration']
     default["jenkinsHomes"][jenkinsHome]["env"]['PATH']               = "/bin:/usr/bin:#{jvmBin}:#{gitBin}:#{chocoBin}"
     default["jenkinsHomes"][jenkinsHome]["env"]['sbtLauncher']        = node['sbt']['launcher_path']
     default["jenkinsHomes"][jenkinsHome]["env"]['WIX']                = node['wix']['home']
@@ -98,7 +98,7 @@
     default["jenkinsHomes"][jenkinsHome]["env"]['ANT_OPTS']           = workerJavaOpts
     default["jenkinsHomes"][jenkinsHome]["env"]['MAVEN_OPTS']         = workerJavaOpts
     default["jenkinsHomes"][jenkinsHome]["env"]['prRepoUrl']          = node['repos']['private']['pr-snap']
-    default["jenkinsHomes"][jenkinsHome]["env"]['releaseTempRepoUrl'] = node['repos']['private']['release-temp']
+    default["jenkinsHomes"][jenkinsHome]["env"]['integrationRepoUrl'] = node['repos']['private']['integration']
     default["jenkinsHomes"][jenkinsHome]["env"]['sbtLauncher']        = node['sbt']['launcher_path']
     default["jenkinsHomes"][jenkinsHome]["env"]['sshCharaArgs']       = '("[email protected]" "-i" "/home/jenkins/.ssh/for_chara")'
     default["jenkinsHomes"][jenkinsHome]["env"]['sbtCmd']             = File.join(node['sbt-extras']['setup_dir'], node['sbt-extras']['script_name']) # sbt-extras

doc/genesis.md

@@ -394,7 +394,7 @@ The jenkins token for scabot has to be configured manually:
 
  - Set admin password.
  - create repos (TODO: automate)
- - Create scala-ci user that can push to scala-release-temp and scala-pr-validation-snapshots,
+ - Create scala-ci user that can push to scala-integration and scala-pr-validation-snapshots,
  - coordinate scala-ci credentials with jenkins via
 ```
 knife vault update worker-publish private-repo -J private-repo.json

externalScripts/jobs/integrate/bootstrap

@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+baseDir=${WORKSPACE-`pwd`}
+scriptsDir="$baseDir/scripts"
+. $scriptsDir/common
+
+cd $baseDir
+
+function containsCommit(){
+  local mb=$(git merge-base $1 HEAD 2> /dev/null)
+  [[ $1 != "" && $mb == $1 ]]
+}
+
+rNewVersionFormat212=18b8f932928bbc440d68babd1270c0bd5406d4fd
+rNewVersionFormat211=not-yet-backported
+function fNewVersionFormat(){
+  source $scriptsDir/jobs/integrate/bootstrap
+}
+
+function setNewScalaVersionStyle(){
+  publishToSonatype="no"
+
+  # change to use `generateBuildCharacterPropertiesFile` to get the scala version
+  local useBuildCharacter=e2b6c7b608b82fca5343f6d09236c2b48b5d824d
+  if containsCommit $useBuildCharacter; then
+    $SBT_CMD 'set baseVersionSuffix in Global := "UNUSED-SUFFIX"' generateBuildCharacterPropertiesFile
+    parseScalaProperties "buildcharacter.properties"
+    SCALA_VER_BASE=$maven_version_base
+  else
+    parseScalaProperties "build.number"
+    SCALA_VER_BASE="$version_major.$version_minor.$version_patch"
+  fi
+
+  local shaSuffix=$(git rev-parse HEAD | cut -c1-7)
+  local cross="bin"
+  if [[ $SCALA_VER_BASE =~ ^.*\.0$ ]]; then
+    cross="pre"
+  fi
+  SCALA_VER_SUFFIX="-$cross-$shaSuffix"
+}
+
+# - changed default repo from "scala-release-temp" to "scala-integration"
+# - renamed $releaseTempRepoUrl to $integrationRepoUrl
+rIntegrationRepo212=9b6820f259a7583671ae910a63d6e6d95a5e7cdf
+rIntegrationRepo211=not-yet-backported
+function fIntegrationRepo(){
+  setNewScalaVersionStyle
+  source $scriptsDir/jobs/integrate/bootstrap
+}
+
+# - introducded the $releaseTempRepoUrl variable
+rReleaseTempVarName=706d68f863ebffb0f63811afe0835c5f975ba225
+function fReleaseTempVarName(){
+  setNewScalaVersionStyle
+  releaseTempRepoUrl="https://scala-ci.typesafe.com/artifactory/scala-integration/"
+  source $scriptsDir/jobs/integrate/bootstrap
+}
+
+# TODO: support older revisions
+
+# main
+if containsCommit $rNewVersionFormat212 || containsCommit $rNewVersionFormat211; then
+  fNewVersionFormat
+
+if containsCommit $rIntegrationRepo212 || containsCommit $rIntegrationRepo211; then
+  fIntegrationRepo
+
+else if containsCommit $rReleaseTempVarName; then
+  fReleaseTempVarName
+
+else
+  echo "Cannot build revision $(git rev-parse HEAD)"
+  exit 42
+
+fi

libraries/job_blurbs.rb

@@ -123,6 +123,9 @@ def githubProject(options = {})
         jvmSelectScript=jvmSelect
       end
 
+      useExternalScript = options.fetch(:useExternalScript, false)
+      mainScript = useExternalScript ? externalScript : "source scripts/#{@scriptName}"
+
       restriction =
       """<assignedNode>%{nodes}</assignedNode>
       <canRoam>false</canRoam>""".gsub(/      /, '')
@@ -139,7 +142,7 @@ def env(name)
         <concurrentBuild>#{concurrent}</concurrentBuild>
         <builders>
           #{groovySysScript(buildNameScript)}
-          #{scriptBuild(jvmSelectScript)}
+          #{scriptBuild(jvmSelectScript, mainScript)}
         </builders>
         <buildWrappers>
           <hudson.plugins.build__timeout.BuildTimeoutWrapper plugin="[email protected]">
@@ -204,12 +207,19 @@ def jvmSelect
       EOH
     end
 
-    def scriptBuild(setup)
+    def externalScript
+      <<-EOH.gsub(/^      /, '')
+      wget https://raw.githubusercontent.com/scala/scala-jenkins-infra/master/externalScripts/#{@scriptName} -O externalScript
+      source externalScript
+      EOH
+    end
+
+    def scriptBuild(setup, mainScript)
       <<-EOH.gsub(/^      /, '')
       <hudson.tasks.Shell>
         <command>#!/bin/bash -ex
       #{setup}
-      source scripts/#{@scriptName}
+      #{mainScript}
         </command>
       </hudson.tasks.Shell>
       EOH

templates/default/jobs/scala/integrate/bootstrap.xml.erb

@@ -18,9 +18,10 @@
   buildTimeoutMinutes: 200,
   jvmVersion: @jvmVersionForBranch,
   jvmFlavor:  @jvmFlavorForBranch,
+  useExternalScript: true,
   params: [
     {:name => "SCALA_VER_BASE",
-     :desc => "Specifying this parameter will cause a release to be built. If it's empty (and HEAD does not have a tag that parses as a version), a -nightly release is built."},
+     :desc => "Specifying this parameter will cause a release to be built. If it's empty (and HEAD does not have a tag that parses as a version), an integration release is built."},
     {:name => "SCALA_VER_SUFFIX",
      :desc => "Empty for a final release, otherwise, e.g., -RC1 for the first RC or -M3 for the third milestone. (Will be parsed from the tag on SCALA_REF, if any.)"},
     {:name => "SCALA_BINARY_VER",

templates/default/jobs/scala/integrate/main.xml.erb

@@ -11,7 +11,7 @@
       {:name => "publishToSonatype",    :desc => "[For bootstrap] Set to anything but "yes" to avoid publishing to sonatype. Otherwise, release builds (tagged or SCALA_VER_BASE defined) will be published."},
       {:name => "testStability",        :desc => "[For bootstrap] To avoid running the stability test, set to e.g., `no`", :default => "yes"},
       {:name => "moduleVersioning",     :desc => "[For bootstrap] Set to anything but "versions.properties" to build the latest HEAD for all modules."},
-      {:name => "SCALA_VER_BASE",       :desc => "[For bootstrap] Specifying this parameter will cause a release to be built. If it's empty (and HEAD does not have a tag that parses as a version), a -nightly release is built."},
+      {:name => "SCALA_VER_BASE",       :desc => "[For bootstrap] Specifying this parameter will cause a release to be built. If it's empty (and HEAD does not have a tag that parses as a version), an integration release is built."},
       {:name => "SCALA_VER_SUFFIX",     :desc => "[For bootstrap] Empty for a final release, otherwise, e.g., -RC1 for the first RC or -M3 for the third milestone. (Will be parsed from the tag on SCALA_REF, if any.)"},
       {:name => "SCALA_BINARY_VER",     :desc => "[For bootstrap] For binary compatible releases (such as 2.11.0 and 2.11.1), the short binary version (e.g. 2.11). Otherwise, the full Scala version."},
       {:name => "SCALA_FULL_VER",       :desc => "[For bootstrap] For non-snapshot releases, the scala version, otherwise the closest scala version that's a valid binary version. Used to cross-version the scala continuations plugin."},

templates/default/jobs/scala/release/website/archives.xml.erb

@@ -10,7 +10,7 @@
        re-organizes them into the right hierarchy for scala-lang.org/files/archive.</p>
     <p>Also expands api docs for library, reflect, compiler and other modules
        to http://www.scala-lang.org/api/2.11.x/$module (it expands under api/$version).</p>
-    <p>When version ends in -nightly, the root for the expansion process
+    <p>When version ends in `(bin|pre)-sha`, the root for the expansion process
        is http://www.scala-lang.org/files/archive/nightly/2.11.x/
        instead of http://www.scala-lang.org/files/archive/</p>
    }.gsub(/^ {4}/, ''),

templates/default/m2-settings-public-jobs.xml.erb

@@ -26,9 +26,9 @@
           <url><%=node['repos']['private']['pr-snap']%></url>
         </repository>
         <repository>
-          <id>scala-release-temp</id>
+          <id>scala-integration</id>
           <name>Scala Release snapshots</name>
-          <url><%=node['repos']['private']['release-temp']%></url>
+          <url><%=node['repos']['private']['integration']%></url>
         </repository>
       </repositories>
     </profile>