[Backport 7.x] Introduce automated scheduled codegeneration (#5449)

Co-authored-by: Martijn Laarman <[email protected]>

Author: github-actions[bot]

.ci/make.sh

@@ -42,6 +42,11 @@ case $CMD in
         TASK=release
         TASK_ARGS=("$VERSION" "$output_folder" "skiptests")
         ;;
+    codegen)
+        TASK=codegen
+        # VERSION is BRANCH here for now
+        TASK_ARGS=("$VERSION") 
+        ;;
     *)
         echo -e "\nUsage:\n\t $CMD is not supported right now\n"
         exit 1

.github/workflows/make-codegen.yml

@@ -0,0 +1,47 @@
+name: Code Generation 
+on:
+  schedule:
+    - cron: "0 0/2 * * *"
+
+
+jobs:
+  active-branches:
+    runs-on: ubuntu-latest
+
+    steps:
+      - id: dump-branches
+        run: |
+          curl -s "https://artifacts-api.elastic.co/v1/branches"  | jq "[ .branches[] | select(. | startswith(\"6\") | not) ]" --compact-output
+          curl -s "https://artifacts-api.elastic.co/v1/branches"  | jq "[ .branches[] | select(. | startswith(\"6\") | not) ]" --compact-output > branches.json
+      - id: set-matrix
+        run: echo "::set-output name=matrix::$(cat branches.json)"
+    outputs:
+      matrix: ${{ steps.set-matrix.outputs.matrix }}
+      
+  codegen:
+    name: Sync
+    needs: active-branches
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        branch: ${{ fromJson(needs.active-branches.outputs.matrix) }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          ref: "${{ matrix.branch }}"
+      - run: "./.ci/make.sh codegen ${{ matrix.branch }}"
+        name: "code-gen ${{ matrix.branch }}"
+      - name: "PR ${{ matrix.branch }}"
+        # fixate to known release. 
+        uses: peter-evans/create-pull-request@5ea31358e901bfaf28ca19849903d802fd0a891b
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          branch: "fix/${{ matrix.branch }}/code-gen"
+          delete-branch: true
+          base: "${{ matrix.branch }}"
+          commit-message: "[codegen] ${{ matrix.branch }} synchronization" 
+          title: '[codegen] ${{ matrix.branch }} synchronization'
+          body: |
+            This synchronizes the client's ${{ matrix.branch }} branch with the server. 
+          labels: "infra,code-gen"

build/scripts/Commandline.fs

@@ -39,8 +39,9 @@ Targets:
 * benchmark [non-interactive] [url] [username] [password] 
   - Runs a benchmark from Tests.Benchmarking and indexes the results to [url] when provided.
     If non-interactive runs all benchmarks without prompting
-* codegen
-  - runs the code generator interactively
+* codegen <branch> [arguments]
+  - runs the code generator, the first argument is required and dictates the branch to sync from.
+  - Any other arguments are passed down to the tool directly
 * documentation
   - runs the doc generation, without running any tests
   
@@ -88,13 +89,15 @@ Execution hints can be provided anywhere on the command line
 
     type BenchmarkArguments = { Endpoint: string; Username: string option; Password: string option; }
     type ClusterArguments = { Name: string; Version: string option; }
+    type CodeGenArguments = { Branch: string; }
     type CommandArguments =
         | Unknown
         | SetVersion of VersionArguments
         | Test of TestArguments
         | Integration of IntegrationArguments
         | Benchmark of BenchmarkArguments
         | Cluster of ClusterArguments
+        | CodeGen of CodeGenArguments
 
     type PassedArguments = {
         NonInteractive: bool;
@@ -193,11 +196,13 @@ Execution hints can be provided anywhere on the command line
         | ["build"]
         | ["clean"]
         | ["benchmark"]
-        | ["codegen"; ] 
         | ["documentation"; ] 
         | ["profile"] -> parsed
         | "rest-spec-tests" :: tail -> { parsed with RemainingArguments = tail }
 
+        | "codegen" :: branch :: tail  ->
+            { parsed with CommandArguments = CodeGen { Branch = branch }; RemainingArguments = tail }
+        
         | ["release"; version] -> { parsed with CommandArguments = SetVersion { Version = version; OutputLocation = None }; }
         | ["release"; version; path] ->
             if (not <| System.IO.Directory.Exists path) then failwithf "'%s' is not an existing directory" (Path.getFullName path)

build/scripts/ReposTooling.fs

@@ -29,13 +29,13 @@ module ReposTooling =
 
         Shell.deleteDir tempDir
 
-    let GenerateApi () =
-        //TODO allow branch name to be passed for CI
+    let GenerateApi args =
+        let branch = match args.CommandArguments with | CodeGen a -> a.Branch | _ -> raise <| Exception("Branch is required")
         let folder = Path.getDirectory (Paths.ProjFile "ApiGenerator")
         let timeout = TimeSpan.FromMinutes(120.)
-        // Building to make sure XML docs files are there, faster then relying on the ApiGenerator to emit these
-        // from a compilation unit
-        Tooling.DotNet.ExecInWithTimeout folder ["run"; "-c"; " Release"; ] timeout  |> ignore
+        Tooling.DotNet.ExecInWithTimeout folder
+            (["run"; "-c"; " Release"; "--"; "--branch"; branch; "--download"] @ args.RemainingArguments) timeout
+            |> ignore
 
     let RestSpecTests args =
         let folder = Path.getDirectory (Paths.TestProjFile "Tests.YamlRunner")

build/scripts/Targets.fs

@@ -109,7 +109,7 @@ module Main =
         command "cluster" [ "restore"; "full-build" ] <| fun _ ->
             ReposTooling.LaunchCluster parsed
 
-        command "codegen" [ ] ReposTooling.GenerateApi
+        command "codegen" [ ] <| fun _ -> ReposTooling.GenerateApi parsed
 
         command "rest-spec-tests" [ ] <| fun _ ->
             ReposTooling.RestSpecTests parsed.RemainingArguments

build/scripts/scripts.fsproj

@@ -31,6 +31,9 @@
     <Content Include="..\..\.github\workflows\unified-release.yml">
       <Link>unified-release.yml</Link>
     </Content>
+    <Content Include="..\..\.github\workflows\make-codegen.yml">
+      <Link>make-codegen.yml</Link>
+    </Content>
   </ItemGroup>
   <ItemGroup>
     <PackageReference Include="FSharp.Core" Version="5.0.0" />

src/ApiGenerator/Configuration/GeneratorLocations.cs

@@ -31,6 +31,14 @@ public static string Root
 
 				var directoryInfo = new DirectoryInfo(Directory.GetCurrentDirectory());
 
+				//If we run this tool using dotnet run --project src/ApiGenerator
+				var pathFromSolutionRoot = Path.Combine(directoryInfo.FullName, "src", "ApiGenerator");
+				if (Directory.Exists(pathFromSolutionRoot))
+				{
+					_root = pathFromSolutionRoot + "/";
+					return _root;
+				}
+
 				var dotnetRun =
 					directoryInfo.Name == "ApiGenerator" &&
 					directoryInfo.Parent != null &&

src/ApiGenerator/Configuration/ViewLocations.cs

@@ -2,15 +2,15 @@
 // Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
 // See the LICENSE file in the project root for more information
 
-namespace ApiGenerator.Configuration 
+namespace ApiGenerator.Configuration
 {
 	public static class ViewLocations
 	{
 		public static string Root { get; } = $@"{GeneratorLocations.Root}Views/";
 		private static string HighLevelRoot { get; } = $@"{Root}/HighLevel/";
 		public static string HighLevel(params string[] paths) => HighLevelRoot + string.Join("/", paths);
-		
+
 		private static string LowLevelRoot { get; } = $@"{Root}/LowLevel/";
 		public static string LowLevel(params string[] paths) => LowLevelRoot + string.Join("/", paths);
 	}
-}
+}