[ci.yaml] Support branch regexes

CI_YAML.md

@@ -15,8 +15,13 @@ are welcome.
 Example config:
 ```yaml
 # /.ci.yaml
+
+# Enabled branches is a list of regexes, with the assumption that these are full line matches.
+# Internally, Cocoon prefixes these with $ and suffixes with ^ to enable matches.
 enabled_branches:
-  - master
+  - main
+  - flutter-\\d+\\.\\d+-candidate\\.\\d+
+
 targets:
 # A Target is an individual unit of work that is scheduled by Flutter infra
 # Target's are composed of the following properties:

app_dart/integration_test/validate_all_ci_configs_test.dart

@@ -2,9 +2,15 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:core';
+import 'dart:io';
+
 import 'package:cocoon_service/cocoon_service.dart';
+import 'package:cocoon_service/src/model/ci_yaml/ci_yaml.dart';
+import 'package:cocoon_service/src/model/proto/internal/scheduler.pbserver.dart' as pb;
 import 'package:github/github.dart';
 import 'package:http/http.dart' as http;
+import 'package:process/process.dart';
 import 'package:test/test.dart';
 import 'package:yaml/yaml.dart';
 
@@ -13,7 +19,7 @@ import 'common.dart';
 /// List of repositories that have supported .ci.yaml config files.
 final List<SupportedConfig> configs = <SupportedConfig>[
   SupportedConfig(RepositorySlug('flutter', 'cocoon'), 'main'),
-  SupportedConfig(RepositorySlug('flutter', 'engine')),
+  SupportedConfig(RepositorySlug('flutter', 'engine'), 'main'),
   SupportedConfig(RepositorySlug('flutter', 'flutter')),
   SupportedConfig(RepositorySlug('flutter', 'packages')),
   SupportedConfig(RepositorySlug('flutter', 'plugins')),
@@ -35,5 +41,72 @@ Future<void> main() async {
         fail(e.message);
       }
     });
+
+    test('validate enabled branches of $config', () async {
+      final String configContent = await githubFileContent(
+        config.slug,
+        kCiYamlPath,
+        httpClientProvider: () => http.Client(),
+        ref: config.branch,
+      );
+      final YamlMap configYaml = loadYaml(configContent) as YamlMap;
+      final pb.SchedulerConfig schedulerConfig = schedulerConfigFromYaml(configYaml);
+      final List<String> githubBranches = getBranchesForRepository(config.slug);
+
+      final Map<String, bool> validEnabledBranches = <String, bool>{};
+      // Add config wide enabled branches
+      for (String enabledBranch in schedulerConfig.enabledBranches) {
+        validEnabledBranches[enabledBranch] = false;
+      }
+      // Add all target specific enabled branches
+      for (pb.Target target in schedulerConfig.targets) {
+        for (String enabledBranch in target.enabledBranches) {
+          validEnabledBranches[enabledBranch] = false;
+        }
+      }
+
+      // N^2 scan to verify all enabled branch patterns match an exist branch on the repo.
+      for (String enabledBranch in validEnabledBranches.keys) {
+        for (String githubBranch in githubBranches) {
+          if (CiYaml.enabledBranchesMatchesCurrentBranch(<String>[enabledBranch], githubBranch)) {
+            validEnabledBranches[enabledBranch] = true;
+          }
+        }
+      }
+
+      if (config.slug.name == 'engine') {
+        print(githubBranches);
+        print(validEnabledBranches);
+      }
+
+      // Verify the enabled branches
+      for (String enabledBranch in validEnabledBranches.keys) {
+        expect(validEnabledBranches[enabledBranch], isTrue,
+            reason: '$enabledBranch does not match to a branch in ${config.slug.fullName}');
+      }
+    }, skip: config.slug.name == 'flutter');
   }
 }
+
+/// Gets all branches for [slug].
+///
+/// Internally, uses the git on path to get the branches from the remote for [slug].
+List<String> getBranchesForRepository(RepositorySlug slug) {
+  const ProcessManager processManager = LocalProcessManager();
+  final ProcessResult result =
+      processManager.runSync(<String>['git', 'ls-remote', '--head', 'https://github.com/${slug.fullName}']);
+  final List<String> lines = (result.stdout as String).split('\n');
+
+  final List<String> githubBranches = <String>[];
+  for (String line in lines) {
+    if (line.isEmpty) {
+      continue;
+    }
+    // Lines follow the format of `$sha\t$ref`
+    final String ref = line.split('\t')[1];
+    final String branch = ref.replaceAll('refs/heads/', '');
+    githubBranches.add(branch);
+  }
+
+  return githubBranches;
+}

app_dart/lib/src/model/ci_yaml/ci_yaml.dart

@@ -28,13 +28,14 @@ class CiYaml {
 
   /// Gets all [Target] that run on presubmit for this config.
   List<Target> get presubmitTargets {
-    if (!config.enabledBranches.contains(branch)) {
-      throw Exception('$branch is not enabled for this .ci.yaml.\nAdd it to run tests against this PR.');
-    }
     final Iterable<Target> presubmitTargets =
         _targets.where((Target target) => target.value.presubmit && !target.value.bringup);
 
-    return _filterEnabledTargets(presubmitTargets);
+    final List<Target> enabledTargets = _filterEnabledTargets(presubmitTargets);
+    if (enabledTargets.isEmpty) {
+      throw Exception('$branch is not enabled for this .ci.yaml.\nAdd it to run tests against this PR.');
+    }
+    return enabledTargets;
   }
 
   /// Gets all [Target] that run on postsubmit for this config.
@@ -62,25 +63,38 @@ class CiYaml {
   /// Filter [targets] to only those that are expected to run for [branch].
   ///
   /// A [Target] is expected to run if:
-  ///   1. [Target.enabledBranches] exists and contains [branch].
-  ///   2. Otherwise, [config.enabledBranches] contains [branch].
+  ///   1. [Target.enabledBranches] exists and matches [branch].
+  ///   2. Otherwise, [config.enabledBranches] matches [branch].
   List<Target> _filterEnabledTargets(Iterable<Target> targets) {
     final List<Target> filteredTargets = <Target>[];
 
     // 1. Add targets with local definition
     final Iterable<Target> overrideBranchTargets =
         targets.where((Target target) => target.value.enabledBranches.isNotEmpty);
-    final Iterable<Target> enabledTargets =
-        overrideBranchTargets.where((Target target) => target.value.enabledBranches.contains(branch));
+    final Iterable<Target> enabledTargets = overrideBranchTargets
+        .where((Target target) => enabledBranchesMatchesCurrentBranch(target.value.enabledBranches, branch));
     filteredTargets.addAll(enabledTargets);
 
     // 2. Add targets with global definition (this is the majority of targets)
-    if (config.enabledBranches.contains(branch)) {
+    if (enabledBranchesMatchesCurrentBranch(config.enabledBranches, branch)) {
       final Iterable<Target> defaultBranchTargets =
           targets.where((Target target) => target.value.enabledBranches.isEmpty);
       filteredTargets.addAll(defaultBranchTargets);
     }
 
     return filteredTargets;
   }
+
+  /// Whether any of the possible [RegExp] in [enabledBranches] match [branch].
+  static bool enabledBranchesMatchesCurrentBranch(List<String> enabledBranches, String branch) {
+    final List<String> regexes = <String>[];
+    for (String enabledBranch in enabledBranches) {
+      // Prefix with start of line and suffix with end of line
+      regexes.add('^$enabledBranch\$');
+    }
+    final String rawRegexp = regexes.join('|');
+    final RegExp regexp = RegExp(rawRegexp);
+
+    return regexp.hasMatch(branch);
+  }
 }

app_dart/lib/src/service/scheduler.dart

@@ -244,8 +244,13 @@ class Scheduler {
   Future<void> triggerPresubmitTargets({
     required github.PullRequest pullRequest,
     String reason = 'Newer commit available',
-    RetryOptions retryOptions = const RetryOptions(maxAttempts: 3),
   }) async {
+    // Always cancel running builds so we don't ever schedule duplicates.
+    log.fine('about to cancel presubmit targets');
+    await cancelPreSubmitTargets(
+      pullRequest: pullRequest,
+      reason: reason,
+    );
     final github.CheckRun ciValidationCheckRun = await githubChecksService.githubChecksUtil.createCheckRun(
       config,
       pullRequest.base!.repo!.slug(),
@@ -257,10 +262,20 @@ class Scheduler {
       ),
     );
     final github.RepositorySlug slug = pullRequest.base!.repo!.slug();
-    final dynamic exception = await retryOptions.retry<dynamic>(() async => _triggerPresubmitTargets(
-          pullRequest: pullRequest,
-          reason: reason,
-        ));
+    dynamic exception;
+    try {
+      final List<Target> presubmitTargets = await getPresubmitTargets(pullRequest);
+      await luciBuildService.scheduleTryBuilds(
+        targets: presubmitTargets,
+        pullRequest: pullRequest,
+      );
+    } on FormatException catch (error, backtrace) {
+      log.warning(backtrace.toString());
+      exception = error;
+    } catch (error, backtrace) {
+      log.warning(backtrace.toString());
+      exception = error;
+    }
 
     // Update validate ci.yaml check
     if (exception == null) {
@@ -293,35 +308,6 @@ class Scheduler {
         'Finished triggering builds for: pr ${pullRequest.number}, commit ${pullRequest.base!.sha}, branch ${pullRequest.head!.ref} and slug ${pullRequest.base!.repo!.slug()}}');
   }
 
-  /// Internal wrapper for [triggerPresubmitTargets] retry logic.
-  Future<dynamic> _triggerPresubmitTargets({
-    required github.PullRequest pullRequest,
-    String reason = 'Newer commit available',
-  }) async {
-    dynamic exception;
-    // Always cancel running builds so we don't ever schedule duplicates.
-    await cancelPreSubmitTargets(
-      pullRequest: pullRequest,
-      reason: reason,
-    );
-    log.fine('Cancelled existing presubmit runs');
-    try {
-      final List<Target> presubmitTargets = await getPresubmitTargets(pullRequest);
-      await luciBuildService.scheduleTryBuilds(
-        targets: presubmitTargets,
-        pullRequest: pullRequest,
-      );
-    } on FormatException catch (error, backtrace) {
-      log.warning(backtrace.toString());
-      exception = error;
-    } catch (error, backtrace) {
-      log.warning(backtrace.toString());
-      exception = error;
-    }
-
-    return exception;
-  }
-
   /// Given a pull request event, retry all failed LUCI checks.
   ///
   /// 1. Aggregate .ci.yaml and try_builders.json presubmit builds.

app_dart/pubspec.lock

@@ -477,6 +477,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "1.11.1"
+  platform:
+    dependency: transitive
+    description:
+      name: platform
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "3.0.2"
   pointycastle:
     dependency: transitive
     description:
@@ -491,6 +498,13 @@ packages:
       url: "https://pub.dartlang.org"
     source: hosted
     version: "1.5.0"
+  process:
+    dependency: "direct dev"
+    description:
+      name: process
+      url: "https://pub.dartlang.org"
+    source: hosted
+    version: "4.2.4"
   protobuf:
     dependency: "direct main"
     description:

app_dart/pubspec.yaml

@@ -37,6 +37,7 @@ dev_dependencies:
   json_serializable: ^6.0.0
   mockito: ^5.0.14
   pedantic: ^1.11.1
+  process: ^4.2.4
   test: ^1.17.11
 
 builders:

app_dart/test/model/ci_yaml/ci_yaml_test.dart

@@ -0,0 +1,35 @@
+// Copyright 2019 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:cocoon_service/src/model/ci_yaml/ci_yaml.dart';
+import 'package:test/test.dart';
+
+void main() {
+  group('enabledBranchesMatchesCurrentBranch', () {
+    final List<EnabledBranchesRegexTest> tests = <EnabledBranchesRegexTest>[
+      EnabledBranchesRegexTest('matches main', 'main', <String>['main']),
+      EnabledBranchesRegexTest(
+          'matches candidate branch', 'flutter-2.4-candidate.3', <String>['flutter-\\d+\\.\\d+-candidate\\.\\d+']),
+      EnabledBranchesRegexTest('matches main when not first pattern', 'main', <String>['dev', 'main']),
+      EnabledBranchesRegexTest('does not do partial matches', 'super-main', <String>['main'], false),
+    ];
+
+    for (EnabledBranchesRegexTest regexTest in tests) {
+      test(regexTest.name, () {
+        expect(CiYaml.enabledBranchesMatchesCurrentBranch(regexTest.enabledBranches, regexTest.branch),
+            regexTest.expectation);
+      });
+    }
+  });
+}
+
+/// Wrapper class for table driven design of [CiYaml.enabledBranchesMatchesCurrentBranch].
+class EnabledBranchesRegexTest {
+  EnabledBranchesRegexTest(this.name, this.branch, this.enabledBranches, [this.expectation = true]);
+
+  final String branch;
+  final List<String> enabledBranches;
+  final String name;
+  final bool expectation;
+}