From 9806b80d5d8b65a50c2265746ff3ab76c2cb0595 Mon Sep 17 00:00:00 2001 From: Zach Anderson Date: Sun, 18 Feb 2024 22:43:16 -0800 Subject: [PATCH] Starts a .ci.yaml parser --- .ci.yaml | 1 + tools/engine_tool/pubspec.yaml | 8 + .../engine_build_configs/lib/src/ci_yaml.dart | 221 ++++++++++++++++++ tools/pkg/engine_build_configs/pubspec.yaml | 10 + .../test/ci_yaml_test.dart | 160 +++++++++++++ 5 files changed, 400 insertions(+) create mode 100644 tools/pkg/engine_build_configs/lib/src/ci_yaml.dart create mode 100644 tools/pkg/engine_build_configs/test/ci_yaml_test.dart diff --git a/.ci.yaml b/.ci.yaml index 1ce61aeba07dc..57efcca4676d6 100644 --- a/.ci.yaml +++ b/.ci.yaml @@ -44,6 +44,7 @@ platform_properties: ] device_type: none os: Windows-10 + # The current android emulator config names can be found here: # https://chromium.googlesource.com/chromium/src.git/+/HEAD/tools/android/avd/proto # You may use those names for the android_virtual_device version. diff --git a/tools/engine_tool/pubspec.yaml b/tools/engine_tool/pubspec.yaml index 3142458e89137..1fc8f9fb25552 100644 --- a/tools/engine_tool/pubspec.yaml +++ b/tools/engine_tool/pubspec.yaml @@ -66,3 +66,11 @@ dependency_overrides: path: ../../third_party/pkg/process_runner smith: path: ../../../third_party/dart/pkg/smith + source_span: + path: ../../../third_party/dart/third_party/pkg/source_span + string_scanner: + path: ../../../third_party/dart/third_party/pkg/string_scanner + term_glyph: + path: ../../../third_party/dart/third_party/pkg/term_glyph + yaml: + path: ../../../third_party/dart/third_party/pkg/yaml diff --git a/tools/pkg/engine_build_configs/lib/src/ci_yaml.dart b/tools/pkg/engine_build_configs/lib/src/ci_yaml.dart new file mode 100644 index 0000000000000..94a2e36ee055e --- /dev/null +++ b/tools/pkg/engine_build_configs/lib/src/ci_yaml.dart @@ -0,0 +1,221 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:yaml/yaml.dart' as y; + +// This file contains classes for parsing information about CI configuration +// from the .ci.yaml file at the root of the flutter/engine repository. +// The meanings of the sections and fields are documented at: +// +// https://github.com/flutter/cocoon/blob/main/CI_YAML.md +// +// The classes here don't parse every possible field, but rather only those that +// are useful for working locally in the engine repo. + +const String _targetsField = 'targets'; +const String _nameField = 'name'; +const String _recipeField = 'recipe'; +const String _propertiesField = 'properties'; +const String _configNameField = 'config_name'; + +/// A class containing the information deserialized from the .ci.yaml file. +/// +/// The file contains three sections. "enabled_branches", "platform_properties", +/// and "targets". The "enabled_branches" section is not meaningful when working +/// locally. The configurations listed in the "targets" section inherit +/// properties listed in the "platform_properties" section depending on their +/// names. The configurations listed in the "targets" section are the names, +/// recipes, build configs, etc. of the builders in CI. +class CiConfig { + /// Builds a [CiConfig] instance from parsed yaml data. + /// + /// If the yaml was malformed, then `CiConfig.valid` will be false, and + /// `CiConfig.error` will be populated with an informative error message. + /// Otherwise, `CiConfig.ciTargets` will contain a mapping from target name + /// to [CiTarget] instance. + factory CiConfig.fromYaml(y.YamlNode yaml) { + if (yaml is! y.YamlMap) { + final String error = yaml.span.message('Expected a map'); + return CiConfig._error(error); + } + final y.YamlMap ymap = yaml; + final y.YamlNode? targetsNode = ymap.nodes[_targetsField]; + if (targetsNode == null) { + final String error = ymap.span.message('Expected a "$_targetsField" key'); + return CiConfig._error(error); + } + if (targetsNode is! y.YamlList) { + final String error = targetsNode.span.message( + 'Expected "$_targetsField" to be a list.', + ); + return CiConfig._error(error); + } + final y.YamlList targetsList = targetsNode; + + final Map result = {}; + for (final y.YamlNode yamlTarget in targetsList.nodes) { + final CiTarget target = CiTarget.fromYaml(yamlTarget); + if (!target.valid) { + return CiConfig._error(target.error); + } + result[target.name] = target; + } + + return CiConfig._(ciTargets: result); + } + + CiConfig._({ + required this.ciTargets, + }) : error = null; + + CiConfig._error( + this.error, + ) : ciTargets = {}; + + /// Information about CI builder configurations, which .ci.yaml calls + /// "targets". + final Map ciTargets; + + /// An error message when this instance is invalid. + final String? error; + + /// Whether this is a valid instance. + late final bool valid = error == null; +} + +/// Information about the configuration of a builder on CI, which .ci.yaml +/// calls a "target". +class CiTarget { + /// Builds a [CiTarget] from parsed yaml data. + /// + /// If the yaml was malformed then `CiTarget.valid` is false and + /// `CiTarget.error` contains a useful error message. Otherwise, the other + /// fields contain information about the target. + factory CiTarget.fromYaml(y.YamlNode yaml) { + if (yaml is! y.YamlMap) { + final String error = yaml.span.message('Expected a map.'); + return CiTarget._error(error); + } + final y.YamlMap targetMap = yaml; + final String? name = _stringOfNode(targetMap.nodes[_nameField]); + if (name == null) { + final String error = targetMap.span.message( + 'Expected map to contain a string value for key "$_nameField".', + ); + return CiTarget._error(error); + } + + final String? recipe = _stringOfNode(targetMap.nodes[_recipeField]); + if (recipe == null) { + final String error = targetMap.span.message( + 'Expected map to contain a string value for key "$_recipeField".', + ); + return CiTarget._error(error); + } + + final y.YamlNode? propertiesNode = targetMap.nodes[_propertiesField]; + if (propertiesNode == null) { + final String error = targetMap.span.message( + 'Expected map to contain a string value for key "$_propertiesField".', + ); + return CiTarget._error(error); + } + final CiTargetProperties properties = CiTargetProperties.fromYaml( + propertiesNode, + ); + if (!properties.valid) { + return CiTarget._error(properties.error); + } + + return CiTarget._( + name: name, + recipe: recipe, + properties: properties, + ); + } + + CiTarget._({ + required this.name, + required this.recipe, + required this.properties, + }) : error = null; + + CiTarget._error( + this.error, + ) : name = '', + recipe = '', + properties = CiTargetProperties._error('Invalid'); + + /// The name of the builder in CI. + final String name; + + /// The CI recipe used to run the build. + final String recipe; + + /// The properties of the build or builder. + final CiTargetProperties properties; + + /// An error message when this instance is invalid. + final String? error; + + /// Whether this is a valid instance. + late final bool valid = error == null; +} + +/// Various properties of a [CiTarget]. +class CiTargetProperties { + /// Builds a [CiTargetProperties] instance from parsed yaml data. + /// + /// If the yaml was malformed then `CiTargetProperties.valid` is false and + /// `CiTargetProperties.error` contains a useful error message. Otherwise, the + /// other fields contain information about the target properties. + factory CiTargetProperties.fromYaml(y.YamlNode yaml) { + if (yaml is! y.YamlMap) { + final String error = yaml.span.message( + 'Expected "$_propertiesField" to be a map.', + ); + return CiTargetProperties._error(error); + } + final y.YamlMap propertiesMap = yaml; + final String? configName = _stringOfNode( + propertiesMap.nodes[_configNameField], + ); + return CiTargetProperties._( + configName: configName ?? '', + ); + } + + CiTargetProperties._({ + required this.configName, + }) : error = null; + + CiTargetProperties._error( + this.error, + ) : configName = ''; + + /// The name of the build configuration. If the containing [CiTarget] instance + /// is using the engine_v2 recipes, then this name is the same as the name + /// of the build config json file under ci/builders. + final String configName; + + /// An error message when this instance is invalid. + final String? error; + + /// Whether this is a valid instance. + late final bool valid = error == null; +} + +String? _stringOfNode(y.YamlNode? stringNode) { + if (stringNode == null) { + return null; + } + if (stringNode is! y.YamlScalar) { + return null; + } + final y.YamlScalar stringScalar = stringNode; + if (stringScalar.value is! String) { + return null; + } + return stringScalar.value as String; +} diff --git a/tools/pkg/engine_build_configs/pubspec.yaml b/tools/pkg/engine_build_configs/pubspec.yaml index e357389521966..07297d29dd9e9 100644 --- a/tools/pkg/engine_build_configs/pubspec.yaml +++ b/tools/pkg/engine_build_configs/pubspec.yaml @@ -25,6 +25,7 @@ dependencies: path: any platform: any process_runner: any + yaml: any dev_dependencies: async_helper: any @@ -33,6 +34,7 @@ dev_dependencies: process_fakes: path: ../process_fakes smith: any + source_span: any dependency_overrides: args: @@ -61,3 +63,11 @@ dependency_overrides: path: ../../../third_party/pkg/process_runner smith: path: ../../../../third_party/dart/pkg/smith + source_span: + path: ../../../../third_party/dart/third_party/pkg/source_span + string_scanner: + path: ../../../../third_party/dart/third_party/pkg/string_scanner + term_glyph: + path: ../../../../third_party/dart/third_party/pkg/term_glyph + yaml: + path: ../../../../third_party/dart/third_party/pkg/yaml diff --git a/tools/pkg/engine_build_configs/test/ci_yaml_test.dart b/tools/pkg/engine_build_configs/test/ci_yaml_test.dart new file mode 100644 index 0000000000000..df5e4489ca2d3 --- /dev/null +++ b/tools/pkg/engine_build_configs/test/ci_yaml_test.dart @@ -0,0 +1,160 @@ +// Copyright 2013 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io' as io; + +import 'package:engine_build_configs/src/ci_yaml.dart'; +import 'package:engine_repo_tools/engine_repo_tools.dart'; +import 'package:litetest/litetest.dart'; +import 'package:path/path.dart' as path; +import 'package:source_span/source_span.dart'; +import 'package:yaml/yaml.dart' as y; + +void main() { + y.yamlWarningCallback = (String message, [SourceSpan? span]) {}; + + // Find the engine repo. + final Engine engine; + try { + engine = Engine.findWithin(); + } catch (e) { + io.stderr.writeln(e); + io.exitCode = 1; + return; + } + + final String ciYamlPath = path.join(engine.flutterDir.path, '.ci.yaml'); + final String realCiYaml = io.File(ciYamlPath).readAsStringSync(); + + test('Can load the real .ci.yaml file', () { + final y.YamlNode yamlNode = y.loadYamlNode( + realCiYaml, sourceUrl: Uri.file(ciYamlPath), + ); + final CiConfig config = CiConfig.fromYaml(yamlNode); + if (!config.valid) { + io.stderr.writeln(config.error); + } + expect(config.valid, isTrue); + }); + + test('Parses all supported fields', () { + const String yamlData = ''' +targets: + - name: Linux linux_build + recipe: engine_v2/engine_v2 + properties: + config_name: linux_build +'''; + final y.YamlNode yamlNode = y.loadYamlNode( + yamlData, sourceUrl: Uri.file(ciYamlPath), + ); + final CiConfig config = CiConfig.fromYaml(yamlNode); + if (!config.valid) { + io.stderr.writeln(config.error); + } + expect(config.valid, isTrue); + expect(config.ciTargets.entries.isNotEmpty, isTrue); + expect(config.ciTargets['Linux linux_build'], isNotNull); + expect(config.ciTargets['Linux linux_build']!.valid, isTrue); + expect(config.ciTargets['Linux linux_build']!.name, equals('Linux linux_build')); + expect(config.ciTargets['Linux linux_build']!.recipe, equals('engine_v2/engine_v2')); + expect(config.ciTargets['Linux linux_build']!.properties.valid, isTrue); + expect(config.ciTargets['Linux linux_build']!.properties.configName, equals('linux_build')); + }); + + test('Invalid when targets is malformed', () { + const String yamlData = ''' +targets: 4 +'''; + final y.YamlNode yamlNode = y.loadYamlNode( + yamlData, sourceUrl: Uri.file(ciYamlPath), + ); + final CiConfig config = CiConfig.fromYaml(yamlNode); + expect(config.valid, isFalse); + expect(config.error, contains('Expected "targets" to be a list.')); + }); + + test('Invalid when a target is malformed', () { + const String yamlData = ''' +targets: + - name: 4 + recipe: engine_v2/engine_v2 + properties: + config_name: linux_build +'''; + final y.YamlNode yamlNode = y.loadYamlNode( + yamlData, sourceUrl: Uri.file(ciYamlPath), + ); + final CiConfig config = CiConfig.fromYaml(yamlNode); + expect(config.valid, isFalse); + expect(config.error, contains('Expected map to contain a string value for key "name".')); + }); + + test('Invalid when a recipe is malformed', () { + const String yamlData = ''' +targets: + - name: Linux linux_build + recipe: 4 + properties: + config_name: linux_build +'''; + final y.YamlNode yamlNode = y.loadYamlNode( + yamlData, sourceUrl: Uri.file(ciYamlPath), + ); + final CiConfig config = CiConfig.fromYaml(yamlNode); + expect(config.valid, isFalse); + expect(config.error, contains('Expected map to contain a string value for key "recipe".')); + }); + + test('Invalid when a properties list is malformed', () { + const String yamlData = ''' +targets: + - name: Linux linux_build + recipe: engine_v2/engine_v2 + properties: 4 +'''; + final y.YamlNode yamlNode = y.loadYamlNode( + yamlData, sourceUrl: Uri.file(ciYamlPath), + ); + final CiConfig config = CiConfig.fromYaml(yamlNode); + expect(config.valid, isFalse); + expect(config.error, contains('Expected "properties" to be a map.')); + }); + + test('Still valid when a config_name is not present', () { + const String yamlData = ''' +targets: + - name: Linux linux_build + recipe: engine_v2/engine_v2 + properties: + field: value +'''; + final y.YamlNode yamlNode = y.loadYamlNode( + yamlData, sourceUrl: Uri.file(ciYamlPath), + ); + final CiConfig config = CiConfig.fromYaml(yamlNode); + expect(config.valid, isTrue); + }); + + test('Invalid when any target is malformed', () { + const String yamlData = ''' +targets: + - name: Linux linux_build + recipe: engine_v2/engine_v2 + properties: + config_name: linux_build + + - name: 4 + recipe: engine_v2/engine_v2 + properties: + config_name: linux_build +'''; + final y.YamlNode yamlNode = y.loadYamlNode( + yamlData, sourceUrl: Uri.file(ciYamlPath), + ); + final CiConfig config = CiConfig.fromYaml(yamlNode); + expect(config.valid, isFalse); + expect(config.error, contains('Expected map to contain a string value for key "name".')); + }); +}