This repository was archived by the owner on Feb 25, 2025. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 6k
Starts a .ci.yaml parser #50783
Merged
Merged
Starts a .ci.yaml parser #50783
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,221 @@ | ||
| // Copyright 2013 The Flutter Authors. All rights reserved. | ||
| // Use of this source code is governed by a BSD-style license that can be | ||
| // found in the LICENSE file. | ||
|
|
||
| import 'package:yaml/yaml.dart' as y; | ||
|
|
||
| // This file contains classes for parsing information about CI configuration | ||
| // from the .ci.yaml file at the root of the flutter/engine repository. | ||
| // The meanings of the sections and fields are documented at: | ||
| // | ||
| // https://github.com/flutter/cocoon/blob/main/CI_YAML.md | ||
| // | ||
| // The classes here don't parse every possible field, but rather only those that | ||
| // are useful for working locally in the engine repo. | ||
|
|
||
| const String _targetsField = 'targets'; | ||
| const String _nameField = 'name'; | ||
| const String _recipeField = 'recipe'; | ||
| const String _propertiesField = 'properties'; | ||
| const String _configNameField = 'config_name'; | ||
|
|
||
| /// A class containing the information deserialized from the .ci.yaml file. | ||
| /// | ||
| /// The file contains three sections. "enabled_branches", "platform_properties", | ||
| /// and "targets". The "enabled_branches" section is not meaningful when working | ||
| /// locally. The configurations listed in the "targets" section inherit | ||
| /// properties listed in the "platform_properties" section depending on their | ||
| /// names. The configurations listed in the "targets" section are the names, | ||
| /// recipes, build configs, etc. of the builders in CI. | ||
| class CiConfig { | ||
| /// Builds a [CiConfig] instance from parsed yaml data. | ||
| /// | ||
| /// If the yaml was malformed, then `CiConfig.valid` will be false, and | ||
| /// `CiConfig.error` will be populated with an informative error message. | ||
| /// Otherwise, `CiConfig.ciTargets` will contain a mapping from target name | ||
| /// to [CiTarget] instance. | ||
| factory CiConfig.fromYaml(y.YamlNode yaml) { | ||
| if (yaml is! y.YamlMap) { | ||
| final String error = yaml.span.message('Expected a map'); | ||
| return CiConfig._error(error); | ||
| } | ||
| final y.YamlMap ymap = yaml; | ||
| final y.YamlNode? targetsNode = ymap.nodes[_targetsField]; | ||
| if (targetsNode == null) { | ||
| final String error = ymap.span.message('Expected a "$_targetsField" key'); | ||
| return CiConfig._error(error); | ||
| } | ||
| if (targetsNode is! y.YamlList) { | ||
| final String error = targetsNode.span.message( | ||
| 'Expected "$_targetsField" to be a list.', | ||
| ); | ||
| return CiConfig._error(error); | ||
| } | ||
| final y.YamlList targetsList = targetsNode; | ||
|
|
||
| final Map<String, CiTarget> result = <String, CiTarget>{}; | ||
| for (final y.YamlNode yamlTarget in targetsList.nodes) { | ||
| final CiTarget target = CiTarget.fromYaml(yamlTarget); | ||
| if (!target.valid) { | ||
| return CiConfig._error(target.error); | ||
| } | ||
| result[target.name] = target; | ||
| } | ||
|
|
||
| return CiConfig._(ciTargets: result); | ||
| } | ||
|
|
||
| CiConfig._({ | ||
| required this.ciTargets, | ||
| }) : error = null; | ||
|
|
||
| CiConfig._error( | ||
| this.error, | ||
| ) : ciTargets = <String, CiTarget>{}; | ||
|
|
||
| /// Information about CI builder configurations, which .ci.yaml calls | ||
| /// "targets". | ||
| final Map<String, CiTarget> ciTargets; | ||
|
|
||
| /// An error message when this instance is invalid. | ||
| final String? error; | ||
|
|
||
| /// Whether this is a valid instance. | ||
| late final bool valid = error == null; | ||
| } | ||
|
|
||
| /// Information about the configuration of a builder on CI, which .ci.yaml | ||
| /// calls a "target". | ||
| class CiTarget { | ||
| /// Builds a [CiTarget] from parsed yaml data. | ||
| /// | ||
| /// If the yaml was malformed then `CiTarget.valid` is false and | ||
| /// `CiTarget.error` contains a useful error message. Otherwise, the other | ||
| /// fields contain information about the target. | ||
| factory CiTarget.fromYaml(y.YamlNode yaml) { | ||
| if (yaml is! y.YamlMap) { | ||
| final String error = yaml.span.message('Expected a map.'); | ||
| return CiTarget._error(error); | ||
| } | ||
| final y.YamlMap targetMap = yaml; | ||
| final String? name = _stringOfNode(targetMap.nodes[_nameField]); | ||
| if (name == null) { | ||
| final String error = targetMap.span.message( | ||
| 'Expected map to contain a string value for key "$_nameField".', | ||
| ); | ||
| return CiTarget._error(error); | ||
| } | ||
|
|
||
| final String? recipe = _stringOfNode(targetMap.nodes[_recipeField]); | ||
| if (recipe == null) { | ||
| final String error = targetMap.span.message( | ||
| 'Expected map to contain a string value for key "$_recipeField".', | ||
| ); | ||
| return CiTarget._error(error); | ||
| } | ||
|
|
||
| final y.YamlNode? propertiesNode = targetMap.nodes[_propertiesField]; | ||
| if (propertiesNode == null) { | ||
| final String error = targetMap.span.message( | ||
| 'Expected map to contain a string value for key "$_propertiesField".', | ||
| ); | ||
| return CiTarget._error(error); | ||
| } | ||
| final CiTargetProperties properties = CiTargetProperties.fromYaml( | ||
| propertiesNode, | ||
| ); | ||
| if (!properties.valid) { | ||
| return CiTarget._error(properties.error); | ||
| } | ||
|
|
||
| return CiTarget._( | ||
| name: name, | ||
| recipe: recipe, | ||
| properties: properties, | ||
| ); | ||
| } | ||
|
|
||
| CiTarget._({ | ||
| required this.name, | ||
| required this.recipe, | ||
| required this.properties, | ||
| }) : error = null; | ||
|
|
||
| CiTarget._error( | ||
| this.error, | ||
| ) : name = '', | ||
| recipe = '', | ||
| properties = CiTargetProperties._error('Invalid'); | ||
|
|
||
| /// The name of the builder in CI. | ||
| final String name; | ||
|
|
||
| /// The CI recipe used to run the build. | ||
| final String recipe; | ||
|
|
||
| /// The properties of the build or builder. | ||
| final CiTargetProperties properties; | ||
|
|
||
| /// An error message when this instance is invalid. | ||
| final String? error; | ||
|
|
||
| /// Whether this is a valid instance. | ||
| late final bool valid = error == null; | ||
| } | ||
|
|
||
| /// Various properties of a [CiTarget]. | ||
| class CiTargetProperties { | ||
| /// Builds a [CiTargetProperties] instance from parsed yaml data. | ||
| /// | ||
| /// If the yaml was malformed then `CiTargetProperties.valid` is false and | ||
| /// `CiTargetProperties.error` contains a useful error message. Otherwise, the | ||
| /// other fields contain information about the target properties. | ||
| factory CiTargetProperties.fromYaml(y.YamlNode yaml) { | ||
| if (yaml is! y.YamlMap) { | ||
| final String error = yaml.span.message( | ||
| 'Expected "$_propertiesField" to be a map.', | ||
| ); | ||
| return CiTargetProperties._error(error); | ||
| } | ||
| final y.YamlMap propertiesMap = yaml; | ||
| final String? configName = _stringOfNode( | ||
| propertiesMap.nodes[_configNameField], | ||
| ); | ||
| return CiTargetProperties._( | ||
| configName: configName ?? '', | ||
| ); | ||
| } | ||
|
|
||
| CiTargetProperties._({ | ||
| required this.configName, | ||
| }) : error = null; | ||
|
|
||
| CiTargetProperties._error( | ||
| this.error, | ||
| ) : configName = ''; | ||
|
|
||
| /// The name of the build configuration. If the containing [CiTarget] instance | ||
| /// is using the engine_v2 recipes, then this name is the same as the name | ||
| /// of the build config json file under ci/builders. | ||
| final String configName; | ||
|
|
||
| /// An error message when this instance is invalid. | ||
| final String? error; | ||
|
|
||
| /// Whether this is a valid instance. | ||
| late final bool valid = error == null; | ||
| } | ||
|
|
||
| String? _stringOfNode(y.YamlNode? stringNode) { | ||
| if (stringNode == null) { | ||
| return null; | ||
| } | ||
| if (stringNode is! y.YamlScalar) { | ||
| return null; | ||
| } | ||
| final y.YamlScalar stringScalar = stringNode; | ||
| if (stringScalar.value is! String) { | ||
| return null; | ||
| } | ||
| return stringScalar.value as String; | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
For a few months, I maintained package:cocoon_scheduler which contained Dart helpers for stuff like this used by Cocoon. I ended up removing it in flutter/cocoon#1254 as Cocoon ended up being the only customer, and it was annoying to roll the versions.
For simplicity, would you be blocked from importing the Cocoon server into the engine? We could set up a script to publish new versions to pub.dev. If it pulls in too much, we could move https://github.com/flutter/cocoon/tree/main/app_dart/lib/src/model/ci_yaml into a separate package.
\cc @keyonghan
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It depends on how extensive the transitive dependencies will be, since they would all need to be explicitly brought in via the DEPS file. If it's just a few things, then that's fine, but if the cocoon scheduler has a big list of transitive deps, then that would be a lot of trouble.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@CaseyHillers I'm having trouble finding where in https://github.com/flutter/cocoon/tree/main/app_dart/lib/src/model/ci_yaml the yaml file is parsed. It looks like the code in that directory is mostly poking around in the fields of a protobuf.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is there a way we can estimate that by importing it in pubspec via a git package?
ci.yaml is proto backed, so the pipeline looks like package:yaml -> toJson() -> package:protobuf mergeFromProtoJson -> ci_yaml.dart
https://github.com/flutter/cocoon/blob/main/app_dart/lib/src/service/scheduler.dart#L272
For Cocoon, it reads the file content over the network, then