Skip to content
This repository was archived by the owner on Feb 25, 2025. It is now read-only.

Commit 64ae66a

Browse files
matanlureymatanlurey
authored
Add a build_bucket_golden_scraper tool. (#45243)
As discussed offline, this is best deleted when Skia-gold is used for all of our engine tests. However, this will be useful for unblocking some PRs until then :) See README.md for details!
1 parent 73e8636 commit 64ae66a

File tree

7 files changed

+453
-0
lines changed

7 files changed

+453
-0
lines changed

‎testing/run_tests.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -953,6 +953,25 @@ def gather_clang_tidy_tests(build_dir):
953953
)
954954

955955

956+
def gather_build_bucket_golden_scraper_tests(build_dir):
957+
test_dir = os.path.join(
958+
BUILDROOT_DIR, 'flutter', 'tools', 'build_bucket_golden_scraper'
959+
)
960+
dart_tests = glob.glob('%s/test/*_test.dart' % test_dir)
961+
for dart_test_file in dart_tests:
962+
opts = [
963+
'--disable-dart-dev',
964+
dart_test_file,
965+
]
966+
yield EngineExecutableTask(
967+
build_dir,
968+
os.path.join('dart-sdk', 'bin', 'dart'),
969+
None,
970+
flags=opts,
971+
cwd=test_dir
972+
)
973+
974+
956975
def gather_engine_repo_tools_tests(build_dir):
957976
test_dir = os.path.join(
958977
BUILDROOT_DIR, 'flutter', 'tools', 'pkg', 'engine_repo_tools'
@@ -1249,6 +1268,7 @@ def main():
12491268
tasks += list(gather_litetest_tests(build_dir))
12501269
tasks += list(gather_githooks_tests(build_dir))
12511270
tasks += list(gather_clang_tidy_tests(build_dir))
1271+
tasks += list(gather_build_bucket_golden_scraper_tests(build_dir))
12521272
tasks += list(gather_engine_repo_tools_tests(build_dir))
12531273
tasks += list(gather_api_consistency_tests(build_dir))
12541274
tasks += list(gather_path_ops_tests(build_dir))

‎tools/build_bucket_golden_scraper/README.md‎

Lines changed: 86 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,86 @@
1+
# `build_bucket_golden_scraper`
2+
3+
Given logging on Flutter's CI, scrapes the log for golden file changes.
4+
5+
```shell
6+
$ dart bin/main.dart <path to log file, which can be http or a file>
7+
8+
Wrote 3 golden file changes:
9+
testing/resources/performance_overlay_gold_60fps.png
10+
testing/resources/performance_overlay_gold_90fps.png
11+
testing/resources/performance_overlay_gold_120fps.png
12+
```
13+
14+
It can also be run with `--dry-run` to just print what it _would_ do:
15+
16+
```shell
17+
$ dart bin/main.dart --dry-run <path to log file, which can be http or a file>
18+
19+
Found 3 golden file changes:
20+
testing/resources/performance_overlay_gold_60fps.png
21+
testing/resources/performance_overlay_gold_90fps.png
22+
testing/resources/performance_overlay_gold_120fps.png
23+
24+
Run again without --dry-run to apply these changes.
25+
```
26+
27+
You're recommended to still use `git diff` to verify the changes look good.
28+
29+
## Upgrading `git diff`
30+
31+
By default, `git diff` is not very helpful for binary files. You can install
32+
[`imagemagick`](https://imagemagick.org/) and configure your local git client
33+
to make `git diff` show a PNG diff:
34+
35+
```shell
36+
# On MacOS.
37+
$ brew install imagemagick
38+
39+
# Create a comparison script.
40+
$ cat > ~/bin/git-imgdiff <<EOF
41+
#!/bin/sh
42+
echo "Comparing $2 and $5"
43+
44+
# Find a temporary directory to store the diff.
45+
if [ -z "$TMPDIR" ]; then
46+
TMPDIR=/tmp
47+
fi
48+
49+
compare \
50+
"$2" "$5" \
51+
/tmp/git-imgdiff-diff.png
52+
53+
# Display the diff.
54+
open /tmp/git-imgdiff-diff.png
55+
EOF
56+
57+
# Setup git.
58+
git config --global core.attributesfile '~/.gitattributes'
59+
60+
# Add the following to ~/.gitattributes.
61+
cat >> ~/.gitattributes <<EOF
62+
*.png diff=imgdiff
63+
*.jpg diff=imgdiff
64+
*.gif diff=imgdiff
65+
EOF
66+
67+
git config --global diff.imgdiff.command '~/bin/git-imgdiff'
68+
```
69+
70+
## Motivation
71+
72+
Due to <https://github.com/flutter/flutter/issues/53784>, on non-Linux OSes
73+
there is no way to get golden-file changes locally for a variety of engine
74+
tests.
75+
76+
This tool, given log output from a Flutter CI run, will scrape the log for:
77+
78+
```txt
79+
Golden file mismatch. Please check the difference between /b/s/w/ir/cache/builder/src/flutter/testing/resources/performance_overlay_gold_90fps.png and /b/s/w/ir/cache/builder/src/flutter/testing/resources/performance_overlay_gold_90fps_new.png, and replace the former with the latter if the difference looks good.
80+
S
81+
See also the base64 encoded /b/s/w/ir/cache/builder/src/flutter/testing/resources/performance_overlay_gold_90fps_new.png:
82+
iVBORw0KGgoAAAANSUhEUgAAA+gAAAPoCAYAAABNo9TkAAAABHNCSVQICAgIfAhkiAAAIABJREFUeJzs3elzFWeeJ/rnHB3tSEILktgEBrPvYBbbUF4K24X3t (...omitted)
83+
```
84+
85+
And convert the base64 encoded image into a PNG file, and overwrite the old
86+
golden file with the new one.

‎tools/build_bucket_golden_scraper/bin/main.dart‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
// Copyright 2013 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'dart:io' as io;
6+
7+
import 'package:build_bucket_golden_scraper/build_bucket_golden_scraper.dart';
8+
9+
void main(List<String> arguments) async {
10+
final int result;
11+
try {
12+
result = await BuildBucketGoldenScraper.fromCommandLine(arguments).run();
13+
} on FormatException catch (e) {
14+
io.stderr.writeln(e.message);
15+
io.exit(1);
16+
}
17+
if (result != 0) {
18+
io.exit(result);
19+
}
20+
}

‎tools/build_bucket_golden_scraper/lib/build_bucket_golden_scraper.dart‎

Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
// Copyright 2013 The Flutter Authors. All rights reserved.
2+
// Use of this source code is governed by a BSD-style license that can be
3+
// found in the LICENSE file.
4+
5+
import 'dart:convert';
6+
import 'dart:io' as io;
7+
8+
import 'package:args/args.dart';
9+
import 'package:engine_repo_tools/engine_repo_tools.dart';
10+
import 'package:meta/meta.dart';
11+
import 'package:path/path.dart' as p;
12+
13+
/// "Downloads" (i.e. decodes base64 encoded strings) goldens from buildbucket.
14+
///
15+
/// See ../README.md for motivation and usage.
16+
final class BuildBucketGoldenScraper {
17+
/// Creates a scraper with the given configuration.
18+
BuildBucketGoldenScraper({
19+
required this.pathOrUrl,
20+
this.dryRun = false,
21+
String? engineSrcPath,
22+
StringSink? outSink,
23+
}) :
24+
engine = engineSrcPath != null ?
25+
Engine.fromSrcPath(engineSrcPath) :
26+
Engine.findWithin(p.dirname(p.fromUri(io.Platform.script))),
27+
_outSink = outSink ?? io.stdout;
28+
29+
/// Creates a scraper from the command line arguments.
30+
///
31+
/// Throws [FormatException] if the arguments are invalid.
32+
factory BuildBucketGoldenScraper.fromCommandLine(
33+
List<String> args, {
34+
StringSink? outSink,
35+
StringSink? errSink,
36+
}) {
37+
outSink ??= io.stdout;
38+
errSink ??= io.stderr;
39+
40+
final ArgResults argResults = _argParser.parse(args);
41+
if (argResults['help'] as bool) {
42+
_usage(args);
43+
}
44+
final String? pathOrUrl = argResults.rest.isEmpty ? null : argResults.rest.first;
45+
if (pathOrUrl == null) {
46+
_usage(args);
47+
}
48+
return BuildBucketGoldenScraper(
49+
pathOrUrl: pathOrUrl,
50+
dryRun: argResults['dry-run'] as bool,
51+
outSink: outSink,
52+
engineSrcPath: argResults['engine-src-path'] as String?,
53+
);
54+
}
55+
56+
static Never _usage(List<String> args) {
57+
final StringBuffer output = StringBuffer();
58+
output.writeln('Usage: build_bucket_golden_scraper [options] <path or URL>');
59+
output.writeln();
60+
output.writeln(_argParser.usage);
61+
throw FormatException(output.toString(), args.join(' '));
62+
}
63+
64+
static final ArgParser _argParser = ArgParser()
65+
..addFlag(
66+
'help',
67+
abbr: 'h',
68+
help: 'Print this help message.',
69+
negatable: false,
70+
)
71+
..addFlag(
72+
'dry-run',
73+
help: "If true, don't write any files to disk (other than temporary files).",
74+
negatable: false,
75+
)
76+
..addOption(
77+
'engine-src-path',
78+
help: 'The path to the engine source code.',
79+
valueHelp: 'path/that/contains/src (defaults to the directory containing this script)',
80+
);
81+
82+
/// A local path or a URL to a buildbucket log file.
83+
final String pathOrUrl;
84+
85+
/// If true, don't write any files to disk (other than temporary files).
86+
final bool dryRun;
87+
88+
/// The path to the engine source code.
89+
final Engine engine;
90+
91+
/// How to print output, typically [io.stdout].
92+
final StringSink _outSink;
93+
94+
/// Runs the scraper.
95+
Future<int> run() async {
96+
// If the path is a URL, download it and store it in a temporary file.
97+
final Uri? maybeUri = Uri.tryParse(pathOrUrl);
98+
if (maybeUri == null) {
99+
throw FormatException('Invalid path or URL: $pathOrUrl');
100+
}
101+
102+
final String contents;
103+
if (maybeUri.hasScheme) {
104+
contents = await _downloadFile(maybeUri);
105+
} else {
106+
final io.File readFile = io.File(pathOrUrl);
107+
if (!readFile.existsSync()) {
108+
throw FormatException('File does not exist: $pathOrUrl');
109+
}
110+
contents = readFile.readAsStringSync();
111+
}
112+
113+
// Check that it is a buildbucket log file.
114+
if (!contents.contains(_buildBucketMagicString)) {
115+
throw FormatException('Not a buildbucket log file: $pathOrUrl');
116+
}
117+
118+
// Check for occurences of a base64 encoded string.
119+
//
120+
// The format looks something like this:
121+
// [LINE N+0]: See also the base64 encoded /b/s/w/ir/cache/builder/src/flutter/testing/resources/performance_overlay_gold_120fps_new.png:
122+
// [LINE N+1]: {{BASE_64_ENCODED_IMAGE}}
123+
//
124+
// We want to extract the file name (relative to the engine root) and then
125+
// decode the base64 encoded string (and write it to disk if we are not in
126+
// dry-run mode).
127+
final List<_Golden> goldens = <_Golden>[];
128+
final List<String> lines = contents.split('\n');
129+
for (int i = 0; i < lines.length; i++) {
130+
final String line = lines[i];
131+
if (line.startsWith(_base64MagicString)) {
132+
final String relativePath = line.split(_buildBucketMagicString).last.split(':').first;
133+
134+
// Remove the _new suffix from the file name.
135+
final String pathWithouNew = relativePath.replaceAll('_new', '');
136+
137+
final String base64EncodedString = lines[i + 1];
138+
final List<int> bytes = base64Decode(base64EncodedString);
139+
final io.File outFile = io.File(p.join(engine.srcDir.path, pathWithouNew));
140+
goldens.add(_Golden(outFile, bytes));
141+
}
142+
}
143+
144+
if (goldens.isEmpty) {
145+
_outSink.writeln('No goldens found.');
146+
return 0;
147+
}
148+
149+
// Sort and de-duplicate the goldens.
150+
goldens.sort();
151+
final Set<_Golden> uniqueGoldens = goldens.toSet();
152+
153+
// Write the goldens to disk (or pretend to in dry-run mode).
154+
_outSink.writeln('${dryRun ? 'Found' : 'Wrote'} ${uniqueGoldens.length} golden file changes:');
155+
for (final _Golden golden in uniqueGoldens) {
156+
final String truncatedPathAfterFlutterDir = golden.outFile.path.split('flutter${p.separator}').last;
157+
_outSink.writeln(' $truncatedPathAfterFlutterDir');
158+
if (!dryRun) {
159+
await golden.outFile.writeAsBytes(golden.bytes);
160+
}
161+
}
162+
if (dryRun) {
163+
_outSink.writeln('Run again without --dry-run to apply these changes.');
164+
}
165+
166+
return 0;
167+
}
168+
169+
static const String _buildBucketMagicString = '/b/s/w/ir/cache/builder/src/';
170+
static const String _base64MagicString = 'See also the base64 encoded $_buildBucketMagicString';
171+
172+
static Future<String> _downloadFile(Uri uri) async {
173+
final io.HttpClient client = io.HttpClient();
174+
final io.HttpClientRequest request = await client.getUrl(uri);
175+
final io.HttpClientResponse response = await request.close();
176+
final StringBuffer contents = StringBuffer();
177+
await response.transform(utf8.decoder).forEach(contents.write);
178+
client.close();
179+
return contents.toString();
180+
}
181+
}
182+
183+
@immutable
184+
final class _Golden implements Comparable<_Golden> {
185+
const _Golden(this.outFile, this.bytes);
186+
187+
/// Where to write the golden file.
188+
final io.File outFile;
189+
190+
/// The bytes of the golden file to write.
191+
final List<int> bytes;
192+
193+
@override
194+
int get hashCode => outFile.path.hashCode;
195+
196+
@override
197+
bool operator ==(Object other) {
198+
return other is _Golden && other.outFile.path == outFile.path;
199+
}
200+
201+
@override
202+
int compareTo(_Golden other) {
203+
return outFile.path.compareTo(other.outFile.path);
204+
}
205+
}

‎tools/build_bucket_golden_scraper/pubspec.yaml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
# Copyright 2013 The Flutter Authors. All rights reserved.
2+
# Use of this source code is governed by a BSD-style license that can be
3+
# found in the LICENSE file.
4+
5+
name: build_bucket_golden_scraper
6+
publish_to: none
7+
environment:
8+
sdk: ^3.0.0
9+
10+
# Do not add any dependencies that require more than what is provided in
11+
# //third_party/pkg, //third_party/dart/pkg, or
12+
# //third_party/dart/third_party/pkg. In particular, package:test is not usable
13+
# here.
14+
15+
# If you do add packages here, make sure you can run `pub get --offline`, and
16+
# check the .packages and .package_config to make sure all the paths are
17+
# relative to this directory into //third_party/dart
18+
19+
dependencies:
20+
args: any
21+
engine_repo_tools: any
22+
meta: any
23+
path: any
24+
25+
dev_dependencies:
26+
async_helper: any
27+
expect: any
28+
litetest: any
29+
smith: any
30+
31+
dependency_overrides:
32+
async_helper:
33+
path: ../../../third_party/dart/pkg/async_helper
34+
args:
35+
path: ../../../third_party/dart/third_party/pkg/args
36+
engine_repo_tools:
37+
path: ../pkg/engine_repo_tools
38+
expect:
39+
path: ../../../third_party/dart/pkg/expect
40+
litetest:
41+
path: ../../testing/litetest
42+
meta:
43+
path: ../../../third_party/dart/pkg/meta
44+
path:
45+
path: ../../../third_party/dart/third_party/pkg/path
46+
smith:
47+
path: ../../../third_party/dart/pkg/smith

0 commit comments

Comments
 (0)