Auto merge of #22011 - fhahn:separate-parse-fail-tests, r=nikomatsakis

This PR moves all `compile-fail` tests that fail at the parsing stage to a `parse-fail` directory, in order to use the tests in the `parse-fail` directory to test if the new LALR parser rejects the same files as the Rust parser. I also adjusted the `testparser.py` script to handle the tests in `parse-fail` differently.

However during working on this, I discovered, that Rust's parser sometimes fails during parsing, but does not return a nonzero return code, e.g. compiling `/test/compile-fail/doc-before-semi.rs` with `-Z parse-only` prints an error message, but returns status code 0. Compiling the same file without `-Z parse-only`, the same error message is displayed, but error code 101 returned. I'll look into that over the next week.

Author: bors

mk/tests.mk

@@ -452,6 +452,7 @@ RPASS_FULL_RS := $(wildcard $(S)src/test/run-pass-fulldeps/*.rs)
 CFAIL_FULL_RS := $(wildcard $(S)src/test/compile-fail-fulldeps/*.rs)
 RFAIL_RS := $(wildcard $(S)src/test/run-fail/*.rs)
 CFAIL_RS := $(wildcard $(S)src/test/compile-fail/*.rs)
+PFAIL_RS := $(wildcard $(S)src/test/parse-fail/*.rs)
 BENCH_RS := $(wildcard $(S)src/test/bench/*.rs)
 PRETTY_RS := $(wildcard $(S)src/test/pretty/*.rs)
 DEBUGINFO_GDB_RS := $(wildcard $(S)src/test/debuginfo/*.rs)
@@ -468,7 +469,7 @@ RPASS_VALGRIND_TESTS := $(RPASS_VALGRIND_RS)
 RPASS_FULL_TESTS := $(RPASS_FULL_RS)
 CFAIL_FULL_TESTS := $(CFAIL_FULL_RS)
 RFAIL_TESTS := $(RFAIL_RS)
-CFAIL_TESTS := $(CFAIL_RS)
+CFAIL_TESTS := $(CFAIL_RS) $(PFAIL_RS)
 BENCH_TESTS := $(BENCH_RS)
 PERF_TESTS := $(PERF_RS)
 PRETTY_TESTS := $(PRETTY_RS)

src/grammar/testparser.py

@@ -35,34 +35,42 @@
     ok[parser] = 0
     bad[parser] = []
 devnull = open(os.devnull, 'w')
-print "\n"
+print("\n")
 
 for base, dirs, files in os.walk(args.source_dir[0]):
     for f in filter(lambda p: p.endswith('.rs'), files):
         p = os.path.join(base, f)
-        compile_fail = 'compile-fail' in p
-        ignore = any('ignore-test' in line or 'ignore-lexer-test' in line
-                     for line in open(p).readlines())
-        if compile_fail or ignore:
+        parse_fail = 'parse-fail' in p
+        if sys.version_info.major == 3:
+            lines = open(p, encoding='utf-8').readlines()
+        else:
+            lines = open(p).readlines()
+        if any('ignore-test' in line or 'ignore-lexer-test' in line for line in lines):
             continue
         total += 1
         for parser in args.parser:
             if subprocess.call(parser, stdin=open(p), stderr=subprocess.STDOUT, stdout=devnull) == 0:
-                ok[parser] += 1
+                if parse_fail:
+                    bad[parser].append(p)
+                else:
+                    ok[parser] += 1
             else:
-                bad[parser].append(p)
+                if parse_fail:
+                    ok[parser] += 1
+                else:
+                    bad[parser].append(p)
         parser_stats = ', '.join(['{}: {}'.format(parser, ok[parser]) for parser in args.parser])
         sys.stdout.write("\033[K\r total: {}, {}, scanned {}"
                          .format(total, os.path.relpath(parser_stats), os.path.relpath(p)))
 
 devnull.close()
 
-print "\n"
+print("\n")
 
 for parser in args.parser:
     filename = os.path.basename(parser) + '.bad'
-    print("writing {} files that failed to parse with {} to {}".format(len(bad[parser]), parser, filename))
+    print("writing {} files that did not yield the correct result with {} to {}".format(len(bad[parser]), parser, filename))
     with open(filename, "w") as f:
-          for p in bad[parser]:
-              f.write(p)
-              f.write("\n")
+        for p in bad[parser]:
+            f.write(p)
+            f.write("\n")