Lenient prefix / parsing

Author: hamishknight

include/swift/Parse/Parser.h

@@ -1763,10 +1763,7 @@ class Parser {
   /// Try re-lex a '/' operator character as a regex literal. This should be
   /// called when parsing in an expression position to ensure a regex literal is
   /// correctly parsed.
-  ///
-  /// If \p mustBeRegex is set to true, a regex literal will always be lexed if
-  /// enabled. Otherwise, it will not be lexed if it may be ambiguous.
-  void tryLexRegexLiteral(bool mustBeRegex);
+  void tryLexRegexLiteral(bool forUnappliedOperator);
 
   void validateCollectionElement(ParserResult<Expr> element);
 

lib/Parse/ParseExpr.cpp

@@ -513,7 +513,7 @@ ParserResult<Expr> Parser::parseExprUnary(Diag<> Message, bool isExprBasic) {
   UnresolvedDeclRefExpr *Operator;
 
   // First check to see if we have the start of a regex literal `/.../`.
-  tryLexRegexLiteral(/*mustBeRegex*/ true);
+  tryLexRegexLiteral(/*forUnappliedOperator*/ false);
 
   switch (Tok.getKind()) {
   default:
@@ -880,56 +880,70 @@ UnresolvedDeclRefExpr *Parser::parseExprOperator() {
   return new (Context) UnresolvedDeclRefExpr(name, refKind, DeclNameLoc(loc));
 }
 
-void Parser::tryLexRegexLiteral(bool mustBeRegex) {
+void Parser::tryLexRegexLiteral(bool forUnappliedOperator) {
   if (!Context.LangOpts.EnableBareSlashRegexLiterals)
     return;
 
+  // Never a regex literal.
+  if (Tok.isEscapedOperator())
+    return;
+
   // Check to see if we have a regex literal `/.../`, optionally with a prefix
   // operator e.g `!/.../`.
+  bool mustBeRegex = false;
   switch (Tok.getKind()) {
   case tok::oper_prefix:
+    // Prefix operators may contain `/` characters, so this may not be a regex,
+    // and as such need to make sure we have a closing `/`. The first character
+    // heuristics aren't relevant here as a right-bound operator will not have
+    // a space, tab, or `)` character.
+    break;
   case tok::oper_binary_spaced:
-  case tok::oper_binary_unspaced: {
-    // Check to see if we have an operator containing '/'.
-    auto slashIdx = Tok.getText().find("/");
-    if (slashIdx == StringRef::npos)
-      break;
+  case tok::oper_binary_unspaced:
+    // When re-lexing for a 'proper' expression, binary operators are always
+    // invalid, so we can be confident in always lexing a regex literal.
+    mustBeRegex = !forUnappliedOperator;
+    break;
+  default:
+    // We only re-lex regex literals for operator tokens.
+    return;
+  }
 
-    CancellableBacktrackingScope backtrack(*this);
-    {
-      Optional<Lexer::ForwardSlashRegexRAII> regexScope;
-      regexScope.emplace(*L, mustBeRegex);
-
-      // Try re-lex as a `/.../` regex literal, this will split an operator if
-      // necessary.
-      L->restoreState(getParserPosition().LS, /*enableDiagnostics*/ true);
-
-      // If we didn't split a prefix operator, reset the regex lexing scope.
-      // Otherwise, we want to keep it in place for the next token.
-      auto didSplit = L->peekNextToken().getLength() == slashIdx;
-      if (!didSplit)
-        regexScope.reset();
-
-      // Discard the current token, which will be replaced by the re-lexed
-      // token, which will either be a regex literal token, a prefix operator,
-      // or the original unchanged token.
-      discardToken();
-
-      // If we split a prefix operator from the regex literal, and are not sure
-      // whether this should be a regex, backtrack if we didn't end up lexing a
-      // regex literal.
-      if (didSplit && !mustBeRegex &&
-          !L->peekNextToken().is(tok::regex_literal)) {
-        return;
-      }
+  // Check to see if we have an operator containing '/'.
+  auto slashIdx = Tok.getText().find("/");
+  if (slashIdx == StringRef::npos)
+    return;
+
+  CancellableBacktrackingScope backtrack(*this);
+  {
+    Optional<Lexer::ForwardSlashRegexRAII> regexScope;
+    regexScope.emplace(*L, mustBeRegex);
+
+    // Try re-lex as a `/.../` regex literal, this will split an operator if
+    // necessary.
+    L->restoreState(getParserPosition().LS, /*enableDiagnostics*/ true);
+
+    // If we didn't split a prefix operator, reset the regex lexing scope.
+    // Otherwise, we want to keep it in place for the next token.
+    auto didSplit = L->peekNextToken().getLength() == slashIdx;
+    if (!didSplit)
+      regexScope.reset();
+
+    // Discard the current token, which will be replaced by the re-lexed
+    // token, which will either be a regex literal token, a prefix operator,
+    // or the original unchanged token.
+    discardToken();
 
-      // Otherwise, accept the result.
-      backtrack.cancelBacktrack();
+    // If we split a prefix operator from the regex literal, and are not sure
+    // whether this should be a regex, backtrack if we didn't end up lexing a
+    // regex literal.
+    if (didSplit && !mustBeRegex &&
+        !L->peekNextToken().is(tok::regex_literal)) {
+      return;
     }
-    break;
-  }
-  default:
-    break;
+
+    // Otherwise, accept the result.
+    backtrack.cancelBacktrack();
   }
 }
 
@@ -3226,7 +3240,7 @@ ParserStatus Parser::parseExprList(tok leftTok, tok rightTok,
     // First check to see if we have the start of a regex literal `/.../`. We
     // need to do this before handling unapplied operator references, as e.g
     // `(/, /)` might be a regex literal.
-    tryLexRegexLiteral(/*mustBeRegex*/ false);
+    tryLexRegexLiteral(/*forUnappliedOperator*/ true);
 
     // See if we have an operator decl ref '(<op>)'. The operator token in
     // this case lexes as a binary operator because it neither leads nor

test/StringProcessing/Frontend/enable-flag.swift

@@ -4,7 +4,7 @@
 
 // REQUIRES: swift_in_compiler
 
-prefix operator / // expected-error {{prefix operator may not contain '/'}}
+prefix operator /
 
 _ = /x/
 _ = #/x/#

test/StringProcessing/Parse/forward-slash-regex.swift

@@ -2,9 +2,11 @@
 // REQUIRES: swift_in_compiler
 // REQUIRES: concurrency
 
-prefix operator /  // expected-error {{prefix operator may not contain '/'}}
-prefix operator ^/ // expected-error {{prefix operator may not contain '/'}}
-prefix operator /^/ // expected-error {{prefix operator may not contain '/'}}
+prefix operator /
+prefix operator ^/
+prefix operator /^/
+
+prefix func ^/ <T> (_ x: T) -> T { x } // expected-note {{'^/' declared here}}
 
 prefix operator !!
 prefix func !! <T>(_ x: T) -> T { x }
@@ -53,8 +55,9 @@ do {
   // expected-error@-3 {{'/' is not a postfix unary operator}}
 }
 
+// No closing '/' so a prefix operator.
 _ = /x
-// expected-error@-1 {{unterminated regex literal}}
+// expected-error@-1 {{'/' is not a prefix unary operator}}
 
 _ = !/x/
 // expected-error@-1 {{cannot convert value of type 'Regex<Substring>' to expected argument type 'Bool'}}
@@ -250,13 +253,15 @@ _ = await /x / // expected-warning {{no 'async' operations occur within 'await'
 // written a comment and is still in the middle of writing the characters before
 // it.
 _ = /x// comment
-// expected-error@-1 {{unterminated regex literal}}
+// expected-error@-1 {{'/' is not a prefix unary operator}}
 
 _ = /x // comment
-// expected-error@-1 {{unterminated regex literal}}
+// expected-error@-1 {{'/' is not a prefix unary operator}}
 
 _ = /x/*comment*/
-// expected-error@-1 {{unterminated regex literal}}
+// expected-error@-1 {{'/' is not a prefix unary operator}}
+
+// MARK: Unapplied operators
 
 // These become regex literals, unless surrounded in parens.
 func baz(_ x: (Int, Int) -> Int, _ y: (Int, Int) -> Int) {} // expected-note 4{{'baz' declared here}}
@@ -320,6 +325,26 @@ let arr: [Double] = [2, 3, 4]
 _ = arr.reduce(1, /) / 3
 _ = arr.reduce(1, /) + arr.reduce(1, /)
 
+// MARK: Backticks behavior
+
+// This is a prefix operator, even if there is a closing '/'.
+_ = `/`x
+// expected-error@-1 {{'/' is not a prefix unary operator}}
+
+_ = `/`x/
+// expected-error@-1 {{'/' is not a prefix unary operator}}
+// expected-error@-2 {{'/' is not a postfix unary operator}}
+
+_ = ^/x/
+// expected-error@-1 {{'^' is not a prefix unary operator}}
+
+_ = `^/`x/
+// expected-error@-1 {{'/' is not a postfix unary operator}}
+
+_ = `!!`/x/
+
+// MARK: Starting characters
+
 // Fine.
 _ = /./
 

test/StringProcessing/Parse/regex_parse_error.swift

@@ -30,17 +30,15 @@ _ = #/\(?'abc/#
 do {
   _ = /\
   /
-  // expected-error@-2:7 {{unterminated regex literal}}
-  // expected-error@-3:9 {{expected escape sequence}}
-} // expected-error@:1 {{expected expression after operator}}
+  // expected-error@-1:3 {{expected expression path in Swift key path}}
+}
 
 do {
   _ = #/\
   /#
   // expected-error@-2:7 {{unterminated regex literal}}
   // expected-error@-3:10 {{expected escape sequence}}
-  // expected-error@-3:3 {{unterminated regex literal}}
-  // expected-warning@-4:3 {{regular expression literal is unused}}
+  // expected-error@-3:4 {{expected expression}}
 }
 
 func foo<T>(_ x: T, _ y: T) {}