[GR-35151] Fix RegExp case-folding for non-Unicode regexps.

PullRequest: graal/10339

Author: jirkamarsik

regex/src/com.oracle.truffle.regex/src/com/oracle/truffle/regex/tregex/parser/CaseFoldTable.java

@@ -545,21 +545,11 @@ public void appendRangesTo(RangesBuffer buffer, int startIndex, int endIndex) {
                     0x001f78, 0x001f79, INTEGER_OFFSET, 128,
                     0x001f7a, 0x001f7b, INTEGER_OFFSET, 112,
                     0x001f7c, 0x001f7d, INTEGER_OFFSET, 126,
-                    0x001f80, 0x001f87, INTEGER_OFFSET, 8,
-                    0x001f88, 0x001f8f, INTEGER_OFFSET, -8,
-                    0x001f90, 0x001f97, INTEGER_OFFSET, 8,
-                    0x001f98, 0x001f9f, INTEGER_OFFSET, -8,
-                    0x001fa0, 0x001fa7, INTEGER_OFFSET, 8,
-                    0x001fa8, 0x001faf, INTEGER_OFFSET, -8,
                     0x001fb0, 0x001fb1, INTEGER_OFFSET, 8,
-                    0x001fb3, 0x001fb3, INTEGER_OFFSET, 9,
                     0x001fb8, 0x001fb9, INTEGER_OFFSET, -8,
                     0x001fba, 0x001fbb, INTEGER_OFFSET, -74,
-                    0x001fbc, 0x001fbc, INTEGER_OFFSET, -9,
                     0x001fbe, 0x001fbe, DIRECT_MAPPING, 5,
-                    0x001fc3, 0x001fc3, INTEGER_OFFSET, 9,
                     0x001fc8, 0x001fcb, INTEGER_OFFSET, -86,
-                    0x001fcc, 0x001fcc, INTEGER_OFFSET, -9,
                     0x001fd0, 0x001fd1, INTEGER_OFFSET, 8,
                     0x001fd8, 0x001fd9, INTEGER_OFFSET, -8,
                     0x001fda, 0x001fdb, INTEGER_OFFSET, -100,
@@ -568,10 +558,8 @@ public void appendRangesTo(RangesBuffer buffer, int startIndex, int endIndex) {
                     0x001fe8, 0x001fe9, INTEGER_OFFSET, -8,
                     0x001fea, 0x001feb, INTEGER_OFFSET, -112,
                     0x001fec, 0x001fec, INTEGER_OFFSET, -7,
-                    0x001ff3, 0x001ff3, INTEGER_OFFSET, 9,
                     0x001ff8, 0x001ff9, INTEGER_OFFSET, -128,
                     0x001ffa, 0x001ffb, INTEGER_OFFSET, -126,
-                    0x001ffc, 0x001ffc, INTEGER_OFFSET, -9,
                     0x002132, 0x002132, INTEGER_OFFSET, 28,
                     0x00214e, 0x00214e, INTEGER_OFFSET, -28,
                     0x002160, 0x00216f, INTEGER_OFFSET, 16,
@@ -635,27 +623,7 @@ public void appendRangesTo(RangesBuffer buffer, int startIndex, int endIndex) {
                     0x00ab53, 0x00ab53, INTEGER_OFFSET, -928,
                     0x00ab70, 0x00abbf, INTEGER_OFFSET, -38864,
                     0x00ff21, 0x00ff3a, INTEGER_OFFSET, 32,
-                    0x00ff41, 0x00ff5a, INTEGER_OFFSET, -32,
-                    0x010400, 0x010427, INTEGER_OFFSET, 40,
-                    0x010428, 0x01044f, INTEGER_OFFSET, -40,
-                    0x0104b0, 0x0104d3, INTEGER_OFFSET, 40,
-                    0x0104d8, 0x0104fb, INTEGER_OFFSET, -40,
-                    0x010570, 0x01057a, INTEGER_OFFSET, 39,
-                    0x01057c, 0x01058a, INTEGER_OFFSET, 39,
-                    0x01058c, 0x010592, INTEGER_OFFSET, 39,
-                    0x010594, 0x010595, INTEGER_OFFSET, 39,
-                    0x010597, 0x0105a1, INTEGER_OFFSET, -39,
-                    0x0105a3, 0x0105b1, INTEGER_OFFSET, -39,
-                    0x0105b3, 0x0105b9, INTEGER_OFFSET, -39,
-                    0x0105bb, 0x0105bc, INTEGER_OFFSET, -39,
-                    0x010c80, 0x010cb2, INTEGER_OFFSET, 64,
-                    0x010cc0, 0x010cf2, INTEGER_OFFSET, -64,
-                    0x0118a0, 0x0118bf, INTEGER_OFFSET, 32,
-                    0x0118c0, 0x0118df, INTEGER_OFFSET, -32,
-                    0x016e40, 0x016e5f, INTEGER_OFFSET, 32,
-                    0x016e60, 0x016e7f, INTEGER_OFFSET, -32,
-                    0x01e900, 0x01e921, INTEGER_OFFSET, 34,
-                    0x01e922, 0x01e943, INTEGER_OFFSET, -34
+                    0x00ff41, 0x00ff5a, INTEGER_OFFSET, -32
     });
 
     private static final CaseFoldTableImpl UNICODE_TABLE_ENTRIES = new CaseFoldTableImpl(new int[]{

regex/src/com.oracle.truffle.regex/tools/generate_case_fold_table.clj

@@ -1,7 +1,5 @@
-#!/usr/bin/env boot
-
 ; ------------------------------------------------------------------------------
-; Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+; Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
 ; DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 ;
 ; The Universal Permissive License (UPL), Version 1.0
@@ -48,8 +46,9 @@
 ;; This script assumes that the current working directory contains a folder "dat"
 ;; with the files NonUnicodeFoldTable.txt and UnicodeFoldTable.txt.
 
-(require '[clojure.set :as set]
-         '[clojure.string :as str])
+(ns generate-case-fold-table
+  (:require [clojure.set :as set]
+            [clojure.string :as str]))
 
 (defn pairwise
   "Given a sequence `x_1`, `x_2`, `x_3`..., returns the sequence of pairs `[x_1 x_2]`, `[x_2 x_3]`..."
@@ -324,17 +323,17 @@
         footer               "\n    });\n"
         method-name-and-args (fn [entry]
                                (case (:kind entry)
-                                 :delta       {:lo          (:lo entry) 
-                                               :hi          (:hi entry) 
+                                 :delta       {:lo          (:lo entry)
+                                               :hi          (:hi entry)
                                                :method-name "INTEGER_OFFSET"
                                                :arg         (:delta entry)}
-                                 :alternating {:lo          (:lo entry) 
+                                 :alternating {:lo          (:lo entry)
                                                :hi          (:hi entry)
                                                :method-name (if (:aligned entry)
                                                               "ALTERNATING_AL"
                                                               "ALTERNATING_UL")
                                                :arg         0 }
-                                 :class       {:lo          (:lo entry) 
+                                 :class       {:lo          (:lo entry)
                                                :hi          (:hi entry)
                                                :method-name "DIRECT_MAPPING"
                                                :arg         (:class-id entry)}))

regex/src/com.oracle.truffle.regex/tools/generate_nonunicode_fold_table.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+#
+# Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
+# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+#
+# The Universal Permissive License (UPL), Version 1.0
+#
+# Subject to the condition set forth below, permission is hereby granted to any
+# person obtaining a copy of this software, associated documentation and/or
+# data (collectively the "Software"), free of charge and under any and all
+# copyright rights in the Software, and any and all patent rights owned or
+# freely licensable by each licensor hereunder covering either (i) the
+# unmodified Software as contributed to or provided by such licensor, or (ii)
+# the Larger Works (as defined below), to deal in both
+#
+# (a) the Software, and
+#
+# (b) any piece of software and/or hardware listed in the lrgrwrks.txt file if
+# one is included with the Software each a "Larger Work" to which the Software
+# is contributed by such licensors),
+#
+# without restriction, including without limitation the rights to copy, create
+# derivative works of, display, perform, and distribute the Software and make,
+# use, sell, offer for sale, import, export, have made, and have sold the
+# Software and the Larger Work(s), and to sublicense the foregoing rights on
+# either these or other terms.
+#
+# This license is subject to the following condition:
+#
+# The above copyright notice and either this complete permission notice or at a
+# minimum a reference to the UPL must be included in all copies or substantial
+# portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+# This reads dat/UnicodeData.txt and dat/SpecialCasing.txt and produces a file
+# that gives all the non-trivial pairs of inputs-outputs of the ECMAScript
+# Canonicalize when Unicode is false and IgnoreCase is true.
+
+upper_map = {}
+for line in open("dat/UnicodeData.txt"):
+    tokens = line.split(";")
+    # Drop entries without toUppercase mapping
+    if tokens[12].strip() == "":
+        continue
+    char = int(tokens[0].strip(), 16)
+    upper = int(tokens[12].strip(), 16)
+    upper_map[char] = [upper]
+
+for line in open("dat/SpecialCasing.txt"):
+    # Drop comments and empty lines
+    if line.startswith("#") or line.strip() == "":
+        continue
+    tokens = line.split(";")
+    # Drop entries with conditions
+    if len(tokens) > 5:
+        continue
+    char = int(tokens[0].strip(), 16)
+    upper = [int(c, 16) for c in tokens[3].split()]
+    upper_map[char] = upper
+
+for (char, upper) in upper_map.items():
+    # Only follow rules which give map to a single code unit
+    if len(upper) > 1 or upper[0] >= 0x10000:
+        continue
+    # Do not allow non-ASCII characters to cross into ASCII.
+    if char >= 128 and upper[0] < 128:
+        continue
+    # Drop trivial mappings
+    if (char == upper[0]):
+        continue
+    print("%X;%X" % (char, upper[0]))

regex/src/com.oracle.truffle.regex/tools/run_scripts.sh

@@ -40,6 +40,7 @@
 # SOFTWARE.
 #
 
+set -e
 
 if [[ $(pwd) != *graal/regex/src/com.oracle.truffle.regex/tools ]]
 then
@@ -65,7 +66,7 @@ unzip -d dat dat/ucd.nounihan.flat.zip
 
 ./unicode-script.sh
 
-clojure --init generate_case_fold_table.clj --eval '(-main)' > dat/case-fold-table.txt
+clojure -Sdeps '{:paths ["."]}' -M --main generate-case-fold-table > dat/case-fold-table.txt
 
 ./update_case_fold_table.py
 

regex/src/com.oracle.truffle.regex/tools/unicode-script.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 #
-# Copyright (c) 2019, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2021, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # The Universal Permissive License (UPL), Version 1.0
@@ -40,6 +40,8 @@
 # SOFTWARE.
 #
 
+set -e
+
 
 # This script takes the CaseFolding.txt and UnicodeData.txt files of the Unicode
 # character database and extracts from them the files UnicodeFoldTable.txt and
@@ -66,15 +68,12 @@ cat dat/CaseFolding.txt \
     > dat/UnicodeFoldTable.txt
 
 # We produce the table for the Canonicalize abstract function when the Unicode
-# flag is not present. We use the UnicodeData.txt file and extract the
-# Uppercase_Character field. We remove entries which do not have an
-# Uppercase_Character mapping and entries which map from non-ASCII
-# code points (>= 128) to ASCII code points (< 128), as per the ECMAScript spec.
-cat dat/UnicodeData.txt \
-    | cut -d\; -f1,13 \
-    | sed -e '/;$/d' \
-          -e '/^\(00[8-F][0-9A-F]\|0[^0][0-9A-F]\+\|[^0][0-9A-F]\+\);00[0-7][0-9A-F]$/d' \
-    > dat/NonUnicodeFoldTable.txt
+# flag is not present. We extract the Unicode Case Conversion table from the
+# UnicodeData.txt and SpecialCasing.txt files. We remove entries which map from
+# non-ASCII code points (>= 128) to ASCII code points (< 128), as per the
+# ECMAScript spec. We also drop the special entries which produce strings of more
+# than one UTF-16 code unit.
+./generate_nonunicode_fold_table.py > dat/NonUnicodeFoldTable.txt
 
 # In Python's case insensitive regular expressions, characters are considered
 # equivalent if they have the same Lowercase mapping. However, in some cases