From 1bfeec29be20ee70fad65b4fb1f82597e0618ac6 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Thu, 16 Aug 2018 17:23:47 +0300 Subject: [PATCH 1/4] Added the rest of the Locales that have different behavior for uppercasing/lowercasing scenarios to the skip list --- .../scalar/string/StringFunctionProcessorTests.java | 11 +++++++++-- .../xpack/qa/sql/jdbc/SqlSpecTestCase.java | 9 +++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java index a4d9d4cb57ab6..5c30a08838e6f 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java @@ -75,7 +75,7 @@ public void testLCase() { stringCharInputValidation(proc); } - public void testLCaseWithTRLocale() { + public void testLCaseWithAZTRLocale() { Locale.setDefault(Locale.forLanguageTag("tr")); StringProcessor proc = new StringProcessor(StringOperation.LCASE); @@ -86,6 +86,10 @@ public void testLCaseWithTRLocale() { // unicode 0049 = I (regular capital letter i) // in Turkish locale this would be lowercased to a "i" without dot (unicode 0131) assertEquals("\u0069", proc.process("\u0049")); + + Locale.setDefault(Locale.forLanguageTag("az")); + assertEquals("\u0069\u0307", proc.process("\u0130")); + assertEquals("\u0069", proc.process("\u0049")); } public void testUCase() { @@ -102,13 +106,16 @@ public void testUCase() { stringCharInputValidation(proc); } - public void testUCaseWithTRLocale() { + public void testUCaseWithAZTRLocale() { Locale.setDefault(Locale.forLanguageTag("tr")); StringProcessor proc = new StringProcessor(StringOperation.UCASE); // ES-SQL is not Locale sensitive (so far). // in Turkish locale, small letter "i" is uppercased to "I" with a dot above (unicode 130), otherwise in "i" (unicode 49) assertEquals("\u0049", proc.process("\u0069")); + + Locale.setDefault(Locale.forLanguageTag("az")); + assertEquals("\u0049", proc.process("\u0069")); } public void testLength() { diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java index d61b4b9a946bd..453bebac5a755 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java @@ -13,6 +13,7 @@ import java.sql.Connection; import java.sql.ResultSet; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Locale; @@ -37,8 +38,7 @@ public static List readScriptSpec() throws Exception { tests.addAll(readScriptSpec("/agg.sql-spec", parser)); tests.addAll(readScriptSpec("/arithmetic.sql-spec", parser)); tests.addAll(readScriptSpec("/string-functions.sql-spec", parser)); - // AwaitsFix: https://github.com/elastic/elasticsearch/issues/32589 - // tests.addAll(readScriptSpec("/case-functions.sql-spec", parser)); + tests.addAll(readScriptSpec("/case-functions.sql-spec", parser)); return tests; } @@ -60,8 +60,9 @@ public SqlSpecTestCase(String fileName, String groupName, String testName, Integ @Override protected final void doTest() throws Throwable { - boolean goodLocale = !(Locale.getDefault().equals(new Locale.Builder().setLanguageTag("tr").build()) - || Locale.getDefault().equals(new Locale.Builder().setLanguageTag("tr-TR").build())); + String[] badLocales = new String[] {"tr", "az", "tr-TR", "tr-CY", "az-Latn", "az-Cyrl", "az-Latn-AZ", "az-Cyrl-AZ"}; + boolean goodLocale = !Arrays.stream(badLocales) + .anyMatch((l) -> Locale.getDefault().equals(new Locale.Builder().setLanguageTag(l).build())); if (fileName.startsWith("case-functions")) { Assume.assumeTrue(goodLocale); } From e0a8e064035bbeccf3e9854c2711478fc9139f9b Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Fri, 17 Aug 2018 09:37:29 +0300 Subject: [PATCH 2/4] Restored the initial Locale in tests --- .../scalar/string/StringFunctionProcessorTests.java | 10 +++++++++- .../xpack/qa/sql/jdbc/SqlSpecTestCase.java | 6 ++++-- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java index 5c30a08838e6f..9cddd23fbe739 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java @@ -76,6 +76,7 @@ public void testLCase() { } public void testLCaseWithAZTRLocale() { + Locale initialLocale = Locale.getDefault(); Locale.setDefault(Locale.forLanguageTag("tr")); StringProcessor proc = new StringProcessor(StringOperation.LCASE); @@ -90,6 +91,9 @@ public void testLCaseWithAZTRLocale() { Locale.setDefault(Locale.forLanguageTag("az")); assertEquals("\u0069\u0307", proc.process("\u0130")); assertEquals("\u0069", proc.process("\u0049")); + + // restore the original Locale + Locale.setDefault(initialLocale); } public void testUCase() { @@ -107,6 +111,7 @@ public void testUCase() { } public void testUCaseWithAZTRLocale() { + Locale initialLocale = Locale.getDefault(); Locale.setDefault(Locale.forLanguageTag("tr")); StringProcessor proc = new StringProcessor(StringOperation.UCASE); @@ -116,6 +121,9 @@ public void testUCaseWithAZTRLocale() { Locale.setDefault(Locale.forLanguageTag("az")); assertEquals("\u0049", proc.process("\u0069")); + + // restore the original Locale + Locale.setDefault(initialLocale); } public void testLength() { @@ -186,7 +194,7 @@ public void testCharLength() { assertEquals(7, proc.process("foo bar")); assertEquals(0, proc.process("")); assertEquals(1, proc.process('f')); - assertEquals(1, proc.process('€')); + assertEquals(1, proc.process('€')); stringCharInputValidation(proc); } diff --git a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java index 453bebac5a755..4d90c9cce502b 100644 --- a/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java +++ b/x-pack/qa/sql/src/main/java/org/elasticsearch/xpack/qa/sql/jdbc/SqlSpecTestCase.java @@ -60,8 +60,10 @@ public SqlSpecTestCase(String fileName, String groupName, String testName, Integ @Override protected final void doTest() throws Throwable { - String[] badLocales = new String[] {"tr", "az", "tr-TR", "tr-CY", "az-Latn", "az-Cyrl", "az-Latn-AZ", "az-Cyrl-AZ"}; - boolean goodLocale = !Arrays.stream(badLocales) + // we skip the tests in case of these locales because ES-SQL is Locale-insensitive for now + // while H2 does take the Locale into consideration + String[] h2IncompatibleLocales = new String[] {"tr", "az", "tr-TR", "tr-CY", "az-Latn", "az-Cyrl", "az-Latn-AZ", "az-Cyrl-AZ"}; + boolean goodLocale = !Arrays.stream(h2IncompatibleLocales) .anyMatch((l) -> Locale.getDefault().equals(new Locale.Builder().setLanguageTag(l).build())); if (fileName.startsWith("case-functions")) { Assume.assumeTrue(goodLocale); From d1f886d48784b7e5017dd0127c92b5545c233dcb Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Fri, 17 Aug 2018 15:03:01 +0300 Subject: [PATCH 3/4] Correct character encoding for utf8 --- .../function/scalar/string/StringFunctionProcessorTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java index 9cddd23fbe739..e8406118a09c4 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java @@ -194,7 +194,7 @@ public void testCharLength() { assertEquals(7, proc.process("foo bar")); assertEquals(0, proc.process("")); assertEquals(1, proc.process('f')); - assertEquals(1, proc.process('€')); + assertEquals(1, proc.process('\u20ac')); // euro symbol stringCharInputValidation(proc); } From 03c434331a552065429639141bd699567bcf56c9 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Tue, 21 Aug 2018 12:02:31 +0300 Subject: [PATCH 4/4] Wrapped the Locale changing tests in try {} finally {} block --- .../string/StringFunctionProcessorTests.java | 54 ++++++++++--------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java index e8406118a09c4..d3336ec89a840 100644 --- a/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java +++ b/x-pack/plugin/sql/src/test/java/org/elasticsearch/xpack/sql/expression/function/scalar/string/StringFunctionProcessorTests.java @@ -78,22 +78,24 @@ public void testLCase() { public void testLCaseWithAZTRLocale() { Locale initialLocale = Locale.getDefault(); Locale.setDefault(Locale.forLanguageTag("tr")); - StringProcessor proc = new StringProcessor(StringOperation.LCASE); - // ES-SQL is not locale sensitive (so far). The obvious test for this is the Turkish language, uppercase letter I conversion - // in non-Turkish locale the lowercasing would create i and an additional dot, while in Turkish Locale it would only create "i" - // unicode 0069 = i - assertEquals("\u0069\u0307", proc.process("\u0130")); - // unicode 0049 = I (regular capital letter i) - // in Turkish locale this would be lowercased to a "i" without dot (unicode 0131) - assertEquals("\u0069", proc.process("\u0049")); - - Locale.setDefault(Locale.forLanguageTag("az")); - assertEquals("\u0069\u0307", proc.process("\u0130")); - assertEquals("\u0069", proc.process("\u0049")); - - // restore the original Locale - Locale.setDefault(initialLocale); + try { + StringProcessor proc = new StringProcessor(StringOperation.LCASE); + // ES-SQL is not locale sensitive (so far). The obvious test for this is the Turkish language, uppercase letter I conversion + // in non-Turkish locale the lowercasing would create i and an additional dot, while in Turkish Locale it would only create "i" + // unicode 0069 = i + assertEquals("\u0069\u0307", proc.process("\u0130")); + // unicode 0049 = I (regular capital letter i) + // in Turkish locale this would be lowercased to a "i" without dot (unicode 0131) + assertEquals("\u0069", proc.process("\u0049")); + + Locale.setDefault(Locale.forLanguageTag("az")); + assertEquals("\u0069\u0307", proc.process("\u0130")); + assertEquals("\u0069", proc.process("\u0049")); + } finally { + // restore the original Locale + Locale.setDefault(initialLocale); + } } public void testUCase() { @@ -113,17 +115,19 @@ public void testUCase() { public void testUCaseWithAZTRLocale() { Locale initialLocale = Locale.getDefault(); Locale.setDefault(Locale.forLanguageTag("tr")); - StringProcessor proc = new StringProcessor(StringOperation.UCASE); - - // ES-SQL is not Locale sensitive (so far). - // in Turkish locale, small letter "i" is uppercased to "I" with a dot above (unicode 130), otherwise in "i" (unicode 49) - assertEquals("\u0049", proc.process("\u0069")); - - Locale.setDefault(Locale.forLanguageTag("az")); - assertEquals("\u0049", proc.process("\u0069")); - // restore the original Locale - Locale.setDefault(initialLocale); + try { + StringProcessor proc = new StringProcessor(StringOperation.UCASE); + // ES-SQL is not Locale sensitive (so far). + // in Turkish locale, small letter "i" is uppercased to "I" with a dot above (unicode 130), otherwise in "i" (unicode 49) + assertEquals("\u0049", proc.process("\u0069")); + + Locale.setDefault(Locale.forLanguageTag("az")); + assertEquals("\u0049", proc.process("\u0069")); + } finally { + // restore the original Locale + Locale.setDefault(initialLocale); + } } public void testLength() {