Use SQLite native representations in JSON (#31490)

Closes #30727

Author: roji

src/EFCore.Sqlite.Core/Properties/SqliteStrings.Designer.cs

@@ -216,9 +216,6 @@
   <data name="OrderByNotSupported" xml:space="preserve">
     <value>SQLite does not support expressions of type '{type}' in ORDER BY clauses. Convert the values to a supported type, or use LINQ to Objects to order the results on the client side.</value>
   </data>
-  <data name="QueryingJsonCollectionOfGivenTypeNotSupported" xml:space="preserve">
-    <value>Querying JSON collections with element provider type '{type}' isn't supported because of SQLite limitations.</value>
-  </data>
   <data name="SequencesNotSupported" xml:space="preserve">
     <value>SQLite does not support sequences. See https://go.microsoft.com/fwlink/?LinkId=723262 for more information and examples.</value>
   </data>

src/EFCore.Sqlite.Core/Properties/SqliteStrings.resx

@@ -671,54 +671,8 @@ private static SqlExpression ApplyJsonSqlConversion(
         bool isNullable)
         => typeMapping switch
         {
-            // The "default" JSON representation of a GUID is a lower-case string, but we do upper-case GUIDs in our non-JSON
-            // implementation.
-            SqliteGuidTypeMapping
-                => sqlExpressionFactory.Function("upper", new[] { expression }, isNullable, new[] { true }, typeof(Guid), typeMapping),
-
-            // The "standard" JSON timestamp representation is ISO8601, with a T between date and time; but SQLite's representation has
-            // no T. The following performs a reliable conversions on the string values coming out of json_each.
-            // Unfortunately, the SQLite datetime() function doesn't present fractional seconds, so we generate the following lovely thing:
-            // rtrim(rtrim(strftime('%Y-%m-%d %H:%M:%f', $value), '0'), '.')
-            SqliteDateTimeTypeMapping
-                => sqlExpressionFactory.Function(
-                    "rtrim",
-                    new SqlExpression[]
-                    {
-                        sqlExpressionFactory.Function(
-                            "rtrim",
-                            new SqlExpression[]
-                            {
-                                sqlExpressionFactory.Function(
-                                    "strftime",
-                                    new[]
-                                    {
-                                        sqlExpressionFactory.Constant("%Y-%m-%d %H:%M:%f"),
-                                        expression
-                                    },
-                                    isNullable, new[] { true }, typeof(DateTime), typeMapping),
-                                sqlExpressionFactory.Constant("0")
-                            },
-                            isNullable, new[] { true }, typeof(DateTime), typeMapping),
-                        sqlExpressionFactory.Constant(".")
-                    },
-                    isNullable, new[] { true }, typeof(DateTime), typeMapping),
-
-            // The JSON representation for decimal is e.g. 1 (JSON int), whereas our literal representation is "1.0" (string).
-            // We can cast the 1 to TEXT, but we'd still get "1" not "1.0".
-            SqliteDecimalTypeMapping
-                => throw new InvalidOperationException(SqliteStrings.QueryingJsonCollectionOfGivenTypeNotSupported("decimal")),
-
-            // The JSON representation for new[] { 1, 2 } is AQI= (base64), and SQLite has no built-in base64 conversion function.
             ByteArrayTypeMapping
-                => throw new InvalidOperationException(SqliteStrings.QueryingJsonCollectionOfGivenTypeNotSupported("byte[]")),
-
-            // The JSON representation for DateTimeOffset is ISO8601 (2023-01-01T12:30:00+02:00), but our SQL literal representation
-            // is 2023-01-01 12:30:00+02:00 (no T).
-            // Note that datetime('2023-01-01T12:30:00+02:00') yields '2023-01-01 10:30:00', converting to UTC (removing the timezone), so
-            // we can't use that.
-            SqliteDateTimeOffsetTypeMapping
-                => throw new InvalidOperationException(SqliteStrings.QueryingJsonCollectionOfGivenTypeNotSupported("DateTimeOffset")),
+                => sqlExpressionFactory.Function("unhex", new[] { expression }, isNullable, new[] { true }, typeof(byte[]), typeMapping),
 
             _ => expression
         };

src/EFCore.Sqlite.Core/Query/Internal/SqliteQueryableMethodTranslatingExpressionVisitor.cs

@@ -0,0 +1,53 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Globalization;
+using System.Text;
+using System.Text.Encodings.Web;
+using System.Text.Json;
+using Microsoft.EntityFrameworkCore.Storage.Json;
+
+namespace Microsoft.EntityFrameworkCore.Sqlite.Storage.Internal.Json;
+
+/// <summary>
+///     The Sqlite-specific JsonValueReaderWrite for byte[]. Generates the SQLite representation (e.g. X'0102') rather than base64, in order
+///     to match our SQLite non-JSON representation.
+/// </summary>
+/// <remarks>
+///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+///     any release. You should only use it directly in your code with extreme caution and knowing that
+///     doing so can result in application failures when updating to a new Entity Framework Core release.
+/// </remarks>
+public sealed class SqliteJsonByteArrayReaderWriter : JsonValueReaderWriter<byte[]>
+{
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public static SqliteJsonByteArrayReaderWriter Instance { get; } = new();
+
+    private SqliteJsonByteArrayReaderWriter()
+    {
+    }
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public override byte[] FromJsonTyped(ref Utf8JsonReaderManager manager, object? existingObject = null)
+        => Convert.FromHexString(manager.CurrentReader.GetString()!);
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public override void ToJsonTyped(Utf8JsonWriter writer, byte[] value)
+        => writer.WriteStringValue(Convert.ToHexString(value));
+}

src/EFCore.Sqlite.Core/Storage/Internal/Json/SqliteJsonByteArrayReaderWriter.cs

@@ -0,0 +1,59 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Globalization;
+using System.Text.Encodings.Web;
+using System.Text.Json;
+using Microsoft.EntityFrameworkCore.Storage.Json;
+
+namespace Microsoft.EntityFrameworkCore.Sqlite.Storage.Internal.Json;
+
+/// <summary>
+///     The Sqlite-specific JsonValueReaderWrite for DateTime. Generates a ISO8601 string representation with a space instead of a T
+///     separating the date and time components, in order to match our SQLite non-JSON representation.
+/// </summary>
+/// <remarks>
+///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+///     any release. You should only use it directly in your code with extreme caution and knowing that
+///     doing so can result in application failures when updating to a new Entity Framework Core release.
+/// </remarks>
+public sealed class SqliteJsonDateTimeOffsetReaderWriter : JsonValueReaderWriter<DateTimeOffset>
+{
+    private const string DateTimeOffsetFormatConst = @"{0:yyyy\-MM\-dd HH\:mm\:ss.FFFFFFFzzz}";
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public static SqliteJsonDateTimeOffsetReaderWriter Instance { get; } = new();
+
+    private SqliteJsonDateTimeOffsetReaderWriter()
+    {
+    }
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public override DateTimeOffset FromJsonTyped(ref Utf8JsonReaderManager manager, object? existingObject = null)
+        // => manager.CurrentReader.GetDateTimeOffset();
+        => DateTimeOffset.Parse(manager.CurrentReader.GetString()!, CultureInfo.InvariantCulture);
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public override void ToJsonTyped(Utf8JsonWriter writer, DateTimeOffset value)
+        // We use UnsafeRelaxedJsonEscaping to prevent the DateTimeOffset plus (+) sign from getting escaped
+        => writer.WriteStringValue(
+            JsonEncodedText.Encode(
+                string.Format(CultureInfo.InvariantCulture, DateTimeOffsetFormatConst, value),
+                JavaScriptEncoder.UnsafeRelaxedJsonEscaping));
+}

src/EFCore.Sqlite.Core/Storage/Internal/Json/SqliteJsonDateTimeOffsetReaderWriter.cs

@@ -0,0 +1,53 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Globalization;
+using System.Text.Json;
+using Microsoft.EntityFrameworkCore.Storage.Json;
+
+namespace Microsoft.EntityFrameworkCore.Sqlite.Storage.Internal.Json;
+
+/// <summary>
+///     The Sqlite-specific JsonValueReaderWrite for DateTime. Generates a ISO8601 string representation with a space instead of a T
+///     separating the date and time components, in order to match our SQLite non-JSON representation.
+/// </summary>
+/// <remarks>
+///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+///     any release. You should only use it directly in your code with extreme caution and knowing that
+///     doing so can result in application failures when updating to a new Entity Framework Core release.
+/// </remarks>
+public sealed class SqliteJsonDateTimeReaderWriter : JsonValueReaderWriter<DateTime>
+{
+    private const string DateTimeFormatConst = @"{0:yyyy\-MM\-dd HH\:mm\:ss.FFFFFFF}";
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public static SqliteJsonDateTimeReaderWriter Instance { get; } = new();
+
+    private SqliteJsonDateTimeReaderWriter()
+    {
+    }
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public override DateTime FromJsonTyped(ref Utf8JsonReaderManager manager, object? existingObject = null)
+        => DateTime.Parse(manager.CurrentReader.GetString()!, CultureInfo.InvariantCulture);
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public override void ToJsonTyped(Utf8JsonWriter writer, DateTime value)
+        => writer.WriteStringValue(string.Format(CultureInfo.InvariantCulture, DateTimeFormatConst, value));
+}

src/EFCore.Sqlite.Core/Storage/Internal/Json/SqliteJsonDateTimeReaderWriter.cs

@@ -0,0 +1,53 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Globalization;
+using System.Text.Json;
+using Microsoft.EntityFrameworkCore.Storage.Json;
+
+namespace Microsoft.EntityFrameworkCore.Sqlite.Storage.Internal.Json;
+
+/// <summary>
+///     The Sqlite-specific JsonValueReaderWrite for decimal. Generates a string representation instead of a JSON number, in order to match
+///     our SQLite non-JSON representation.
+/// </summary>
+/// <remarks>
+///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+///     any release. You should only use it directly in your code with extreme caution and knowing that
+///     doing so can result in application failures when updating to a new Entity Framework Core release.
+/// </remarks>
+public sealed class SqliteJsonDecimalReaderWriter : JsonValueReaderWriter<decimal>
+{
+    private const string DecimalFormatConst = "{0:0.0###########################}";
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public static SqliteJsonDecimalReaderWriter Instance { get; } = new();
+
+    private SqliteJsonDecimalReaderWriter()
+    {
+    }
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public override decimal FromJsonTyped(ref Utf8JsonReaderManager manager, object? existingObject = null)
+        => decimal.Parse(manager.CurrentReader.GetString()!, CultureInfo.InvariantCulture);
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public override void ToJsonTyped(Utf8JsonWriter writer, decimal value)
+        => writer.WriteStringValue(string.Format(CultureInfo.InvariantCulture, DecimalFormatConst, value));
+}