diff --git a/src/libraries/Microsoft.Bcl.Memory/Microsoft.Bcl.Memory.sln b/src/libraries/Microsoft.Bcl.Memory/Microsoft.Bcl.Memory.sln new file mode 100644 index 00000000000000..4309802637cf16 --- /dev/null +++ b/src/libraries/Microsoft.Bcl.Memory/Microsoft.Bcl.Memory.sln @@ -0,0 +1,48 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.10.35004.147 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Bcl.Memory", "ref\Microsoft.Bcl.Memory.csproj", "{97D0AA68-5D2E-4E18-9651-19D0143EA98A}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Bcl.Memory", "src\Microsoft.Bcl.Memory.csproj", "{D4D9D9AC-E482-457A-8047-5267BDF2C2EE}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.Bcl.Memory.Tests", "tests\Microsoft.Bcl.Memory.Tests.csproj", "{6916DB98-2958-498D-83E7-90080BEA9D39}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{76C22FE9-D65E-40C3-BE2D-98BD628779EC}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "ref", "ref", "{76EDDEA8-0E0E-4A94-9D8F-F4DB250E8EB8}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "tests", "tests", "{E78D7A3F-384B-4AA3-ABF1-C4464BE9237C}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {D4D9D9AC-E482-457A-8047-5267BDF2C2EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D4D9D9AC-E482-457A-8047-5267BDF2C2EE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D4D9D9AC-E482-457A-8047-5267BDF2C2EE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D4D9D9AC-E482-457A-8047-5267BDF2C2EE}.Release|Any CPU.Build.0 = Release|Any CPU + {97D0AA68-5D2E-4E18-9651-19D0143EA98A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {97D0AA68-5D2E-4E18-9651-19D0143EA98A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {97D0AA68-5D2E-4E18-9651-19D0143EA98A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {97D0AA68-5D2E-4E18-9651-19D0143EA98A}.Release|Any CPU.Build.0 = Release|Any CPU + {6916DB98-2958-498D-83E7-90080BEA9D39}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6916DB98-2958-498D-83E7-90080BEA9D39}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6916DB98-2958-498D-83E7-90080BEA9D39}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6916DB98-2958-498D-83E7-90080BEA9D39}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(NestedProjects) = preSolution + {D4D9D9AC-E482-457A-8047-5267BDF2C2EE} = {76C22FE9-D65E-40C3-BE2D-98BD628779EC} + {97D0AA68-5D2E-4E18-9651-19D0143EA98A} = {76EDDEA8-0E0E-4A94-9D8F-F4DB250E8EB8} + {6916DB98-2958-498D-83E7-90080BEA9D39} = {E78D7A3F-384B-4AA3-ABF1-C4464BE9237C} + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {CB4D3234-B365-4A38-B9C6-47B72CE8BC85} + EndGlobalSection +EndGlobal diff --git a/src/libraries/Microsoft.Bcl.Memory/ref/Microsoft.Bcl.Memory.Forwards.cs b/src/libraries/Microsoft.Bcl.Memory/ref/Microsoft.Bcl.Memory.Forwards.cs new file mode 100644 index 00000000000000..c732dbfb671a8f --- /dev/null +++ b/src/libraries/Microsoft.Bcl.Memory/ref/Microsoft.Bcl.Memory.Forwards.cs @@ -0,0 +1,4 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +[assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.Buffers.Text.Base64Url))] diff --git a/src/libraries/Microsoft.Bcl.Memory/ref/Microsoft.Bcl.Memory.cs b/src/libraries/Microsoft.Bcl.Memory/ref/Microsoft.Bcl.Memory.cs new file mode 100644 index 00000000000000..5ec52cbbe15d92 --- /dev/null +++ b/src/libraries/Microsoft.Bcl.Memory/ref/Microsoft.Bcl.Memory.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace System.Buffers.Text +{ + public static class Base64Url + { + public static byte[] DecodeFromChars(System.ReadOnlySpan source) { throw null; } + public static int DecodeFromChars(System.ReadOnlySpan source, System.Span destination) { throw null; } + public static System.Buffers.OperationStatus DecodeFromChars(System.ReadOnlySpan source, System.Span destination, out int charsConsumed, out int bytesWritten, bool isFinalBlock = true) { throw null; } + public static byte[] DecodeFromUtf8(System.ReadOnlySpan source) { throw null; } + public static int DecodeFromUtf8(System.ReadOnlySpan source, System.Span destination) { throw null; } + public static System.Buffers.OperationStatus DecodeFromUtf8(System.ReadOnlySpan source, System.Span destination, out int bytesConsumed, out int bytesWritten, bool isFinalBlock = true) { throw null; } + public static int DecodeFromUtf8InPlace(System.Span buffer) { throw null; } + public static char[] EncodeToChars(System.ReadOnlySpan source) { throw null; } + public static int EncodeToChars(System.ReadOnlySpan source, System.Span destination) { throw null; } + public static System.Buffers.OperationStatus EncodeToChars(System.ReadOnlySpan source, System.Span destination, out int bytesConsumed, out int charsWritten, bool isFinalBlock = true) { throw null; } + public static string EncodeToString(System.ReadOnlySpan source) { throw null; } + public static byte[] EncodeToUtf8(System.ReadOnlySpan source) { throw null; } + public static int EncodeToUtf8(System.ReadOnlySpan source, System.Span destination) { throw null; } + public static System.Buffers.OperationStatus EncodeToUtf8(System.ReadOnlySpan source, System.Span destination, out int bytesConsumed, out int bytesWritten, bool isFinalBlock = true) { throw null; } + public static int GetEncodedLength(int bytesLength) { throw null; } + public static int GetMaxDecodedLength(int base64Length) { throw null; } + public static bool IsValid(System.ReadOnlySpan base64UrlText) { throw null; } + public static bool IsValid(System.ReadOnlySpan base64UrlText, out int decodedLength) { throw null; } + public static bool IsValid(System.ReadOnlySpan utf8Base64UrlText) { throw null; } + public static bool IsValid(System.ReadOnlySpan utf8Base64UrlText, out int decodedLength) { throw null; } + public static bool TryDecodeFromChars(System.ReadOnlySpan source, System.Span destination, out int bytesWritten) { throw null; } + public static bool TryDecodeFromUtf8(System.ReadOnlySpan source, System.Span destination, out int bytesWritten) { throw null; } + public static bool TryEncodeToChars(System.ReadOnlySpan source, System.Span destination, out int charsWritten) { throw null; } + public static bool TryEncodeToUtf8(System.ReadOnlySpan source, System.Span destination, out int bytesWritten) { throw null; } + public static bool TryEncodeToUtf8InPlace(System.Span buffer, int dataLength, out int bytesWritten) { throw null; } + } +} diff --git a/src/libraries/Microsoft.Bcl.Memory/ref/Microsoft.Bcl.Memory.csproj b/src/libraries/Microsoft.Bcl.Memory/ref/Microsoft.Bcl.Memory.csproj new file mode 100644 index 00000000000000..a122f2cfeebb50 --- /dev/null +++ b/src/libraries/Microsoft.Bcl.Memory/ref/Microsoft.Bcl.Memory.csproj @@ -0,0 +1,19 @@ + + + + netstandard2.0;$(NetFrameworkMinimum);$(NetCoreAppCurrent) + + + + + + + + + + + + + + + diff --git a/src/libraries/Microsoft.Bcl.Memory/src/Microsoft.Bcl.Memory.csproj b/src/libraries/Microsoft.Bcl.Memory/src/Microsoft.Bcl.Memory.csproj new file mode 100644 index 00000000000000..8c8600b3c459dd --- /dev/null +++ b/src/libraries/Microsoft.Bcl.Memory/src/Microsoft.Bcl.Memory.csproj @@ -0,0 +1,56 @@ + + + + netstandard2.0;$(NetFrameworkMinimum);$(NetCoreAppCurrent) + true + true + + true + + Provides Base64Url encoding, decoding and validation APIs support for .NET Framework and .NET Standard. + + Commonly Used Types: + System.Buffers.Text.Base64Url + + + + + + true + + + + + + + + + System\Buffers\Text\Base64Helper\Base64Helper.cs + + + System\Buffers\Text\Base64Helper\Base64DecoderHelper.cs + + + System\Buffers\Text\Base64Helper\Base64EncoderHelper.cs + + + System\Buffers\Text\Base64Helper\Base64ValidatorHelper.cs + + + System\Buffers\Text\Base64Url\Base64UrlDecoder.cs + + + System\Buffers\Text\Base64Url\Base64UrlEncoder.cs + + + System\Buffers\Text\Base64Url\Base64UrlValidator.cs + + + + + + + + diff --git a/src/libraries/Microsoft.Bcl.Memory/src/Resources/Strings.resx b/src/libraries/Microsoft.Bcl.Memory/src/Resources/Strings.resx new file mode 100644 index 00000000000000..2d592b1943b671 --- /dev/null +++ b/src/libraries/Microsoft.Bcl.Memory/src/Resources/Strings.resx @@ -0,0 +1,126 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + text/microsoft-resx + + + 2.0 + + + System.Resources.ResXResourceReader, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + System.Resources.ResXResourceWriter, System.Windows.Forms, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089 + + + Destination is too short. + + + The input is not a valid Base64Url string as it contains a non-Base64Url character, more than two padding characters, or an illegal character among the padding characters. + + \ No newline at end of file diff --git a/src/libraries/Microsoft.Bcl.Memory/tests/Microsoft.Bcl.Memory.Tests.csproj b/src/libraries/Microsoft.Bcl.Memory/tests/Microsoft.Bcl.Memory.Tests.csproj new file mode 100644 index 00000000000000..f3f7262755c19d --- /dev/null +++ b/src/libraries/Microsoft.Bcl.Memory/tests/Microsoft.Bcl.Memory.Tests.csproj @@ -0,0 +1,33 @@ + + + + $(NetFrameworkMinimum);$(NetCoreAppCurrent); + true + + + + + System\Memory\AllocationHelper.cs + + + System\Memory\Base64\Base64TestBase.cs + + + System\Memory\Base64\Base64TestHelper.cs + + + System\Memory\Base64Url\Base64UrlDecoderUnitTests.cs + + + System\Memory\Base64Url\Base64UrlEncoderUnitTests.cs + + + System\Memory\Base64Url\Base64UrlValidationUnitTests.cs + + + + + + + + diff --git a/src/libraries/System.Memory/tests/Base64/Base64TestBase.cs b/src/libraries/System.Memory/tests/Base64/Base64TestBase.cs index 828442602c7f22..d3070aaf76c4b7 100644 --- a/src/libraries/System.Memory/tests/Base64/Base64TestBase.cs +++ b/src/libraries/System.Memory/tests/Base64/Base64TestBase.cs @@ -113,7 +113,7 @@ public static IEnumerable BasicDecodingWithExtraWhitespaceShouldBeCoun var r = new Random(42); for (int i = 0; i < 5; i++) { - yield return new object[] { "AQ==" + new string(r.GetItems(" \n\t\r", i)), 4 + i, 1 }; + yield return new object[] { "AQ==" + new string(GetChars(r, i)), 4 + i, 1 }; } foreach (string s in new[] { "MTIz", "M TIz", "MT Iz", "MTI z", "MTIz ", "M TI z", "M T I Z " }) @@ -121,5 +121,27 @@ public static IEnumerable BasicDecodingWithExtraWhitespaceShouldBeCoun yield return new object[] { s + s + s + s, s.Length * 4, 12 }; } } + + private static char[] GetChars(Random r, int i) + { +#if NETCOREAPP + return r.GetItems(" \n\t\r", i); +#else + byte[] bytes = new byte[i]; + char[] chars = new char[i]; + r.NextBytes(bytes); + for (int j = 0; j < bytes.Length; j++) + { + switch (bytes[j] % 4) + { + case 0: chars[j] = ' '; break; + case 1: chars[j] = '\n'; break; + case 2: chars[j] = '\t'; break; + case 3: chars[j] = '\r'; break; + } + } + return chars; +#endif + } } } diff --git a/src/libraries/System.Memory/tests/Base64/Base64TestHelper.cs b/src/libraries/System.Memory/tests/Base64/Base64TestHelper.cs index 42dca70bd66605..95da69c0164c9b 100644 --- a/src/libraries/System.Memory/tests/Base64/Base64TestHelper.cs +++ b/src/libraries/System.Memory/tests/Base64/Base64TestHelper.cs @@ -174,29 +174,29 @@ public static int[] FindAllIndexOf(this IEnumerable values, T valueToFind) public static bool VerifyEncodingCorrectness(int expectedConsumed, int expectedWritten, Span source, Span encodedBytes) { - string expectedText = Convert.ToBase64String(source.Slice(0, expectedConsumed)); - string encodedText = Encoding.ASCII.GetString(encodedBytes.Slice(0, expectedWritten)); + string expectedText = Convert.ToBase64String(source.Slice(0, expectedConsumed).ToArray()); + string encodedText = Encoding.ASCII.GetString(encodedBytes.Slice(0, expectedWritten).ToArray()); return expectedText.Equals(encodedText); } public static bool VerifyUrlEncodingCorrectness(int expectedConsumed, int expectedWritten, Span source, Span encodedBytes) { - string expectedText = Convert.ToBase64String(source.Slice(0, expectedConsumed)) + string expectedText = Convert.ToBase64String(source.Slice(0, expectedConsumed).ToArray()) .Replace('+', '-').Replace('/', '_').TrimEnd('='); - string encodedText = Encoding.ASCII.GetString(encodedBytes.Slice(0, expectedWritten)); + string encodedText = Encoding.ASCII.GetString(encodedBytes.Slice(0, expectedWritten).ToArray()); return expectedText.Equals(encodedText); } public static bool VerifyDecodingCorrectness(int expectedConsumed, int expectedWritten, Span source, Span decodedBytes) { - string sourceString = Encoding.ASCII.GetString(source.Slice(0, expectedConsumed)); + string sourceString = Encoding.ASCII.GetString(source.Slice(0, expectedConsumed).ToArray()); byte[] expectedBytes = Convert.FromBase64String(sourceString); return expectedBytes.AsSpan().SequenceEqual(decodedBytes.Slice(0, expectedWritten)); } public static bool VerifyUrlDecodingCorrectness(int expectedConsumed, int expectedWritten, Span source, Span decodedBytes) { - string sourceString = Encoding.ASCII.GetString(source.Slice(0, expectedConsumed)); + string sourceString = Encoding.ASCII.GetString(source.Slice(0, expectedConsumed).ToArray()); string padded = sourceString.Length % 4 == 0 ? sourceString : sourceString.PadRight(sourceString.Length + (4 - sourceString.Length % 4), '='); string base64 = padded.Replace('_', '/').Replace('-', '+').Replace('%', '='); diff --git a/src/libraries/System.Memory/tests/Base64Url/Base64UrlDecoderUnitTests.cs b/src/libraries/System.Memory/tests/Base64Url/Base64UrlDecoderUnitTests.cs index 23a16f6bb07a37..1a321a7ba9544b 100644 --- a/src/libraries/System.Memory/tests/Base64Url/Base64UrlDecoderUnitTests.cs +++ b/src/libraries/System.Memory/tests/Base64Url/Base64UrlDecoderUnitTests.cs @@ -801,6 +801,29 @@ public void DecodingLessThan4BytesWithWhiteSpaces(byte[] utf8Bytes, byte decoded Assert.Equal(decoded, utf8Bytes[0]); } + [Theory] + [InlineData(new char[] { '\r', '\r', '-', '-' }, 251)] + [InlineData(new char[] { '\r', '_', '\r', '-' }, 255)] + [InlineData(new char[] { '_', '_', '\r', '\r' }, 255)] + [InlineData(new char[] { 'p', '\r', 'a', '\r' }, 165)] + [InlineData(new char[] { '\r', 'p', '\r', 'a', '\r' }, 165)] + [InlineData(new char[] { 'p', '\r', 'a', '\r', '=', '=' }, 165)] + public void DecodingLessThan4CharsWithWhiteSpaces(char[] utf8Bytes, byte decoded) + { + Assert.True(Base64Url.IsValid(utf8Bytes, out int decodedLength)); + Assert.Equal(1, decodedLength); + Span decodedSpan = new byte[decodedLength]; + OperationStatus status = Base64Url.DecodeFromChars(utf8Bytes, decodedSpan, out int bytesRead, out int bytesDecoded); + Assert.Equal(OperationStatus.Done, status); + Assert.Equal(utf8Bytes.Length, bytesRead); + Assert.Equal(decodedLength, bytesDecoded); + Assert.Equal(decoded, decodedSpan[0]); + decodedSpan.Clear(); + Assert.True(Base64Url.TryDecodeFromChars(utf8Bytes, decodedSpan, out bytesDecoded)); + Assert.Equal(decodedLength, bytesDecoded); + Assert.Equal(decoded, decodedSpan[0]); + } + [Theory] [InlineData(new byte[] { 0x4a, 0x74, 0xa, 0x4a, 0x4a, 0x74, 0xa, 0x4a }, new byte[] { 38, 210, 73, 180 })] [InlineData(new byte[] { 0xa, 0x2d, 0x56, 0xa, 0xa, 0xa, 0x2d, 0x4a, 0x4a, 0x4a, }, new byte[] { 249, 95, 137, 36 })] @@ -813,14 +836,33 @@ public void DecodingNotMultipleOf4WithWhiteSpace(byte[] utf8Bytes, byte[] decode Assert.Equal(OperationStatus.Done, status); Assert.Equal(utf8Bytes.Length, bytesRead); Assert.Equal(decodedLength, bytesDecoded); - Assert.Equal(decoded, decodedSpan); + Assert.True(decodedSpan.SequenceEqual(decoded)); decodedSpan.Clear(); Assert.True(Base64Url.TryDecodeFromUtf8(utf8Bytes, decodedSpan, out bytesDecoded)); Assert.Equal(decodedLength, bytesDecoded); - Assert.Equal(decoded, decodedSpan); + Assert.True(decodedSpan.SequenceEqual(decoded)); bytesDecoded = Base64Url.DecodeFromUtf8InPlace(utf8Bytes); Assert.Equal(decodedLength, bytesDecoded); - Assert.Equal(decoded, utf8Bytes.AsSpan().Slice(0, bytesDecoded)); + Assert.True(utf8Bytes.AsSpan().Slice(0, bytesDecoded).SequenceEqual(decoded)); + } + + [Theory] + [InlineData(new char[] { 'J', 't', '\r', 'J', 'J', 't', '\r', 'J' }, new byte[] { 38, 210, 73, 180 })] + [InlineData(new char[] { '\r', '-', 'V', '\r', '\r', '\r', '-', 'J', 'J', 'J', }, new byte[] { 249, 95, 137, 36 })] + public void DecodingNotMultipleOf4CharsWithWhiteSpace(char[] utf8Bytes, byte[] decoded) + { + Assert.True(Base64Url.IsValid(utf8Bytes, out int decodedLength)); + Assert.Equal(4, decodedLength); + Span decodedSpan = new byte[decodedLength]; + OperationStatus status = Base64Url.DecodeFromChars(utf8Bytes, decodedSpan, out int bytesRead, out int bytesDecoded); + Assert.Equal(OperationStatus.Done, status); + Assert.Equal(utf8Bytes.Length, bytesRead); + Assert.Equal(decodedLength, bytesDecoded); + Assert.True(decodedSpan.SequenceEqual(decoded)); + decodedSpan.Clear(); + Assert.True(Base64Url.TryDecodeFromChars(utf8Bytes, decodedSpan, out bytesDecoded)); + Assert.Equal(decodedLength, bytesDecoded); + Assert.True(decodedSpan.SequenceEqual(decoded)); } [Theory] diff --git a/src/libraries/System.Memory/tests/Base64Url/Base64UrlValidationUnitTests.cs b/src/libraries/System.Memory/tests/Base64Url/Base64UrlValidationUnitTests.cs index b2cb650fa6e3f6..410aec37d88306 100644 --- a/src/libraries/System.Memory/tests/Base64Url/Base64UrlValidationUnitTests.cs +++ b/src/libraries/System.Memory/tests/Base64Url/Base64UrlValidationUnitTests.cs @@ -21,7 +21,7 @@ public class Base64UrlValidationUnitTests : Base64TestBase [InlineData("6066=")] public void BasicValidationEdgeCaseScenario(string base64UrlText) { - Assert.False(Base64Url.IsValid(base64UrlText, out int decodedLength)); + Assert.False(Base64Url.IsValid(base64UrlText.AsSpan(), out int decodedLength)); Assert.Equal(0, decodedLength); } @@ -277,7 +277,7 @@ public void SmallSizeBytes(string utf8Text, bool isValid, int expectedDecodedLen [InlineData("Y", false, 0)] public void SmallSizeChars(string utf8Text, bool isValid, int expectedDecodedLength) { - ReadOnlySpan utf8BytesWithByteToBeIgnored = utf8Text; + ReadOnlySpan utf8BytesWithByteToBeIgnored = utf8Text.AsSpan(); Assert.Equal(isValid, Base64Url.IsValid(utf8BytesWithByteToBeIgnored)); Assert.Equal(isValid, Base64Url.IsValid(utf8BytesWithByteToBeIgnored, out int decodedLength)); @@ -341,7 +341,7 @@ public void InvalidBase64UrlBytes(string utf8WithByteToBeIgnored) [InlineData(" a ")] public void InvalidBase64UrlChars(string utf8WithByteToBeIgnored) { - ReadOnlySpan utf8CharsWithCharToBeIgnored = utf8WithByteToBeIgnored; + ReadOnlySpan utf8CharsWithCharToBeIgnored = utf8WithByteToBeIgnored.AsSpan(); Assert.False(Base64Url.IsValid(utf8CharsWithCharToBeIgnored)); Assert.False(Base64Url.IsValid(utf8CharsWithCharToBeIgnored, out int decodedLength)); diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index 77475d432d57a1..233d19f0d5b5f4 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -125,7 +125,10 @@ - + + + + diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Decoder.cs b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Decoder.cs index b2807ef3b710d4..dfd8a08063249e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Decoder.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Decoder.cs @@ -1,18 +1,13 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Diagnostics; using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; -using System.Runtime.Intrinsics.X86; +using static System.Buffers.Text.Base64Helper; namespace System.Buffers.Text { // AVX2 version based on https://github.com/aklomp/base64/tree/e516d769a2a432c08404f1981e73b431566057be/lib/arch/avx2 // Vector128 version based on https://github.com/aklomp/base64/tree/e516d769a2a432c08404f1981e73b431566057be/lib/arch/ssse3 - public static partial class Base64 { /// @@ -35,298 +30,7 @@ public static partial class Base64 /// or if the input is incomplete (i.e. not a multiple of 4) and is . /// public static OperationStatus DecodeFromUtf8(ReadOnlySpan utf8, Span bytes, out int bytesConsumed, out int bytesWritten, bool isFinalBlock = true) => - DecodeFrom(utf8, bytes, out bytesConsumed, out bytesWritten, isFinalBlock, ignoreWhiteSpace: true); - - internal static unsafe OperationStatus DecodeFrom(ReadOnlySpan source, Span bytes, - out int bytesConsumed, out int bytesWritten, bool isFinalBlock, bool ignoreWhiteSpace) - where TBase64Decoder : IBase64Decoder - where T : unmanaged - { - if (source.IsEmpty) - { - bytesConsumed = 0; - bytesWritten = 0; - return OperationStatus.Done; - } - - fixed (T* srcBytes = &MemoryMarshal.GetReference(source)) - fixed (byte* destBytes = &MemoryMarshal.GetReference(bytes)) - { - int srcLength = TBase64Decoder.SrcLength(isFinalBlock, source.Length); - int destLength = bytes.Length; - int maxSrcLength = srcLength; - int decodedLength = TBase64Decoder.GetMaxDecodedLength(srcLength); - - // max. 2 padding chars - if (destLength < decodedLength - 2) - { - // For overflow see comment below - maxSrcLength = destLength / 3 * 4; - } - - T* src = srcBytes; - byte* dest = destBytes; - T* srcEnd = srcBytes + (uint)srcLength; - T* srcMax = srcBytes + (uint)maxSrcLength; - - if (maxSrcLength >= 24) - { - T* end = srcMax - 88; - if (Vector512.IsHardwareAccelerated && Avx512Vbmi.IsSupported && (end >= src)) - { - Avx512Decode(ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); - - if (src == srcEnd) - { - goto DoneExit; - } - } - - end = srcMax - 45; - if (Avx2.IsSupported && (end >= src)) - { - Avx2Decode(ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); - - if (src == srcEnd) - { - goto DoneExit; - } - } - - end = srcMax - 66; - if (AdvSimd.Arm64.IsSupported && (end >= src)) - { - AdvSimdDecode(ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); - - if (src == srcEnd) - { - goto DoneExit; - } - } - - end = srcMax - 24; - if ((Ssse3.IsSupported || AdvSimd.Arm64.IsSupported) && BitConverter.IsLittleEndian && (end >= src)) - { - Vector128Decode(ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); - - if (src == srcEnd) - { - goto DoneExit; - } - } - } - - // Last bytes could have padding characters, so process them separately and treat them as valid only if isFinalBlock is true - // if isFinalBlock is false, padding characters are considered invalid - int skipLastChunk = isFinalBlock ? 4 : 0; - - if (destLength >= decodedLength) - { - maxSrcLength = srcLength - skipLastChunk; - } - else - { - // This should never overflow since destLength here is less than int.MaxValue / 4 * 3 (i.e. 1610612733) - // Therefore, (destLength / 3) * 4 will always be less than 2147483641 - Debug.Assert(destLength < (int.MaxValue / 4 * 3)); - (maxSrcLength, int remainder) = int.DivRem(destLength, 3); - maxSrcLength *= 4; - if (isFinalBlock && remainder > 0) - { - srcLength &= ~0x3; // In case of Base64UrlDecoder source can be not a multiple of 4, round down to multiple of 4 - } - } - - ref sbyte decodingMap = ref MemoryMarshal.GetReference(TBase64Decoder.DecodingMap); - srcMax = srcBytes + maxSrcLength; - - while (src < srcMax) - { - int result = TBase64Decoder.DecodeFourElements(src, ref decodingMap); - - if (result < 0) - { - goto InvalidDataExit; - } - - WriteThreeLowOrderBytes(dest, result); - src += 4; - dest += 3; - } - - if (maxSrcLength != srcLength - skipLastChunk) - { - goto DestinationTooSmallExit; - } - - if (src == srcEnd) - { - if (isFinalBlock) - { - goto InvalidDataExit; - } - - if (src == srcBytes + source.Length) - { - goto DoneExit; - } - - goto NeedMoreDataExit; - } - - // if isFinalBlock is false, we will never reach this point - // Handle remaining bytes, for Base64 its always 4 bytes, for Base64Url up to 8 bytes left. - // If more than 4 bytes remained it will end up in DestinationTooSmallExit or InvalidDataExit (might succeed after whitespace removed) - long remaining = srcEnd - src; - Debug.Assert(typeof(TBase64Decoder) == typeof(Base64DecoderByte) ? remaining == 4 : remaining < 8); - int i0 = TBase64Decoder.DecodeRemaining(srcEnd, ref decodingMap, remaining, out uint t2, out uint t3); - - byte* destMax = destBytes + (uint)destLength; - - if (!TBase64Decoder.IsValidPadding(t3)) - { - int i2 = Unsafe.Add(ref decodingMap, (IntPtr)t2); - int i3 = Unsafe.Add(ref decodingMap, (IntPtr)t3); - - i2 <<= 6; - - i0 |= i3; - i0 |= i2; - - if (i0 < 0) - { - goto InvalidDataExit; - } - if (dest + 3 > destMax) - { - goto DestinationTooSmallExit; - } - - WriteThreeLowOrderBytes(dest, i0); - dest += 3; - src += 4; - } - else if (!TBase64Decoder.IsValidPadding(t2)) - { - int i2 = Unsafe.Add(ref decodingMap, (IntPtr)t2); - - i2 <<= 6; - - i0 |= i2; - - if (i0 < 0) - { - goto InvalidDataExit; - } - if (dest + 2 > destMax) - { - goto DestinationTooSmallExit; - } - - dest[0] = (byte)(i0 >> 16); - dest[1] = (byte)(i0 >> 8); - dest += 2; - src += remaining; - } - else - { - if (i0 < 0) - { - goto InvalidDataExit; - } - if (dest + 1 > destMax) - { - goto DestinationTooSmallExit; - } - - dest[0] = (byte)(i0 >> 16); - dest += 1; - src += remaining; - } - - if (srcLength != source.Length) - { - goto InvalidDataExit; - } - - DoneExit: - bytesConsumed = (int)(src - srcBytes); - bytesWritten = (int)(dest - destBytes); - return OperationStatus.Done; - - DestinationTooSmallExit: - if (srcLength != source.Length && isFinalBlock) - { - goto InvalidDataExit; // if input is not a multiple of 4, and there is no more data, return invalid data instead - } - - bytesConsumed = (int)(src - srcBytes); - bytesWritten = (int)(dest - destBytes); - return OperationStatus.DestinationTooSmall; - - NeedMoreDataExit: - bytesConsumed = (int)(src - srcBytes); - bytesWritten = (int)(dest - destBytes); - return OperationStatus.NeedMoreData; - - InvalidDataExit: - bytesConsumed = (int)(src - srcBytes); - bytesWritten = (int)(dest - destBytes); - return ignoreWhiteSpace ? - InvalidDataFallback(source, bytes, ref bytesConsumed, ref bytesWritten, isFinalBlock) : - OperationStatus.InvalidData; - } - - static OperationStatus InvalidDataFallback(ReadOnlySpan source, Span bytes, ref int bytesConsumed, ref int bytesWritten, bool isFinalBlock) - { - source = source.Slice(bytesConsumed); - bytes = bytes.Slice(bytesWritten); - - OperationStatus status; - do - { - int localConsumed = TBase64Decoder.IndexOfAnyExceptWhiteSpace(source); - if (localConsumed < 0) - { - // The remainder of the input is all whitespace. Mark it all as having been consumed, - // and mark the operation as being done. - bytesConsumed += source.Length; - status = OperationStatus.Done; - break; - } - - if (localConsumed == 0) - { - // Non-whitespace was found at the beginning of the input. Since it wasn't consumed - // by the previous call to DecodeFromUtf8, it must be part of a Base64 sequence - // that was interrupted by whitespace or something else considered invalid. - // Fall back to block-wise decoding. This is very slow, but it's also very non-standard - // formatting of the input; whitespace is typically only found between blocks, such as - // when Convert.ToBase64String inserts a line break every 76 output characters. - return TBase64Decoder.DecodeWithWhiteSpaceBlockwiseWrapper(source, bytes, ref bytesConsumed, ref bytesWritten, isFinalBlock); - } - - // Skip over the starting whitespace and continue. - bytesConsumed += localConsumed; - source = source.Slice(localConsumed); - - // Try again after consumed whitespace - status = DecodeFrom(source, bytes, out localConsumed, out int localWritten, isFinalBlock, ignoreWhiteSpace: false); - bytesConsumed += localConsumed; - bytesWritten += localWritten; - if (status is not OperationStatus.InvalidData) - { - break; - } - - source = source.Slice(localConsumed); - bytes = bytes.Slice(localWritten); - } - while (!source.IsEmpty); - - return status; - } - } + DecodeFrom(default(Base64DecoderByte), utf8, bytes, out bytesConsumed, out bytesWritten, isFinalBlock, ignoreWhiteSpace: true); /// /// Returns the maximum length (in bytes) of the result if you were to decode base 64 encoded text within a byte span of size "length". @@ -358,1139 +62,6 @@ public static int GetMaxDecodedFromUtf8Length(int length) /// hence can only be called once with all the data in the buffer. /// public static OperationStatus DecodeFromUtf8InPlace(Span buffer, out int bytesWritten) => - DecodeFromUtf8InPlace(buffer, out bytesWritten, ignoreWhiteSpace: true); - - internal static unsafe OperationStatus DecodeFromUtf8InPlace(Span buffer, out int bytesWritten, bool ignoreWhiteSpace) - where TBase64Decoder : IBase64Decoder - { - if (buffer.IsEmpty) - { - bytesWritten = 0; - return OperationStatus.Done; - } - - fixed (byte* bufferBytes = &MemoryMarshal.GetReference(buffer)) - { - uint bufferLength = (uint)buffer.Length; - uint sourceIndex = 0; - uint destIndex = 0; - - if (TBase64Decoder.IsInvalidLength(buffer.Length)) - { - goto InvalidExit; - } - - ref sbyte decodingMap = ref MemoryMarshal.GetReference(TBase64Decoder.DecodingMap); - - if (bufferLength > 4) - { - while (sourceIndex < bufferLength - 4) - { - int result = Base64DecoderByte.DecodeFourElements(bufferBytes + sourceIndex, ref decodingMap); - if (result < 0) - { - goto InvalidExit; - } - - WriteThreeLowOrderBytes(bufferBytes + destIndex, result); - destIndex += 3; - sourceIndex += 4; - } - } - - uint t0; - uint t1; - uint t2; - uint t3; - - switch (bufferLength - sourceIndex) - { - case 2: - t0 = bufferBytes[bufferLength - 2]; - t1 = bufferBytes[bufferLength - 1]; - t2 = EncodingPad; - t3 = EncodingPad; - break; - case 3: - t0 = bufferBytes[bufferLength - 3]; - t1 = bufferBytes[bufferLength - 2]; - t2 = bufferBytes[bufferLength - 1]; - t3 = EncodingPad; - break; - case 4: - t0 = bufferBytes[bufferLength - 4]; - t1 = bufferBytes[bufferLength - 3]; - t2 = bufferBytes[bufferLength - 2]; - t3 = bufferBytes[bufferLength - 1]; - break; - default: - goto InvalidExit; - } - - int i0 = Unsafe.Add(ref decodingMap, t0); - int i1 = Unsafe.Add(ref decodingMap, t1); - - i0 <<= 18; - i1 <<= 12; - - i0 |= i1; - - if (!TBase64Decoder.IsValidPadding(t3)) - { - int i2 = Unsafe.Add(ref decodingMap, t2); - int i3 = Unsafe.Add(ref decodingMap, t3); - - i2 <<= 6; - - i0 |= i3; - i0 |= i2; - - if (i0 < 0) - { - goto InvalidExit; - } - - WriteThreeLowOrderBytes(bufferBytes + destIndex, i0); - destIndex += 3; - } - else if (!TBase64Decoder.IsValidPadding(t2)) - { - int i2 = Unsafe.Add(ref decodingMap, t2); - - i2 <<= 6; - - i0 |= i2; - - if (i0 < 0) - { - goto InvalidExit; - } - - bufferBytes[destIndex] = (byte)(i0 >> 16); - bufferBytes[destIndex + 1] = (byte)(i0 >> 8); - destIndex += 2; - } - else - { - if (i0 < 0) - { - goto InvalidExit; - } - - bufferBytes[destIndex] = (byte)(i0 >> 16); - destIndex += 1; - } - - bytesWritten = (int)destIndex; - return OperationStatus.Done; - - InvalidExit: - bytesWritten = (int)destIndex; - return ignoreWhiteSpace ? - DecodeWithWhiteSpaceFromUtf8InPlace(buffer, ref bytesWritten, sourceIndex) : // The input may have whitespace, attempt to decode while ignoring whitespace. - OperationStatus.InvalidData; - } - } - - internal static OperationStatus DecodeWithWhiteSpaceBlockwise(ReadOnlySpan source, Span bytes, ref int bytesConsumed, ref int bytesWritten, bool isFinalBlock = true) - where TBase64Decoder : IBase64Decoder - { - const int BlockSize = 4; - Span buffer = stackalloc byte[BlockSize]; - OperationStatus status = OperationStatus.Done; - - while (!source.IsEmpty) - { - int encodedIdx = 0; - int bufferIdx = 0; - int skipped = 0; - - for (; encodedIdx < source.Length && (uint)bufferIdx < (uint)buffer.Length; ++encodedIdx) - { - if (IsWhiteSpace(source[encodedIdx])) - { - skipped++; - } - else - { - buffer[bufferIdx] = source[encodedIdx]; - bufferIdx++; - } - } - - source = source.Slice(encodedIdx); - bytesConsumed += skipped; - - if (bufferIdx == 0) - { - continue; - } - - bool hasAnotherBlock; - - if (typeof(TBase64Decoder) == typeof(Base64DecoderByte)) - { - hasAnotherBlock = source.Length >= BlockSize; - } - else - { - hasAnotherBlock = source.Length > 1; - } - - bool localIsFinalBlock = !hasAnotherBlock; - - // If this block contains padding and there's another block, then only whitespace may follow for being valid. - if (hasAnotherBlock) - { - int paddingCount = GetPaddingCount(ref buffer[^1]); - if (paddingCount > 0) - { - hasAnotherBlock = false; - localIsFinalBlock = true; - } - } - - if (localIsFinalBlock && !isFinalBlock) - { - localIsFinalBlock = false; - } - - status = DecodeFrom(buffer.Slice(0, bufferIdx), bytes, out int localConsumed, out int localWritten, localIsFinalBlock, ignoreWhiteSpace: false); - bytesConsumed += localConsumed; - bytesWritten += localWritten; - - if (status != OperationStatus.Done) - { - return status; - } - - // The remaining data must all be whitespace in order to be valid. - if (!hasAnotherBlock) - { - for (int i = 0; i < source.Length; ++i) - { - if (!IsWhiteSpace(source[i])) - { - // Revert previous dest increment, since an invalid state followed. - bytesConsumed -= localConsumed; - bytesWritten -= localWritten; - - return OperationStatus.InvalidData; - } - - bytesConsumed++; - } - - break; - } - - bytes = bytes.Slice(localWritten); - } - - return status; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetPaddingCount(ref byte ptrToLastElement) - where TBase64Decoder : IBase64Decoder - { - int padding = 0; - - if (TBase64Decoder.IsValidPadding(ptrToLastElement)) - { - padding++; - } - - if (TBase64Decoder.IsValidPadding(Unsafe.Subtract(ref ptrToLastElement, 1))) - { - padding++; - } - - return padding; - } - - private static OperationStatus DecodeWithWhiteSpaceFromUtf8InPlace(Span source, ref int destIndex, uint sourceIndex) - where TBase64Decoder : IBase64Decoder - { - int BlockSize = Math.Min(source.Length - (int)sourceIndex, 4); - Span buffer = stackalloc byte[BlockSize]; - - OperationStatus status = OperationStatus.Done; - int localDestIndex = destIndex; - bool hasPaddingBeenProcessed = false; - int localBytesWritten = 0; - - while (sourceIndex < (uint)source.Length) - { - int bufferIdx = 0; - - while (bufferIdx < BlockSize && sourceIndex < (uint)source.Length) - { - if (!IsWhiteSpace(source[(int)sourceIndex])) - { - buffer[bufferIdx] = source[(int)sourceIndex]; - bufferIdx++; - } - - sourceIndex++; - } - - if (bufferIdx == 0) - { - continue; - } - - if (bufferIdx != 4) - { - // Base64 require 4 bytes, for Base64Url it can be less than 4 bytes but not 1 byte. - if (typeof(TBase64Decoder) == typeof(Base64DecoderByte) || bufferIdx == 1) - { - status = OperationStatus.InvalidData; - break; - } - else // For Base64Url fill empty slots in last block with padding - { - while (bufferIdx < BlockSize) // Can happen only for last block - { - Debug.Assert(source.Length == sourceIndex); - buffer[bufferIdx++] = (byte)EncodingPad; - } - } - } - - if (hasPaddingBeenProcessed) - { - // Padding has already been processed, a new valid block cannot be processed. - // Revert previous dest increment, since an invalid state followed. - localDestIndex -= localBytesWritten; - status = OperationStatus.InvalidData; - break; - } - - status = DecodeFromUtf8InPlace(buffer, out localBytesWritten, ignoreWhiteSpace: false); - localDestIndex += localBytesWritten; - hasPaddingBeenProcessed = localBytesWritten < 3; - - if (status != OperationStatus.Done) - { - break; - } - - // Write result to source span in place. - for (int i = 0; i < localBytesWritten; i++) - { - source[localDestIndex - localBytesWritten + i] = buffer[i]; - } - } - - destIndex = localDestIndex; - return status; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Avx512BW))] - [CompExactlyDependsOn(typeof(Avx512Vbmi))] - private static unsafe void Avx512Decode(ref T* srcBytes, ref byte* destBytes, T* srcEnd, int sourceLength, int destLength, T* srcStart, byte* destStart) - where TBase64Decoder : IBase64Decoder - where T : unmanaged - { - // Reference for VBMI implementation : https://github.com/WojciechMula/base64simd/tree/master/decode - // If we have AVX512 support, pick off 64 bytes at a time for as long as we can, - // but make sure that we quit before seeing any == markers at the end of the - // string. Also, because we write 16 zeroes at the end of the output, ensure - // that there are at least 22 valid bytes of input data remaining to close the - // gap. 64 + 2 + 22 = 88 bytes. - T* src = srcBytes; - byte* dest = destBytes; - - // The JIT won't hoist these "constants", so help it - Vector512 vbmiLookup0 = Vector512.Create(TBase64Decoder.VbmiLookup0).AsSByte(); - Vector512 vbmiLookup1 = Vector512.Create(TBase64Decoder.VbmiLookup1).AsSByte(); - Vector512 vbmiPackedLanesControl = Vector512.Create( - 0x06000102, 0x090a0405, 0x0c0d0e08, 0x16101112, - 0x191a1415, 0x1c1d1e18, 0x26202122, 0x292a2425, - 0x2c2d2e28, 0x36303132, 0x393a3435, 0x3c3d3e38, - 0x00000000, 0x00000000, 0x00000000, 0x00000000).AsByte(); - - Vector512 mergeConstant0 = Vector512.Create(0x01400140).AsSByte(); - Vector512 mergeConstant1 = Vector512.Create(0x00011000).AsInt16(); - - // This algorithm requires AVX512VBMI support. - // Vbmi was first introduced in CannonLake and is available from IceLake on. - do - { - if (!TBase64Decoder.TryLoadVector512(src, srcStart, sourceLength, out Vector512 str)) - { - break; - } - - // Step 1: Translate encoded Base64 input to their original indices - // This step also checks for invalid inputs and exits. - // After this, we have indices which are verified to have upper 2 bits set to 0 in each byte. - // origIndex = [...|00dddddd|00cccccc|00bbbbbb|00aaaaaa] - Vector512 origIndex = Avx512Vbmi.PermuteVar64x8x2(vbmiLookup0, str, vbmiLookup1); - Vector512 errorVec = (origIndex.AsInt32() | str.AsInt32()).AsSByte(); - if (errorVec.ExtractMostSignificantBits() != 0) - { - break; - } - - // Step 2: Now we need to reshuffle bits to remove the 0 bits. - // multiAdd1: [...|0000cccc|ccdddddd|0000aaaa|aabbbbbb] - Vector512 multiAdd1 = Avx512BW.MultiplyAddAdjacent(origIndex.AsByte(), mergeConstant0); - // multiAdd1: [...|00000000|aaaaaabb|bbbbcccc|ccdddddd] - Vector512 multiAdd2 = Avx512BW.MultiplyAddAdjacent(multiAdd1, mergeConstant1); - - // Step 3: Pack 48 bytes - str = Avx512Vbmi.PermuteVar64x8(multiAdd2.AsByte(), vbmiPackedLanesControl).AsSByte(); - - AssertWrite>(dest, destStart, destLength); - str.Store((sbyte*)dest); - src += 64; - dest += 48; - } - while (src <= srcEnd); - - srcBytes = src; - destBytes = dest; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Avx2))] - private static unsafe void Avx2Decode(ref T* srcBytes, ref byte* destBytes, T* srcEnd, int sourceLength, int destLength, T* srcStart, byte* destStart) - where TBase64Decoder : IBase64Decoder - where T : unmanaged - { - // If we have AVX2 support, pick off 32 bytes at a time for as long as we can, - // but make sure that we quit before seeing any == markers at the end of the - // string. Also, because we write 8 zeroes at the end of the output, ensure - // that there are at least 11 valid bytes of input data remaining to close the - // gap. 32 + 2 + 11 = 45 bytes. - - // See SSSE3-version below for an explanation of how the code works. - - // The JIT won't hoist these "constants", so help it - Vector256 lutHi = Vector256.Create(TBase64Decoder.Avx2LutHigh); - - Vector256 lutLo = Vector256.Create(TBase64Decoder.Avx2LutLow); - - Vector256 lutShift = Vector256.Create(TBase64Decoder.Avx2LutShift); - - Vector256 packBytesInLaneMask = Vector256.Create( - 2, 1, 0, 6, - 5, 4, 10, 9, - 8, 14, 13, 12, - -1, -1, -1, -1, - 2, 1, 0, 6, - 5, 4, 10, 9, - 8, 14, 13, 12, - -1, -1, -1, -1); - - Vector256 packLanesControl = Vector256.Create( - 0, 0, 0, 0, - 1, 0, 0, 0, - 2, 0, 0, 0, - 4, 0, 0, 0, - 5, 0, 0, 0, - 6, 0, 0, 0, - -1, -1, -1, -1, - -1, -1, -1, -1).AsInt32(); - - Vector256 maskSlashOrUnderscore = Vector256.Create((sbyte)TBase64Decoder.MaskSlashOrUnderscore); - Vector256 shiftForUnderscore = Vector256.Create((sbyte)33); - Vector256 mergeConstant0 = Vector256.Create(0x01400140).AsSByte(); - Vector256 mergeConstant1 = Vector256.Create(0x00011000).AsInt16(); - - T* src = srcBytes; - byte* dest = destBytes; - - //while (remaining >= 45) - do - { - if (!TBase64Decoder.TryLoadAvxVector256(src, srcStart, sourceLength, out Vector256 str)) - { - break; - } - - Vector256 hiNibbles = Avx2.And(Avx2.ShiftRightLogical(str.AsInt32(), 4).AsSByte(), maskSlashOrUnderscore); - - if (!TBase64Decoder.TryDecode256Core(str, hiNibbles, maskSlashOrUnderscore, lutLo, lutHi, lutShift, shiftForUnderscore, out str)) - { - break; - } - - // in, lower lane, bits, upper case are most significant bits, lower case are least significant bits: - // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ - // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG - // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD - // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA - - Vector256 merge_ab_and_bc = Avx2.MultiplyAddAdjacent(str.AsByte(), mergeConstant0); - // 0000kkkk LLllllll 0000JJJJ JJjjKKKK - // 0000hhhh IIiiiiii 0000GGGG GGggHHHH - // 0000eeee FFffffff 0000DDDD DDddEEEE - // 0000bbbb CCcccccc 0000AAAA AAaaBBBB - - Vector256 output = Avx2.MultiplyAddAdjacent(merge_ab_and_bc, mergeConstant1); - // 00000000 JJJJJJjj KKKKkkkk LLllllll - // 00000000 GGGGGGgg HHHHhhhh IIiiiiii - // 00000000 DDDDDDdd EEEEeeee FFffffff - // 00000000 AAAAAAaa BBBBbbbb CCcccccc - - // Pack bytes together in each lane: - output = Avx2.Shuffle(output.AsSByte(), packBytesInLaneMask).AsInt32(); - // 00000000 00000000 00000000 00000000 - // LLllllll KKKKkkkk JJJJJJjj IIiiiiii - // HHHHhhhh GGGGGGgg FFffffff EEEEeeee - // DDDDDDdd CCcccccc BBBBbbbb AAAAAAaa - - // Pack lanes - str = Avx2.PermuteVar8x32(output, packLanesControl).AsSByte(); - - AssertWrite>(dest, destStart, destLength); - Avx.Store(dest, str.AsByte()); - - src += 32; - dest += 24; - } - while (src <= srcEnd); - - srcBytes = src; - destBytes = dest; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Ssse3))] - [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] - internal static Vector128 SimdShuffle(Vector128 left, Vector128 right, Vector128 mask8F) - { - Debug.Assert((Ssse3.IsSupported || AdvSimd.Arm64.IsSupported) && BitConverter.IsLittleEndian); - - if (AdvSimd.Arm64.IsSupported) - { - right &= mask8F; - } - - return Vector128.ShuffleUnsafe(left, right); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] - private static unsafe void AdvSimdDecode(ref T* srcBytes, ref byte* destBytes, T* srcEnd, int sourceLength, int destLength, T* srcStart, byte* destStart) - where TBase64Decoder : IBase64Decoder - where T : unmanaged - { - // C# implementation of https://github.com/aklomp/base64/blob/3a5add8652076612a8407627a42c768736a4263f/lib/arch/neon64/dec_loop.c - // If we have AdvSimd support, pick off 64 bytes at a time for as long as we can, - // but make sure that we quit before seeing any == markers at the end of the - // string. 64 + 2 = 66 bytes. - - // In the decoding process, we want to map each byte, representing a Base64 value, to its 6-bit (0-63) representation. - // It uses the following mapping. Values outside the following groups are invalid and, we abort decoding when encounter one. - // - // # From To Char - // 1 [43] [62] + - // 2 [47] [63] / - // 3 [48..57] [52..61] 0..9 - // 4 [65..90] [0..25] A..Z - // 5 [97..122] [26..51] a..z - // - // To map an input value to its Base64 representation, we use look-up tables 'decLutOne' and 'decLutTwo'. - // 'decLutOne' helps to map groups 1, 2 and 3 while 'decLutTwo' maps groups 4 and 5 in the above list. - // After mapping, each value falls between 0-63. Consequently, the last six bits of each byte now hold a valid value. - // We then compress four such bytes (with valid 4 * 6 = 24 bits) to three UTF8 bytes (3 * 8 = 24 bits). - // For faster decoding, we use SIMD operations that allow the processing of multiple bytes together. - // However, the compress operation on adjacent values of a vector could be slower. Thus, we de-interleave while reading - // the input bytes that store adjacent bytes in separate vectors. This later simplifies the compress step with the help - // of logical operations. This requires interleaving while storing the decoded result. - - // Values in 'decLutOne' maps input values from 0 to 63. - // 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 - // 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 - // 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 255, 63 - // 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 255, 255, 255 - var decLutOne = (Vector128.AllBitsSet, - Vector128.AllBitsSet, - Vector128.Create(TBase64Decoder.AdvSimdLutOne3).AsByte(), - Vector128.Create(0x37363534, 0x3B3A3938, 0xFFFF3D3C, 0xFFFFFFFF).AsByte()); - - // Values in 'decLutTwo' maps input values from 63 to 127. - // 0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 - // 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255 - // 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39 - // 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255 - var decLutTwo = (Vector128.Create(0x0100FF00, 0x05040302, 0x09080706, 0x0D0C0B0A).AsByte(), - Vector128.Create(0x11100F0E, 0x15141312, 0x19181716, 0xFFFFFFFF).AsByte(), - Vector128.Create(TBase64Decoder.AdvSimdLutTwo3Uint1, 0x1F1E1D1C, 0x23222120, 0x27262524).AsByte(), - Vector128.Create(0x2B2A2928, 0x2F2E2D2C, 0x33323130, 0xFFFFFFFF).AsByte()); - - T* src = srcBytes; - byte* dest = destBytes; - Vector128 offset = Vector128.Create(63); - - do - { - // Step 1: Load 64 bytes and de-interleave. - if (!TBase64Decoder.TryLoadArmVector128x4(src, srcStart, sourceLength, - out Vector128 str1, out Vector128 str2, out Vector128 str3, out Vector128 str4)) - { - break; - } - - // Step 2: Map each valid input to its Base64 value. - // We use two look-ups to compute partial results and combine them later. - - // Step 2.1: Detect valid Base64 values from the first three groups. Maps input as, - // 0 to 63 (Invalid) => 255 - // 0 to 63 (Valid) => Their Base64 equivalent - // 64 to 255 => 0 - - // Each input value acts as an index in the look-up table 'decLutOne'. - // e.g., for group 1: index 43 maps to 62 (Base64 '+'). - // Group 4 and 5 values are out-of-range (>64), so they are mapped to zero. - // Other valid indices but invalid values are mapped to 255. - Vector128 decOne1 = AdvSimd.Arm64.VectorTableLookup(decLutOne, str1); - Vector128 decOne2 = AdvSimd.Arm64.VectorTableLookup(decLutOne, str2); - Vector128 decOne3 = AdvSimd.Arm64.VectorTableLookup(decLutOne, str3); - Vector128 decOne4 = AdvSimd.Arm64.VectorTableLookup(decLutOne, str4); - - // Step 2.2: Detect valid Base64 values from groups 4 and 5. Maps input as, - // 0 to 63 => 0 - // 64 to 122 (Valid) => Their Base64 equivalent - // 64 to 122 (Invalid) => 255 - // 123 to 255 => Remains unchanged - - // Subtract/offset each input value by 63 so that it can be used as a valid offset. - // Subtract saturate makes values from the first three groups set to zero that are - // then mapped to zero in the subsequent look-up. - Vector128 decTwo1 = AdvSimd.SubtractSaturate(str1, offset); - Vector128 decTwo2 = AdvSimd.SubtractSaturate(str2, offset); - Vector128 decTwo3 = AdvSimd.SubtractSaturate(str3, offset); - Vector128 decTwo4 = AdvSimd.SubtractSaturate(str4, offset); - - // We use VTBX to map values where out-of-range indices are unchanged. - decTwo1 = AdvSimd.Arm64.VectorTableLookupExtension(decTwo1, decLutTwo, decTwo1); - decTwo2 = AdvSimd.Arm64.VectorTableLookupExtension(decTwo2, decLutTwo, decTwo2); - decTwo3 = AdvSimd.Arm64.VectorTableLookupExtension(decTwo3, decLutTwo, decTwo3); - decTwo4 = AdvSimd.Arm64.VectorTableLookupExtension(decTwo4, decLutTwo, decTwo4); - - // Step 3: Combine the partial result. - // Each look-up above maps valid values to their Base64 equivalent or zero. - // Thus the intermediate results 'decOne' and 'decTwo' could be OR-ed to get final values. - str1 = (decOne1 | decTwo1); - str2 = (decOne2 | decTwo2); - str3 = (decOne3 | decTwo3); - str4 = (decOne4 | decTwo4); - - // Step 4: Detect an invalid input value. - // Invalid values < 122 are set to 255 while the ones above 122 are unchanged. - // Check for invalid input, any value larger than 63. - Vector128 classified = (Vector128.GreaterThan(str1, offset) - | Vector128.GreaterThan(str2, offset) - | Vector128.GreaterThan(str3, offset) - | Vector128.GreaterThan(str4, offset)); - - // Check that all bits are zero. - if (classified != Vector128.Zero) - { - break; - } - - // Step 5: Compress four bytes into three. - Vector128 res1 = ((str1 << 2) | (str2 >> 4)); - Vector128 res2 = ((str2 << 4) | (str3 >> 2)); - Vector128 res3 = ((str3 << 6) | str4); - - // Step 6: Interleave and store decoded results. - AssertWrite>(dest, destStart, destLength); - AdvSimd.Arm64.StoreVectorAndZip(dest, (res1, res2, res3)); - - src += 64; - dest += 48; - } - while (src <= srcEnd); - - srcBytes = src; - destBytes = dest; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] - [CompExactlyDependsOn(typeof(Ssse3))] - private static unsafe void Vector128Decode(ref T* srcBytes, ref byte* destBytes, T* srcEnd, int sourceLength, int destLength, T* srcStart, byte* destStart) - where TBase64Decoder : IBase64Decoder - where T : unmanaged - { - Debug.Assert((Ssse3.IsSupported || AdvSimd.Arm64.IsSupported) && BitConverter.IsLittleEndian); - - // If we have Vector128 support, pick off 16 bytes at a time for as long as we can, - // but make sure that we quit before seeing any == markers at the end of the - // string. Also, because we write four zeroes at the end of the output, ensure - // that there are at least 6 valid bytes of input data remaining to close the - // gap. 16 + 2 + 6 = 24 bytes. - - // The input consists of six character sets in the Base64 alphabet, - // which we need to map back to the 6-bit values they represent. - // There are three ranges, two singles, and then there's the rest. - // - // # From To Add Characters - // 1 [43] [62] +19 + - // 2 [47] [63] +16 / - // 3 [48..57] [52..61] +4 0..9 - // 4 [65..90] [0..25] -65 A..Z - // 5 [97..122] [26..51] -71 a..z - // (6) Everything else => invalid input - - // We will use LUTS for character validation & offset computation - // Remember that 0x2X and 0x0X are the same index for _mm_shuffle_epi8, - // this allows to mask with 0x2F instead of 0x0F and thus save one constant declaration (register and/or memory access) - - // For offsets: - // Perfect hash for lut = ((src>>4)&0x2F)+((src==0x2F)?0xFF:0x00) - // 0000 = garbage - // 0001 = / - // 0010 = + - // 0011 = 0-9 - // 0100 = A-Z - // 0101 = A-Z - // 0110 = a-z - // 0111 = a-z - // 1000 >= garbage - - // For validation, here's the table. - // A character is valid if and only if the AND of the 2 lookups equals 0: - - // hi \ lo 0000 0001 0010 0011 0100 0101 0110 0111 1000 1001 1010 1011 1100 1101 1110 1111 - // LUT 0x15 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x13 0x1A 0x1B 0x1B 0x1B 0x1A - - // 0000 0X10 char NUL SOH STX ETX EOT ENQ ACK BEL BS HT LF VT FF CR SO SI - // andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 - - // 0001 0x10 char DLE DC1 DC2 DC3 DC4 NAK SYN ETB CAN EM SUB ESC FS GS RS US - // andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 - - // 0010 0x01 char ! " # $ % & ' ( ) * + , - . / - // andlut 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x00 0x01 0x01 0x01 0x00 - - // 0011 0x02 char 0 1 2 3 4 5 6 7 8 9 : ; < = > ? - // andlut 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x02 0x02 0x02 0x02 0x02 0x02 - - // 0100 0x04 char @ A B C D E F G H I J K L M N 0 - // andlut 0x04 0x00 0x00 0x00 0X00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 - - // 0101 0x08 char P Q R S T U V W X Y Z [ \ ] ^ _ - // andlut 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x08 0x08 0x08 0x08 0x08 - - // 0110 0x04 char ` a b c d e f g h i j k l m n o - // andlut 0x04 0x00 0x00 0x00 0X00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 - // 0111 0X08 char p q r s t u v w x y z { | } ~ - // andlut 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x08 0x08 0x08 0x08 0x08 - - // 1000 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 - // 1001 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 - // 1010 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 - // 1011 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 - // 1100 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 - // 1101 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 - // 1110 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 - // 1111 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 - - // The JIT won't hoist these "constants", so help it - Vector128 lutHi = Vector128.Create(TBase64Decoder.Vector128LutHigh).AsByte(); - Vector128 lutLo = Vector128.Create(TBase64Decoder.Vector128LutLow).AsByte(); - Vector128 lutShift = Vector128.Create(TBase64Decoder.Vector128LutShift).AsSByte(); - Vector128 packBytesMask = Vector128.Create(0x06000102, 0x090A0405, 0x0C0D0E08, 0xffffffff).AsSByte(); - Vector128 mergeConstant0 = Vector128.Create(0x01400140).AsByte(); - Vector128 mergeConstant1 = Vector128.Create(0x00011000).AsInt16(); - Vector128 one = Vector128.Create((byte)1); - Vector128 mask2F = Vector128.Create(TBase64Decoder.MaskSlashOrUnderscore); - Vector128 mask8F = Vector128.Create((byte)0x8F); - Vector128 shiftForUnderscore = Vector128.Create((byte)33); - T* src = srcBytes; - byte* dest = destBytes; - - //while (remaining >= 24) - do - { - if (!TBase64Decoder.TryLoadVector128(src, srcStart, sourceLength, out Vector128 str)) - { - break; - } - - // lookup - Vector128 hiNibbles = Vector128.ShiftRightLogical(str.AsInt32(), 4).AsByte() & mask2F; - - if (!TBase64Decoder.TryDecode128Core(str, hiNibbles, mask2F, mask8F, lutLo, lutHi, lutShift, shiftForUnderscore, out str)) - { - break; - } - - // in, bits, upper case are most significant bits, lower case are least significant bits - // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ - // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG - // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD - // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA - - Vector128 merge_ab_and_bc; - if (Ssse3.IsSupported) - { - merge_ab_and_bc = Ssse3.MultiplyAddAdjacent(str.AsByte(), mergeConstant0.AsSByte()); - } - else - { - Vector128 evens = AdvSimd.ShiftLeftLogicalWideningLower(AdvSimd.Arm64.UnzipEven(str, one).GetLower(), 6); - Vector128 odds = AdvSimd.Arm64.TransposeOdd(str, Vector128.Zero).AsUInt16(); - merge_ab_and_bc = Vector128.Add(evens, odds).AsInt16(); - } - // 0000kkkk LLllllll 0000JJJJ JJjjKKKK - // 0000hhhh IIiiiiii 0000GGGG GGggHHHH - // 0000eeee FFffffff 0000DDDD DDddEEEE - // 0000bbbb CCcccccc 0000AAAA AAaaBBBB - - Vector128 output; - if (Ssse3.IsSupported) - { - output = Sse2.MultiplyAddAdjacent(merge_ab_and_bc, mergeConstant1); - } - else - { - Vector128 ievens = AdvSimd.ShiftLeftLogicalWideningLower(AdvSimd.Arm64.UnzipEven(merge_ab_and_bc, one.AsInt16()).GetLower(), 12); - Vector128 iodds = AdvSimd.Arm64.TransposeOdd(merge_ab_and_bc, Vector128.Zero).AsInt32(); - output = Vector128.Add(ievens, iodds).AsInt32(); - } - // 00000000 JJJJJJjj KKKKkkkk LLllllll - // 00000000 GGGGGGgg HHHHhhhh IIiiiiii - // 00000000 DDDDDDdd EEEEeeee FFffffff - // 00000000 AAAAAAaa BBBBbbbb CCcccccc - - // Pack bytes together: - str = SimdShuffle(output.AsByte(), packBytesMask.AsByte(), mask8F); - // 00000000 00000000 00000000 00000000 - // LLllllll KKKKkkkk JJJJJJjj IIiiiiii - // HHHHhhhh GGGGGGgg FFffffff EEEEeeee - // DDDDDDdd CCcccccc BBBBbbbb AAAAAAaa - - AssertWrite>(dest, destStart, destLength); - str.Store(dest); - - src += 16; - dest += 12; - } - while (src <= srcEnd); - - srcBytes = src; - destBytes = dest; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe void WriteThreeLowOrderBytes(byte* destination, int value) - { - destination[0] = (byte)(value >> 16); - destination[1] = (byte)(value >> 8); - destination[2] = (byte)value; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - internal static bool IsWhiteSpace(int value) - { - if (Environment.Is64BitProcess) - { - // For description see https://github.com/dotnet/runtime/blob/48e74187cb15386c29eedaa046a5ee2c7ddef161/src/libraries/Common/src/System/HexConverter.cs#L314-L330 - // Lookup bit mask for "\t\n\r ". - const ulong MagicConstant = 0xC800010000000000UL; - ulong i = (uint)value - '\t'; - ulong shift = MagicConstant << (int)i; - ulong mask = i - 64; - return (long)(shift & mask) < 0; - } - - if (value < 32) - { - const int BitMask = (1 << (int)'\t') | (1 << (int)'\n') | (1 << (int)'\r'); - return ((1 << value) & BitMask) != 0; - } - - return value == 32; - } - - internal readonly struct Base64DecoderByte : IBase64Decoder - { - // Pre-computing this table using a custom string(s_characters) and GenerateDecodingMapAndVerify (found in tests) - public static ReadOnlySpan DecodingMap => - [ - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, //62 is placed at index 43 (for +), 63 at index 47 (for /) - 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, //52-61 are placed at index 48-57 (for 0-9) - -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, - 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, //0-25 are placed at index 65-90 (for A-Z) - -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, - 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, //26-51 are placed at index 97-122 (for a-z) - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // Bytes over 122 ('z') are invalid and cannot be decoded - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // Hence, padding the map with 255, which indicates invalid input - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - ]; - - public static ReadOnlySpan VbmiLookup0 => - [ - 0x80808080, 0x80808080, 0x80808080, 0x80808080, - 0x80808080, 0x80808080, 0x80808080, 0x80808080, - 0x80808080, 0x80808080, 0x3e808080, 0x3f808080, - 0x37363534, 0x3b3a3938, 0x80803d3c, 0x80808080 - ]; - - public static ReadOnlySpan VbmiLookup1 => - [ - 0x02010080, 0x06050403, 0x0a090807, 0x0e0d0c0b, - 0x1211100f, 0x16151413, 0x80191817, 0x80808080, - 0x1c1b1a80, 0x201f1e1d, 0x24232221, 0x28272625, - 0x2c2b2a29, 0x302f2e2d, 0x80333231, 0x80808080 - ]; - - public static ReadOnlySpan Avx2LutHigh => - [ - 0x10, 0x10, 0x01, 0x02, - 0x04, 0x08, 0x04, 0x08, - 0x10, 0x10, 0x10, 0x10, - 0x10, 0x10, 0x10, 0x10, - 0x10, 0x10, 0x01, 0x02, - 0x04, 0x08, 0x04, 0x08, - 0x10, 0x10, 0x10, 0x10, - 0x10, 0x10, 0x10, 0x10 - ]; - - public static ReadOnlySpan Avx2LutLow => - [ - 0x15, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x13, 0x1A, - 0x1B, 0x1B, 0x1B, 0x1A, - 0x15, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x11, 0x11, - 0x11, 0x11, 0x13, 0x1A, - 0x1B, 0x1B, 0x1B, 0x1A - ]; - - public static ReadOnlySpan Avx2LutShift => - [ - 0, 16, 19, 4, - -65, -65, -71, -71, - 0, 0, 0, 0, - 0, 0, 0, 0, - 0, 16, 19, 4, - -65, -65, -71, -71, - 0, 0, 0, 0, - 0, 0, 0, 0 - ]; - - public static byte MaskSlashOrUnderscore => (byte)'/'; - - public static ReadOnlySpan Vector128LutHigh => [0x02011010, 0x08040804, 0x10101010, 0x10101010]; - - public static ReadOnlySpan Vector128LutLow => [0x11111115, 0x11111111, 0x1A131111, 0x1A1B1B1B]; - - public static ReadOnlySpan Vector128LutShift => [0x04131000, 0xb9b9bfbf, 0x00000000, 0x00000000]; - - public static ReadOnlySpan AdvSimdLutOne3 => [0xFFFFFFFF, 0xFFFFFFFF, 0x3EFFFFFF, 0x3FFFFFFF]; - - public static uint AdvSimdLutTwo3Uint1 => 0x1B1AFFFF; - - public static int GetMaxDecodedLength(int utf8Length) => GetMaxDecodedFromUtf8Length(utf8Length); - - public static bool IsInvalidLength(int bufferLength) => bufferLength % 4 != 0; // only decode input if it is a multiple of 4 - - public static bool IsValidPadding(uint padChar) => padChar == EncodingPad; - - public static int SrcLength(bool _, int utf8Length) => utf8Length & ~0x3; // only decode input up to the closest multiple of 4. - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] - [CompExactlyDependsOn(typeof(Ssse3))] - public static bool TryDecode128Core( - Vector128 str, - Vector128 hiNibbles, - Vector128 maskSlashOrUnderscore, - Vector128 mask8F, - Vector128 lutLow, - Vector128 lutHigh, - Vector128 lutShift, - Vector128 _, - out Vector128 result) - { - Vector128 loNibbles = str & maskSlashOrUnderscore; - Vector128 hi = SimdShuffle(lutHigh, hiNibbles, mask8F); - Vector128 lo = SimdShuffle(lutLow, loNibbles, mask8F); - - // Check for invalid input: if any "and" values from lo and hi are not zero, - // fall back on bytewise code to do error checking and reporting: - if ((lo & hi) != Vector128.Zero) - { - result = default; - return false; - } - - Vector128 eq2F = Vector128.Equals(str, maskSlashOrUnderscore); - Vector128 shift = SimdShuffle(lutShift.AsByte(), (eq2F + hiNibbles), mask8F); - - // Now simply add the delta values to the input: - result = str + shift; - - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Avx2))] - public static bool TryDecode256Core( - Vector256 str, - Vector256 hiNibbles, - Vector256 maskSlashOrUnderscore, - Vector256 lutLow, - Vector256 lutHigh, - Vector256 lutShift, - Vector256 _, - out Vector256 result) - { - Vector256 loNibbles = Avx2.And(str, maskSlashOrUnderscore); - Vector256 hi = Avx2.Shuffle(lutHigh, hiNibbles); - Vector256 lo = Avx2.Shuffle(lutLow, loNibbles); - - if (!Avx.TestZ(lo, hi)) - { - result = default; - return false; - } - - Vector256 eq2F = Avx2.CompareEqual(str, maskSlashOrUnderscore); - Vector256 shift = Avx2.Shuffle(lutShift, Avx2.Add(eq2F, hiNibbles)); - - result = Avx2.Add(str, shift); - - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe int DecodeFourElements(byte* source, ref sbyte decodingMap) - { - // The 'source' span expected to have at least 4 elements, and the 'decodingMap' consists 256 sbytes - uint t0 = source[0]; - uint t1 = source[1]; - uint t2 = source[2]; - uint t3 = source[3]; - - int i0 = Unsafe.Add(ref decodingMap, t0); - int i1 = Unsafe.Add(ref decodingMap, t1); - int i2 = Unsafe.Add(ref decodingMap, t2); - int i3 = Unsafe.Add(ref decodingMap, t3); - - i0 <<= 18; - i1 <<= 12; - i2 <<= 6; - - i0 |= i3; - i1 |= i2; - - i0 |= i1; - return i0; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe int DecodeRemaining(byte* srcEnd, ref sbyte decodingMap, long remaining, out uint t2, out uint t3) - { - uint t0; - uint t1; - t2 = EncodingPad; - t3 = EncodingPad; - switch (remaining) - { - case 2: - t0 = srcEnd[-2]; - t1 = srcEnd[-1]; - break; - case 3: - t0 = srcEnd[-3]; - t1 = srcEnd[-2]; - t2 = srcEnd[-1]; - break; - case 4: - t0 = srcEnd[-4]; - t1 = srcEnd[-3]; - t2 = srcEnd[-2]; - t3 = srcEnd[-1]; - break; - default: - return -1; - } - - int i0 = Unsafe.Add(ref decodingMap, (IntPtr)t0); - int i1 = Unsafe.Add(ref decodingMap, (IntPtr)t1); - - i0 <<= 18; - i1 <<= 12; - - i0 |= i1; - return i0; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int IndexOfAnyExceptWhiteSpace(ReadOnlySpan span) - { - for (int i = 0; i < span.Length; i++) - { - if (!IsWhiteSpace(span[i])) - { - return i; - } - } - - return -1; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static OperationStatus DecodeWithWhiteSpaceBlockwiseWrapper(ReadOnlySpan utf8, - Span bytes, ref int bytesConsumed, ref int bytesWritten, bool isFinalBlock = true) - where TBase64Decoder : IBase64Decoder => - DecodeWithWhiteSpaceBlockwise(utf8, bytes, ref bytesConsumed, ref bytesWritten, isFinalBlock); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe bool TryLoadVector512(byte* src, byte* srcStart, int sourceLength, out Vector512 str) - { - AssertRead>(src, srcStart, sourceLength); - str = Vector512.Load(src).AsSByte(); - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Avx2))] - public static unsafe bool TryLoadAvxVector256(byte* src, byte* srcStart, int sourceLength, out Vector256 str) - { - AssertRead>(src, srcStart, sourceLength); - str = Avx.LoadVector256(src).AsSByte(); - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe bool TryLoadVector128(byte* src, byte* srcStart, int sourceLength, out Vector128 str) - { - AssertRead>(src, srcStart, sourceLength); - str = Vector128.LoadUnsafe(ref *src); - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] - public static unsafe bool TryLoadArmVector128x4(byte* src, byte* srcStart, int sourceLength, - out Vector128 str1, out Vector128 str2, out Vector128 str3, out Vector128 str4) - { - AssertRead>(src, srcStart, sourceLength); - (str1, str2, str3, str4) = AdvSimd.Arm64.Load4xVector128AndUnzip(src); - - return true; - } - } + Base64Helper.DecodeFromUtf8InPlace(default(Base64DecoderByte), buffer, out bytesWritten, ignoreWhiteSpace: true); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Encoder.cs b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Encoder.cs index dd0af2ee40a7dc..13e5de5868594b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Encoder.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Encoder.cs @@ -2,10 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; -using System.Runtime.Intrinsics; -using System.Runtime.Intrinsics.Arm; -using System.Runtime.Intrinsics.X86; +using static System.Buffers.Text.Base64Helper; namespace System.Buffers.Text { @@ -34,122 +31,8 @@ public static partial class Base64 /// - NeedMoreData - only if is , otherwise the output is padded if the input is not a multiple of 3 /// It does not return InvalidData since that is not possible for base64 encoding. /// - public static unsafe OperationStatus EncodeToUtf8(ReadOnlySpan bytes, Span utf8, out int bytesConsumed, out int bytesWritten, bool isFinalBlock = true) => - EncodeTo(bytes, utf8, out bytesConsumed, out bytesWritten, isFinalBlock); - - internal static unsafe OperationStatus EncodeTo(ReadOnlySpan source, - Span destination, out int bytesConsumed, out int bytesWritten, bool isFinalBlock = true) - where TBase64Encoder : IBase64Encoder - where T : unmanaged - { - if (source.IsEmpty) - { - bytesConsumed = 0; - bytesWritten = 0; - return OperationStatus.Done; - } - - fixed (byte* srcBytes = &MemoryMarshal.GetReference(source)) - fixed (T* destBytes = &MemoryMarshal.GetReference(destination)) - { - int srcLength = source.Length; - int destLength = destination.Length; - int maxSrcLength = TBase64Encoder.GetMaxSrcLength(srcLength, destLength); - - byte* src = srcBytes; - T* dest = destBytes; - byte* srcEnd = srcBytes + (uint)srcLength; - byte* srcMax = srcBytes + (uint)maxSrcLength; - - if (maxSrcLength >= 16) - { - byte* end = srcMax - 64; - if (Vector512.IsHardwareAccelerated && Avx512Vbmi.IsSupported && (end >= src)) - { - Avx512Encode(ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); - - if (src == srcEnd) - goto DoneExit; - } - - end = srcMax - 32; - if (Avx2.IsSupported && (end >= src)) - { - Avx2Encode(ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); - - if (src == srcEnd) - goto DoneExit; - } - - end = srcMax - 48; - if (AdvSimd.Arm64.IsSupported && (end >= src)) - { - AdvSimdEncode(ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); - - if (src == srcEnd) - goto DoneExit; - } - - end = srcMax - 16; - if ((Ssse3.IsSupported || AdvSimd.Arm64.IsSupported) && BitConverter.IsLittleEndian && (end >= src)) - { - Vector128Encode(ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); - - if (src == srcEnd) - goto DoneExit; - } - } - - ref byte encodingMap = ref MemoryMarshal.GetReference(TBase64Encoder.EncodingMap); - - srcMax -= 2; - while (src < srcMax) - { - TBase64Encoder.EncodeThreeAndWrite(src, dest, ref encodingMap); - src += 3; - dest += 4; - } - - if (srcMax + 2 != srcEnd) - goto DestinationTooSmallExit; - - if (!isFinalBlock) - { - if (src == srcEnd) - goto DoneExit; - - goto NeedMoreData; - } - - if (src + 1 == srcEnd) - { - TBase64Encoder.EncodeOneOptionallyPadTwo(src, dest, ref encodingMap); - src += 1; - dest += TBase64Encoder.IncrementPadTwo; - } - else if (src + 2 == srcEnd) - { - TBase64Encoder.EncodeTwoOptionallyPadOne(src, dest, ref encodingMap); - src += 2; - dest += TBase64Encoder.IncrementPadOne; - } - - DoneExit: - bytesConsumed = (int)(src - srcBytes); - bytesWritten = (int)(dest - destBytes); - return OperationStatus.Done; - - DestinationTooSmallExit: - bytesConsumed = (int)(src - srcBytes); - bytesWritten = (int)(dest - destBytes); - return OperationStatus.DestinationTooSmall; - - NeedMoreData: - bytesConsumed = (int)(src - srcBytes); - bytesWritten = (int)(dest - destBytes); - return OperationStatus.NeedMoreData; - } - } + public static OperationStatus EncodeToUtf8(ReadOnlySpan bytes, Span utf8, out int bytesConsumed, out int bytesWritten, bool isFinalBlock = true) => + EncodeTo(default(Base64EncoderByte), bytes, utf8, out bytesConsumed, out bytesWritten, isFinalBlock); /// /// Returns the maximum length (in bytes) of the result if you were to encode binary data within a byte span of size "length". @@ -180,660 +63,7 @@ public static int GetMaxEncodedToUtf8Length(int length) /// It does not return NeedMoreData since this method tramples the data in the buffer and hence can only be called once with all the data in the buffer. /// It does not return InvalidData since that is not possible for base 64 encoding. /// - public static unsafe OperationStatus EncodeToUtf8InPlace(Span buffer, int dataLength, out int bytesWritten) => - EncodeToUtf8InPlace(buffer, dataLength, out bytesWritten); - - internal static unsafe OperationStatus EncodeToUtf8InPlace(Span buffer, int dataLength, out int bytesWritten) - where TBase64Encoder : IBase64Encoder - { - if (buffer.IsEmpty) - { - bytesWritten = 0; - return OperationStatus.Done; - } - - fixed (byte* bufferBytes = &MemoryMarshal.GetReference(buffer)) - { - int encodedLength = TBase64Encoder.GetMaxEncodedLength(dataLength); - if (buffer.Length < encodedLength) - { - bytesWritten = 0; - return OperationStatus.DestinationTooSmall; - } - - int leftover = (int)((uint)dataLength % 3); // how many bytes after packs of 3 - - uint destinationIndex = TBase64Encoder.GetInPlaceDestinationLength(encodedLength, leftover); - uint sourceIndex = (uint)(dataLength - leftover); - ref byte encodingMap = ref MemoryMarshal.GetReference(TBase64Encoder.EncodingMap); - - // encode last pack to avoid conditional in the main loop - if (leftover != 0) - { - if (leftover == 1) - { - TBase64Encoder.EncodeOneOptionallyPadTwo(bufferBytes + sourceIndex, bufferBytes + destinationIndex, ref encodingMap); - } - else - { - TBase64Encoder.EncodeTwoOptionallyPadOne(bufferBytes + sourceIndex, bufferBytes + destinationIndex, ref encodingMap); - } - - destinationIndex -= 4; - } - - sourceIndex -= 3; - while ((int)sourceIndex >= 0) - { - uint result = Encode(bufferBytes + sourceIndex, ref encodingMap); - Unsafe.WriteUnaligned(bufferBytes + destinationIndex, result); - destinationIndex -= 4; - sourceIndex -= 3; - } - - bytesWritten = encodedLength; - return OperationStatus.Done; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Avx512BW))] - [CompExactlyDependsOn(typeof(Avx512Vbmi))] - private static unsafe void Avx512Encode(ref byte* srcBytes, ref T* destBytes, byte* srcEnd, int sourceLength, int destLength, byte* srcStart, T* destStart) - where TBase64Encoder : IBase64Encoder - where T : unmanaged - { - // Reference for VBMI implementation : https://github.com/WojciechMula/base64simd/tree/master/encode - // If we have AVX512 support, pick off 48 bytes at a time for as long as we can. - // But because we read 64 bytes at a time, ensure we have enough room to do a - // full 64-byte read without segfaulting. - - byte* src = srcBytes; - T* dest = destBytes; - - // The JIT won't hoist these "constants", so help it - Vector512 shuffleVecVbmi = Vector512.Create( - 0x01020001, 0x04050304, 0x07080607, 0x0a0b090a, - 0x0d0e0c0d, 0x10110f10, 0x13141213, 0x16171516, - 0x191a1819, 0x1c1d1b1c, 0x1f201e1f, 0x22232122, - 0x25262425, 0x28292728, 0x2b2c2a2b, 0x2e2f2d2e).AsSByte(); - Vector512 vbmiLookup = Vector512.Create(TBase64Encoder.EncodingMap).AsSByte(); - - Vector512 maskAC = Vector512.Create((uint)0x0fc0fc00).AsUInt16(); - Vector512 maskBB = Vector512.Create((uint)0x3f003f00); - Vector512 shiftAC = Vector512.Create((uint)0x0006000a).AsUInt16(); - Vector512 shiftBB = Vector512.Create((uint)0x00080004).AsUInt16(); - - AssertRead>(src, srcStart, sourceLength); - - // This algorithm requires AVX512VBMI support. - // Vbmi was first introduced in CannonLake and is available from IceLake on. - - // str = [...|PONM|LKJI|HGFE|DCBA] - Vector512 str = Vector512.Load(src).AsSByte(); - - while (true) - { - // Step 1 : Split 48 bytes into 64 bytes with each byte using 6-bits from input - // str = [...|KLJK|HIGH|EFDE|BCAB] - str = Avx512Vbmi.PermuteVar64x8(str, shuffleVecVbmi); - - // TO-DO- This can be achieved faster with multishift - // Consider the first 4 bytes - BCAB - // temp1 = [...|0000cccc|cc000000|aaaaaa00|00000000] - Vector512 temp1 = (str.AsUInt16() & maskAC); - - // temp2 = [...|00000000|00cccccc|00000000|00aaaaaa] - Vector512 temp2 = Avx512BW.ShiftRightLogicalVariable(temp1, shiftAC).AsUInt16(); - - // temp3 = [...|ccdddddd|00000000|aabbbbbb|cccc0000] - Vector512 temp3 = Avx512BW.ShiftLeftLogicalVariable(str.AsUInt16(), shiftBB).AsUInt16(); - - // str = [...|00dddddd|00cccccc|00bbbbbb|00aaaaaa] - str = Vector512.ConditionalSelect(maskBB, temp3.AsUInt32(), temp2.AsUInt32()).AsSByte(); - - // Step 2: Now we have the indices calculated. Next step is to use these indices to translate. - str = Avx512Vbmi.PermuteVar64x8(vbmiLookup, str); - - TBase64Encoder.StoreVector512ToDestination(dest, destStart, destLength, str.AsByte()); - - src += 48; - dest += 64; - - if (src > srcEnd) - break; - - AssertRead>(src, srcStart, sourceLength); - str = Vector512.Load(src).AsSByte(); - } - - srcBytes = src; - destBytes = dest; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Avx2))] - private static unsafe void Avx2Encode(ref byte* srcBytes, ref T* destBytes, byte* srcEnd, int sourceLength, int destLength, byte* srcStart, T* destStart) - where TBase64Encoder : IBase64Encoder - where T : unmanaged - { - // If we have AVX2 support, pick off 24 bytes at a time for as long as we can. - // But because we read 32 bytes at a time, ensure we have enough room to do a - // full 32-byte read without segfaulting. - - // translation from SSSE3 into AVX2 of procedure - // This one works with shifted (4 bytes) input in order to - // be able to work efficiently in the 2 128-bit lanes - - // srcBytes, bytes MSB to LSB: - // 0 0 0 0 x w v u t s r q p o n m - // l k j i h g f e d c b a 0 0 0 0 - - // The JIT won't hoist these "constants", so help it - Vector256 shuffleVec = Vector256.Create( - 5, 4, 6, 5, - 8, 7, 9, 8, - 11, 10, 12, 11, - 14, 13, 15, 14, - 1, 0, 2, 1, - 4, 3, 5, 4, - 7, 6, 8, 7, - 10, 9, 11, 10); - - Vector256 lut = Vector256.Create( - 65, 71, -4, -4, - -4, -4, -4, -4, - -4, -4, -4, -4, - TBase64Encoder.Avx2LutChar62, TBase64Encoder.Avx2LutChar63, 0, 0, - 65, 71, -4, -4, - -4, -4, -4, -4, - -4, -4, -4, -4, - TBase64Encoder.Avx2LutChar62, TBase64Encoder.Avx2LutChar63, 0, 0); - - Vector256 maskAC = Vector256.Create(0x0fc0fc00).AsSByte(); - Vector256 maskBB = Vector256.Create(0x003f03f0).AsSByte(); - Vector256 shiftAC = Vector256.Create(0x04000040).AsUInt16(); - Vector256 shiftBB = Vector256.Create(0x01000010).AsInt16(); - Vector256 const51 = Vector256.Create((byte)51); - Vector256 const25 = Vector256.Create((sbyte)25); - - byte* src = srcBytes; - T* dest = destBytes; - - // first load is done at c-0 not to get a segfault - AssertRead>(src, srcStart, sourceLength); - Vector256 str = Avx.LoadVector256(src).AsSByte(); - - // shift by 4 bytes, as required by Reshuffle - str = Avx2.PermuteVar8x32(str.AsInt32(), Vector256.Create( - 0, 0, 0, 0, - 0, 0, 0, 0, - 1, 0, 0, 0, - 2, 0, 0, 0, - 3, 0, 0, 0, - 4, 0, 0, 0, - 5, 0, 0, 0, - 6, 0, 0, 0).AsInt32()).AsSByte(); - - // Next loads are done at src-4, as required by Reshuffle, so shift it once - src -= 4; - - while (true) - { - // Reshuffle - str = Avx2.Shuffle(str, shuffleVec); - // str, bytes MSB to LSB: - // w x v w - // t u s t - // q r p q - // n o m n - // k l j k - // h i g h - // e f d e - // b c a b - - Vector256 t0 = Avx2.And(str, maskAC); - // bits, upper case are most significant bits, lower case are least significant bits. - // 0000wwww XX000000 VVVVVV00 00000000 - // 0000tttt UU000000 SSSSSS00 00000000 - // 0000qqqq RR000000 PPPPPP00 00000000 - // 0000nnnn OO000000 MMMMMM00 00000000 - // 0000kkkk LL000000 JJJJJJ00 00000000 - // 0000hhhh II000000 GGGGGG00 00000000 - // 0000eeee FF000000 DDDDDD00 00000000 - // 0000bbbb CC000000 AAAAAA00 00000000 - - Vector256 t2 = Avx2.And(str, maskBB); - // 00000000 00xxxxxx 000000vv WWWW0000 - // 00000000 00uuuuuu 000000ss TTTT0000 - // 00000000 00rrrrrr 000000pp QQQQ0000 - // 00000000 00oooooo 000000mm NNNN0000 - // 00000000 00llllll 000000jj KKKK0000 - // 00000000 00iiiiii 000000gg HHHH0000 - // 00000000 00ffffff 000000dd EEEE0000 - // 00000000 00cccccc 000000aa BBBB0000 - - Vector256 t1 = Avx2.MultiplyHigh(t0.AsUInt16(), shiftAC); - // 00000000 00wwwwXX 00000000 00VVVVVV - // 00000000 00ttttUU 00000000 00SSSSSS - // 00000000 00qqqqRR 00000000 00PPPPPP - // 00000000 00nnnnOO 00000000 00MMMMMM - // 00000000 00kkkkLL 00000000 00JJJJJJ - // 00000000 00hhhhII 00000000 00GGGGGG - // 00000000 00eeeeFF 00000000 00DDDDDD - // 00000000 00bbbbCC 00000000 00AAAAAA - - Vector256 t3 = Avx2.MultiplyLow(t2.AsInt16(), shiftBB); - // 00xxxxxx 00000000 00vvWWWW 00000000 - // 00uuuuuu 00000000 00ssTTTT 00000000 - // 00rrrrrr 00000000 00ppQQQQ 00000000 - // 00oooooo 00000000 00mmNNNN 00000000 - // 00llllll 00000000 00jjKKKK 00000000 - // 00iiiiii 00000000 00ggHHHH 00000000 - // 00ffffff 00000000 00ddEEEE 00000000 - // 00cccccc 00000000 00aaBBBB 00000000 - - str = Avx2.Or(t1.AsSByte(), t3.AsSByte()); - // 00xxxxxx 00wwwwXX 00vvWWWW 00VVVVVV - // 00uuuuuu 00ttttUU 00ssTTTT 00SSSSSS - // 00rrrrrr 00qqqqRR 00ppQQQQ 00PPPPPP - // 00oooooo 00nnnnOO 00mmNNNN 00MMMMMM - // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ - // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG - // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD - // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA - - // Translation - // LUT contains Absolute offset for all ranges: - // Translate values 0..63 to the Base64 alphabet. There are five sets: - // # From To Abs Index Characters - // 0 [0..25] [65..90] +65 0 ABCDEFGHIJKLMNOPQRSTUVWXYZ - // 1 [26..51] [97..122] +71 1 abcdefghijklmnopqrstuvwxyz - // 2 [52..61] [48..57] -4 [2..11] 0123456789 - // 3 [62] [43] -19 12 + - // 4 [63] [47] -16 13 / - - // Create LUT indices from input: - // the index for range #0 is right, others are 1 less than expected: - Vector256 indices = Avx2.SubtractSaturate(str.AsByte(), const51); - - // mask is 0xFF (-1) for range #[1..4] and 0x00 for range #0: - Vector256 mask = Avx2.CompareGreaterThan(str, const25); - - // subtract -1, so add 1 to indices for range #[1..4], All indices are now correct: - Vector256 tmp = Avx2.Subtract(indices.AsSByte(), mask); - - // Add offsets to input values: - str = Avx2.Add(str, Avx2.Shuffle(lut, tmp)); - - TBase64Encoder.StoreVector256ToDestination(dest, destStart, destLength, str.AsByte()); - - src += 24; - dest += 32; - - if (src > srcEnd) - break; - - // Load at src-4, as required by Reshuffle (already shifted by -4) - AssertRead>(src, srcStart, sourceLength); - str = Avx.LoadVector256(src).AsSByte(); - } - - srcBytes = src + 4; - destBytes = dest; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] - private static unsafe void AdvSimdEncode(ref byte* srcBytes, ref T* destBytes, byte* srcEnd, int sourceLength, int destLength, byte* srcStart, T* destStart) - where TBase64Encoder : IBase64Encoder - where T : unmanaged - { - // C# implementation of https://github.com/aklomp/base64/blob/3a5add8652076612a8407627a42c768736a4263f/lib/arch/neon64/enc_loop.c - Vector128 str1; - Vector128 str2; - Vector128 str3; - Vector128 res1; - Vector128 res2; - Vector128 res3; - Vector128 res4; - Vector128 tblEnc1 = Vector128.Create("ABCDEFGHIJKLMNOP"u8).AsByte(); - Vector128 tblEnc2 = Vector128.Create("QRSTUVWXYZabcdef"u8).AsByte(); - Vector128 tblEnc3 = Vector128.Create("ghijklmnopqrstuv"u8).AsByte(); - Vector128 tblEnc4 = Vector128.Create(TBase64Encoder.AdvSimdLut4).AsByte(); - byte* src = srcBytes; - T* dest = destBytes; - - // If we have Neon support, pick off 48 bytes at a time for as long as we can. - do - { - // Load 48 bytes and deinterleave: - AssertRead>(src, srcStart, sourceLength); - (str1, str2, str3) = AdvSimd.Arm64.Load3xVector128AndUnzip(src); - - // Divide bits of three input bytes over four output bytes: - res1 = AdvSimd.ShiftRightLogical(str1, 2); - res2 = AdvSimd.ShiftRightLogical(str2, 4); - res3 = AdvSimd.ShiftRightLogical(str3, 6); - res2 = AdvSimd.ShiftLeftAndInsert(res2, str1, 4); - res3 = AdvSimd.ShiftLeftAndInsert(res3, str2, 2); - - // Clear top two bits: - res2 &= AdvSimd.DuplicateToVector128((byte)0x3F); - res3 &= AdvSimd.DuplicateToVector128((byte)0x3F); - res4 = str3 & AdvSimd.DuplicateToVector128((byte)0x3F); - - // The bits have now been shifted to the right locations; - // translate their values 0..63 to the Base64 alphabet. - // Use a 64-byte table lookup: - res1 = AdvSimd.Arm64.VectorTableLookup((tblEnc1, tblEnc2, tblEnc3, tblEnc4), res1); - res2 = AdvSimd.Arm64.VectorTableLookup((tblEnc1, tblEnc2, tblEnc3, tblEnc4), res2); - res3 = AdvSimd.Arm64.VectorTableLookup((tblEnc1, tblEnc2, tblEnc3, tblEnc4), res3); - res4 = AdvSimd.Arm64.VectorTableLookup((tblEnc1, tblEnc2, tblEnc3, tblEnc4), res4); - - // Interleave and store result: - TBase64Encoder.StoreArmVector128x4ToDestination(dest, destStart, destLength, res1, res2, res3, res4); - - src += 48; - dest += 64; - } while (src <= srcEnd); - - srcBytes = src; - destBytes = dest; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Ssse3))] - [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] - private static unsafe void Vector128Encode(ref byte* srcBytes, ref T* destBytes, byte* srcEnd, int sourceLength, int destLength, byte* srcStart, T* destStart) - where TBase64Encoder : IBase64Encoder - where T : unmanaged - { - // If we have SSSE3 support, pick off 12 bytes at a time for as long as we can. - // But because we read 16 bytes at a time, ensure we have enough room to do a - // full 16-byte read without segfaulting. - - // srcBytes, bytes MSB to LSB: - // 0 0 0 0 l k j i h g f e d c b a - - // The JIT won't hoist these "constants", so help it - Vector128 shuffleVec = Vector128.Create(0x01020001, 0x04050304, 0x07080607, 0x0A0B090A).AsByte(); - Vector128 lut = Vector128.Create(0xFCFC4741, 0xFCFCFCFC, 0xFCFCFCFC, TBase64Encoder.Ssse3AdvSimdLutE3).AsByte(); - Vector128 maskAC = Vector128.Create(0x0fc0fc00).AsByte(); - Vector128 maskBB = Vector128.Create(0x003f03f0).AsByte(); - Vector128 shiftAC = Vector128.Create(0x04000040).AsUInt16(); - Vector128 shiftBB = Vector128.Create(0x01000010).AsInt16(); - Vector128 const51 = Vector128.Create((byte)51); - Vector128 const25 = Vector128.Create((sbyte)25); - Vector128 mask8F = Vector128.Create((byte)0x8F); - - byte* src = srcBytes; - T* dest = destBytes; - - //while (remaining >= 16) - do - { - AssertRead>(src, srcStart, sourceLength); - Vector128 str = Vector128.LoadUnsafe(ref *src); - - // Reshuffle - str = SimdShuffle(str, shuffleVec, mask8F); - // str, bytes MSB to LSB: - // k l j k - // h i g h - // e f d e - // b c a b - - Vector128 t0 = str & maskAC; - // bits, upper case are most significant bits, lower case are least significant bits - // 0000kkkk LL000000 JJJJJJ00 00000000 - // 0000hhhh II000000 GGGGGG00 00000000 - // 0000eeee FF000000 DDDDDD00 00000000 - // 0000bbbb CC000000 AAAAAA00 00000000 - - Vector128 t2 = str & maskBB; - // 00000000 00llllll 000000jj KKKK0000 - // 00000000 00iiiiii 000000gg HHHH0000 - // 00000000 00ffffff 000000dd EEEE0000 - // 00000000 00cccccc 000000aa BBBB0000 - - Vector128 t1; - if (Ssse3.IsSupported) - { - t1 = Sse2.MultiplyHigh(t0.AsUInt16(), shiftAC); - } - else - { - Vector128 odd = Vector128.ShiftRightLogical(AdvSimd.Arm64.UnzipOdd(t0.AsUInt16(), t0.AsUInt16()), 6); - Vector128 even = Vector128.ShiftRightLogical(AdvSimd.Arm64.UnzipEven(t0.AsUInt16(), t0.AsUInt16()), 10); - t1 = AdvSimd.Arm64.ZipLow(even, odd); - } - // 00000000 00kkkkLL 00000000 00JJJJJJ - // 00000000 00hhhhII 00000000 00GGGGGG - // 00000000 00eeeeFF 00000000 00DDDDDD - // 00000000 00bbbbCC 00000000 00AAAAAA - - Vector128 t3 = t2.AsInt16() * shiftBB; - // 00llllll 00000000 00jjKKKK 00000000 - // 00iiiiii 00000000 00ggHHHH 00000000 - // 00ffffff 00000000 00ddEEEE 00000000 - // 00cccccc 00000000 00aaBBBB 00000000 - - str = t1.AsByte() | t3.AsByte(); - // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ - // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG - // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD - // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA - - // Translation - // LUT contains Absolute offset for all ranges: - // Translate values 0..63 to the Base64 alphabet. There are five sets: - // # From To Abs Index Characters - // 0 [0..25] [65..90] +65 0 ABCDEFGHIJKLMNOPQRSTUVWXYZ - // 1 [26..51] [97..122] +71 1 abcdefghijklmnopqrstuvwxyz - // 2 [52..61] [48..57] -4 [2..11] 0123456789 - // 3 [62] [43] -19 12 + - // 4 [63] [47] -16 13 / - - // Create LUT indices from input: - // the index for range #0 is right, others are 1 less than expected: - Vector128 indices; - if (Ssse3.IsSupported) - { - indices = Sse2.SubtractSaturate(str.AsByte(), const51); - } - else - { - indices = AdvSimd.SubtractSaturate(str.AsByte(), const51); - } - - // mask is 0xFF (-1) for range #[1..4] and 0x00 for range #0: - Vector128 mask = Vector128.GreaterThan(str.AsSByte(), const25); - - // subtract -1, so add 1 to indices for range #[1..4], All indices are now correct: - Vector128 tmp = indices.AsSByte() - mask; - - // Add offsets to input values: - str += SimdShuffle(lut, tmp.AsByte(), mask8F); - - TBase64Encoder.StoreVector128ToDestination(dest, destStart, destLength, str); - - src += 12; - dest += 16; - } - while (src <= srcEnd); - - srcBytes = src; - destBytes = dest; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static unsafe uint Encode(byte* threeBytes, ref byte encodingMap) - { - uint t0 = threeBytes[0]; - uint t1 = threeBytes[1]; - uint t2 = threeBytes[2]; - - uint i = (t0 << 16) | (t1 << 8) | t2; - - uint i0 = Unsafe.Add(ref encodingMap, (IntPtr)(i >> 18)); - uint i1 = Unsafe.Add(ref encodingMap, (IntPtr)((i >> 12) & 0x3F)); - uint i2 = Unsafe.Add(ref encodingMap, (IntPtr)((i >> 6) & 0x3F)); - uint i3 = Unsafe.Add(ref encodingMap, (IntPtr)(i & 0x3F)); - - if (BitConverter.IsLittleEndian) - { - return i0 | (i1 << 8) | (i2 << 16) | (i3 << 24); - } - else - { - return (i0 << 24) | (i1 << 16) | (i2 << 8) | i3; - } - } - - internal const uint EncodingPad = '='; // '=', for padding - - internal const int MaximumEncodeLength = (int.MaxValue / 4) * 3; // 1610612733 - - internal readonly struct Base64EncoderByte : IBase64Encoder - { - public static ReadOnlySpan EncodingMap => "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"u8; - - public static sbyte Avx2LutChar62 => -19; // char '+' diff - - public static sbyte Avx2LutChar63 => -16; // char '/' diff - - public static ReadOnlySpan AdvSimdLut4 => "wxyz0123456789+/"u8; - - public static uint Ssse3AdvSimdLutE3 => 0x0000F0ED; - - public static int IncrementPadTwo => 4; - - public static int IncrementPadOne => 4; - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int GetMaxSrcLength(int srcLength, int destLength) => - srcLength <= MaximumEncodeLength && destLength >= GetMaxEncodedToUtf8Length(srcLength) ? - srcLength : (destLength >> 2) * 3; - - public static uint GetInPlaceDestinationLength(int encodedLength, int _) => (uint)(encodedLength - 4); - - public static int GetMaxEncodedLength(int srcLength) => GetMaxEncodedToUtf8Length(srcLength); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void EncodeOneOptionallyPadTwo(byte* oneByte, byte* dest, ref byte encodingMap) - { - uint t0 = oneByte[0]; - - uint i = t0 << 8; - - uint i0 = Unsafe.Add(ref encodingMap, (IntPtr)(i >> 10)); - uint i1 = Unsafe.Add(ref encodingMap, (IntPtr)((i >> 4) & 0x3F)); - - if (BitConverter.IsLittleEndian) - { - dest[0] = (byte)i0; - dest[1] = (byte)i1; - dest[2] = (byte)EncodingPad; - dest[3] = (byte)EncodingPad; - } - else - { - dest[3] = (byte)i0; - dest[2] = (byte)i1; - dest[1] = (byte)EncodingPad; - dest[0] = (byte)EncodingPad; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void EncodeTwoOptionallyPadOne(byte* twoBytes, byte* dest, ref byte encodingMap) - { - uint t0 = twoBytes[0]; - uint t1 = twoBytes[1]; - - uint i = (t0 << 16) | (t1 << 8); - - uint i0 = Unsafe.Add(ref encodingMap, (IntPtr)(i >> 18)); - uint i1 = Unsafe.Add(ref encodingMap, (IntPtr)((i >> 12) & 0x3F)); - uint i2 = Unsafe.Add(ref encodingMap, (IntPtr)((i >> 6) & 0x3F)); - - if (BitConverter.IsLittleEndian) - { - dest[0] = (byte)i0; - dest[1] = (byte)i1; - dest[2] = (byte)i2; - dest[3] = (byte)EncodingPad; - } - else - { - dest[3] = (byte)i0; - dest[2] = (byte)i1; - dest[1] = (byte)i2; - dest[0] = (byte)EncodingPad; - } - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void StoreVector512ToDestination(byte* dest, byte* destStart, int destLength, Vector512 str) - { - AssertWrite>(dest, destStart, destLength); - str.Store(dest); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Avx2))] - public static unsafe void StoreVector256ToDestination(byte* dest, byte* destStart, int destLength, Vector256 str) - { - AssertWrite>(dest, destStart, destLength); - Avx.Store(dest, str.AsByte()); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void StoreVector128ToDestination(byte* dest, byte* destStart, int destLength, Vector128 str) - { - AssertWrite>(dest, destStart, destLength); - str.Store(dest); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] - public static unsafe void StoreArmVector128x4ToDestination(byte* dest, byte* destStart, int destLength, - Vector128 res1, Vector128 res2, Vector128 res3, Vector128 res4) - { - AssertWrite>(dest, destStart, destLength); - AdvSimd.Arm64.StoreVectorAndZip(dest, (res1, res2, res3, res4)); - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void EncodeThreeAndWrite(byte* threeBytes, byte* destination, ref byte encodingMap) - { - uint t0 = threeBytes[0]; - uint t1 = threeBytes[1]; - uint t2 = threeBytes[2]; - - uint i = (t0 << 16) | (t1 << 8) | t2; - - byte i0 = Unsafe.Add(ref encodingMap, (IntPtr)(i >> 18)); - byte i1 = Unsafe.Add(ref encodingMap, (IntPtr)((i >> 12) & 0x3F)); - byte i2 = Unsafe.Add(ref encodingMap, (IntPtr)((i >> 6) & 0x3F)); - byte i3 = Unsafe.Add(ref encodingMap, (IntPtr)(i & 0x3F)); - - if (BitConverter.IsLittleEndian) - { - destination[0] = i0; - destination[1] = i1; - destination[2] = i2; - destination[3] = i3; - } - else - { - destination[3] = i0; - destination[2] = i1; - destination[1] = i2; - destination[0] = i3; - } - } - } + public static OperationStatus EncodeToUtf8InPlace(Span buffer, int dataLength, out int bytesWritten) => + Base64Helper.EncodeToUtf8InPlace(default(Base64EncoderByte), buffer, dataLength, out bytesWritten); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Helper/Base64DecoderHelper.cs b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Helper/Base64DecoderHelper.cs new file mode 100644 index 00000000000000..628be863e38c66 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Helper/Base64DecoderHelper.cs @@ -0,0 +1,1437 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +#if NET +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; +using System.Runtime.Intrinsics; +#endif + +namespace System.Buffers.Text +{ + internal static partial class Base64Helper + { + internal static unsafe OperationStatus DecodeFrom(TBase64Decoder decoder, ReadOnlySpan source, Span bytes, + out int bytesConsumed, out int bytesWritten, bool isFinalBlock, bool ignoreWhiteSpace) + where TBase64Decoder : IBase64Decoder + where T : unmanaged + { + if (source.IsEmpty) + { + bytesConsumed = 0; + bytesWritten = 0; + return OperationStatus.Done; + } + + fixed (T* srcBytes = &MemoryMarshal.GetReference(source)) + fixed (byte* destBytes = &MemoryMarshal.GetReference(bytes)) + { + int srcLength = decoder.SrcLength(isFinalBlock, source.Length); + int destLength = bytes.Length; + int maxSrcLength = srcLength; + int decodedLength = decoder.GetMaxDecodedLength(srcLength); + + // max. 2 padding chars + if (destLength < decodedLength - 2) + { + // For overflow see comment below + maxSrcLength = destLength / 3 * 4; + } + + T* src = srcBytes; + byte* dest = destBytes; + T* srcEnd = srcBytes + (uint)srcLength; + T* srcMax = srcBytes + (uint)maxSrcLength; + +#if NET + if (maxSrcLength >= 24) + { + T* end = srcMax - 88; + if (Vector512.IsHardwareAccelerated && Avx512Vbmi.IsSupported && (end >= src)) + { + Avx512Decode(decoder, ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); + + if (src == srcEnd) + { + goto DoneExit; + } + } + + end = srcMax - 45; + if (Avx2.IsSupported && (end >= src)) + { + Avx2Decode(decoder, ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); + + if (src == srcEnd) + { + goto DoneExit; + } + } + + end = srcMax - 66; + if (AdvSimd.Arm64.IsSupported && (end >= src)) + { + AdvSimdDecode(decoder, ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); + + if (src == srcEnd) + { + goto DoneExit; + } + } + + end = srcMax - 24; + if ((Ssse3.IsSupported || AdvSimd.Arm64.IsSupported) && BitConverter.IsLittleEndian && (end >= src)) + { + Vector128Decode(decoder, ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); + + if (src == srcEnd) + { + goto DoneExit; + } + } + } +#endif + + // Last bytes could have padding characters, so process them separately and treat them as valid only if isFinalBlock is true + // if isFinalBlock is false, padding characters are considered invalid + int skipLastChunk = isFinalBlock ? 4 : 0; + + if (destLength >= decodedLength) + { + maxSrcLength = srcLength - skipLastChunk; + } + else + { + // This should never overflow since destLength here is less than int.MaxValue / 4 * 3 (i.e. 1610612733) + // Therefore, (destLength / 3) * 4 will always be less than 2147483641 + Debug.Assert(destLength < (int.MaxValue / 4 * 3)); +#if NET + (maxSrcLength, int remainder) = int.DivRem(destLength, 3); + maxSrcLength *= 4; +#else + maxSrcLength = (destLength / 3) * 4; + int remainder = (int)((uint)destLength % 3); +#endif + if (isFinalBlock && remainder > 0) + { + srcLength &= ~0x3; // In case of Base64UrlDecoder source can be not a multiple of 4, round down to multiple of 4 + } + } + + ref sbyte decodingMap = ref MemoryMarshal.GetReference(decoder.DecodingMap); + srcMax = srcBytes + maxSrcLength; + + while (src < srcMax) + { + int result = decoder.DecodeFourElements(src, ref decodingMap); + + if (result < 0) + { + goto InvalidDataExit; + } + + WriteThreeLowOrderBytes(dest, result); + src += 4; + dest += 3; + } + + if (maxSrcLength != srcLength - skipLastChunk) + { + goto DestinationTooSmallExit; + } + + if (src == srcEnd) + { + if (isFinalBlock) + { + goto InvalidDataExit; + } + + if (src == srcBytes + source.Length) + { + goto DoneExit; + } + + goto NeedMoreDataExit; + } + + // if isFinalBlock is false, we will never reach this point + // Handle remaining bytes, for Base64 its always 4 bytes, for Base64Url up to 8 bytes left. + // If more than 4 bytes remained it will end up in DestinationTooSmallExit or InvalidDataExit (might succeed after whitespace removed) + long remaining = srcEnd - src; + Debug.Assert(typeof(TBase64Decoder) == typeof(Base64DecoderByte) ? remaining == 4 : remaining < 8); + int i0 = decoder.DecodeRemaining(srcEnd, ref decodingMap, remaining, out uint t2, out uint t3); + + byte* destMax = destBytes + (uint)destLength; + + if (!decoder.IsValidPadding(t3)) + { + int i2 = Unsafe.Add(ref decodingMap, (IntPtr)t2); + int i3 = Unsafe.Add(ref decodingMap, (IntPtr)t3); + + i2 <<= 6; + + i0 |= i3; + i0 |= i2; + + if (i0 < 0) + { + goto InvalidDataExit; + } + if (dest + 3 > destMax) + { + goto DestinationTooSmallExit; + } + + WriteThreeLowOrderBytes(dest, i0); + dest += 3; + src += 4; + } + else if (!decoder.IsValidPadding(t2)) + { + int i2 = Unsafe.Add(ref decodingMap, (IntPtr)t2); + + i2 <<= 6; + + i0 |= i2; + + if (i0 < 0) + { + goto InvalidDataExit; + } + if (dest + 2 > destMax) + { + goto DestinationTooSmallExit; + } + + dest[0] = (byte)(i0 >> 16); + dest[1] = (byte)(i0 >> 8); + dest += 2; + src += remaining; + } + else + { + if (i0 < 0) + { + goto InvalidDataExit; + } + if (dest + 1 > destMax) + { + goto DestinationTooSmallExit; + } + + dest[0] = (byte)(i0 >> 16); + dest += 1; + src += remaining; + } + + if (srcLength != source.Length) + { + goto InvalidDataExit; + } + + DoneExit: + bytesConsumed = (int)(src - srcBytes); + bytesWritten = (int)(dest - destBytes); + return OperationStatus.Done; + + DestinationTooSmallExit: + if (srcLength != source.Length && isFinalBlock) + { + goto InvalidDataExit; // if input is not a multiple of 4, and there is no more data, return invalid data instead + } + + bytesConsumed = (int)(src - srcBytes); + bytesWritten = (int)(dest - destBytes); + return OperationStatus.DestinationTooSmall; + + NeedMoreDataExit: + bytesConsumed = (int)(src - srcBytes); + bytesWritten = (int)(dest - destBytes); + return OperationStatus.NeedMoreData; + + InvalidDataExit: + bytesConsumed = (int)(src - srcBytes); + bytesWritten = (int)(dest - destBytes); + return ignoreWhiteSpace ? + InvalidDataFallback(decoder, source, bytes, ref bytesConsumed, ref bytesWritten, isFinalBlock) : + OperationStatus.InvalidData; + } + + static OperationStatus InvalidDataFallback(TBase64Decoder decoder, ReadOnlySpan source, Span bytes, ref int bytesConsumed, ref int bytesWritten, bool isFinalBlock) + { + source = source.Slice(bytesConsumed); + bytes = bytes.Slice(bytesWritten); + + OperationStatus status; + do + { + int localConsumed = decoder.IndexOfAnyExceptWhiteSpace(source); + if (localConsumed < 0) + { + // The remainder of the input is all whitespace. Mark it all as having been consumed, + // and mark the operation as being done. + bytesConsumed += source.Length; + status = OperationStatus.Done; + break; + } + + if (localConsumed == 0) + { + // Non-whitespace was found at the beginning of the input. Since it wasn't consumed + // by the previous call to DecodeFromUtf8, it must be part of a Base64 sequence + // that was interrupted by whitespace or something else considered invalid. + // Fall back to block-wise decoding. This is very slow, but it's also very non-standard + // formatting of the input; whitespace is typically only found between blocks, such as + // when Convert.ToBase64String inserts a line break every 76 output characters. + return decoder.DecodeWithWhiteSpaceBlockwiseWrapper(decoder, source, bytes, ref bytesConsumed, ref bytesWritten, isFinalBlock); + } + + // Skip over the starting whitespace and continue. + bytesConsumed += localConsumed; + source = source.Slice(localConsumed); + + // Try again after consumed whitespace + status = DecodeFrom(decoder, source, bytes, out localConsumed, out int localWritten, isFinalBlock, ignoreWhiteSpace: false); + bytesConsumed += localConsumed; + bytesWritten += localWritten; + if (status is not OperationStatus.InvalidData) + { + break; + } + + source = source.Slice(localConsumed); + bytes = bytes.Slice(localWritten); + } + while (!source.IsEmpty); + + return status; + } + } + + internal static unsafe OperationStatus DecodeFromUtf8InPlace(TBase64Decoder decoder, Span buffer, out int bytesWritten, bool ignoreWhiteSpace) + where TBase64Decoder : IBase64Decoder + { + if (buffer.IsEmpty) + { + bytesWritten = 0; + return OperationStatus.Done; + } + + fixed (byte* bufferBytes = &MemoryMarshal.GetReference(buffer)) + { + uint bufferLength = (uint)buffer.Length; + uint sourceIndex = 0; + uint destIndex = 0; + + if (decoder.IsInvalidLength(buffer.Length)) + { + goto InvalidExit; + } + + ref sbyte decodingMap = ref MemoryMarshal.GetReference(decoder.DecodingMap); + + if (bufferLength > 4) + { + while (sourceIndex < bufferLength - 4) + { + int result = decoder.DecodeFourElements(bufferBytes + sourceIndex, ref decodingMap); + if (result < 0) + { + goto InvalidExit; + } + + WriteThreeLowOrderBytes(bufferBytes + destIndex, result); + destIndex += 3; + sourceIndex += 4; + } + } + + uint t0; + uint t1; + uint t2; + uint t3; + + switch (bufferLength - sourceIndex) + { + case 2: + t0 = bufferBytes[bufferLength - 2]; + t1 = bufferBytes[bufferLength - 1]; + t2 = EncodingPad; + t3 = EncodingPad; + break; + case 3: + t0 = bufferBytes[bufferLength - 3]; + t1 = bufferBytes[bufferLength - 2]; + t2 = bufferBytes[bufferLength - 1]; + t3 = EncodingPad; + break; + case 4: + t0 = bufferBytes[bufferLength - 4]; + t1 = bufferBytes[bufferLength - 3]; + t2 = bufferBytes[bufferLength - 2]; + t3 = bufferBytes[bufferLength - 1]; + break; + default: + goto InvalidExit; + } + + int i0 = Unsafe.Add(ref decodingMap, (int)t0); + int i1 = Unsafe.Add(ref decodingMap, (int)t1); + + i0 <<= 18; + i1 <<= 12; + + i0 |= i1; + + if (!decoder.IsValidPadding(t3)) + { + int i2 = Unsafe.Add(ref decodingMap, (int)t2); + int i3 = Unsafe.Add(ref decodingMap, (int)t3); + + i2 <<= 6; + + i0 |= i3; + i0 |= i2; + + if (i0 < 0) + { + goto InvalidExit; + } + + WriteThreeLowOrderBytes(bufferBytes + destIndex, i0); + destIndex += 3; + } + else if (!decoder.IsValidPadding(t2)) + { + int i2 = Unsafe.Add(ref decodingMap, (int)t2); + + i2 <<= 6; + + i0 |= i2; + + if (i0 < 0) + { + goto InvalidExit; + } + + bufferBytes[destIndex] = (byte)(i0 >> 16); + bufferBytes[destIndex + 1] = (byte)(i0 >> 8); + destIndex += 2; + } + else + { + if (i0 < 0) + { + goto InvalidExit; + } + + bufferBytes[destIndex] = (byte)(i0 >> 16); + destIndex += 1; + } + + bytesWritten = (int)destIndex; + return OperationStatus.Done; + + InvalidExit: + bytesWritten = (int)destIndex; + return ignoreWhiteSpace ? + DecodeWithWhiteSpaceFromUtf8InPlace(decoder, buffer, ref bytesWritten, sourceIndex) : // The input may have whitespace, attempt to decode while ignoring whitespace. + OperationStatus.InvalidData; + } + } + + internal static OperationStatus DecodeWithWhiteSpaceBlockwise(TBase64Decoder decoder, ReadOnlySpan source, Span bytes, ref int bytesConsumed, ref int bytesWritten, bool isFinalBlock = true) + where TBase64Decoder : IBase64Decoder + { + const int BlockSize = 4; + Span buffer = stackalloc byte[BlockSize]; + OperationStatus status = OperationStatus.Done; + + while (!source.IsEmpty) + { + int encodedIdx = 0; + int bufferIdx = 0; + int skipped = 0; + + for (; encodedIdx < source.Length && (uint)bufferIdx < (uint)buffer.Length; ++encodedIdx) + { + if (IsWhiteSpace(source[encodedIdx])) + { + skipped++; + } + else + { + buffer[bufferIdx] = source[encodedIdx]; + bufferIdx++; + } + } + + source = source.Slice(encodedIdx); + bytesConsumed += skipped; + + if (bufferIdx == 0) + { + continue; + } + + bool hasAnotherBlock; + + if (typeof(TBase64Decoder) == typeof(Base64DecoderByte)) + { + hasAnotherBlock = source.Length >= BlockSize; + } + else + { + hasAnotherBlock = source.Length > 1; + } + + bool localIsFinalBlock = !hasAnotherBlock; + + // If this block contains padding and there's another block, then only whitespace may follow for being valid. + if (hasAnotherBlock) + { + int paddingCount = GetPaddingCount(decoder, ref buffer[BlockSize - 1]); + if (paddingCount > 0) + { + hasAnotherBlock = false; + localIsFinalBlock = true; + } + } + + if (localIsFinalBlock && !isFinalBlock) + { + localIsFinalBlock = false; + } + + status = DecodeFrom(decoder, buffer.Slice(0, bufferIdx), bytes, out int localConsumed, out int localWritten, localIsFinalBlock, ignoreWhiteSpace: false); + bytesConsumed += localConsumed; + bytesWritten += localWritten; + + if (status != OperationStatus.Done) + { + return status; + } + + // The remaining data must all be whitespace in order to be valid. + if (!hasAnotherBlock) + { + for (int i = 0; i < source.Length; ++i) + { + if (!IsWhiteSpace(source[i])) + { + // Revert previous dest increment, since an invalid state followed. + bytesConsumed -= localConsumed; + bytesWritten -= localWritten; + + return OperationStatus.InvalidData; + } + + bytesConsumed++; + } + + break; + } + + bytes = bytes.Slice(localWritten); + } + + return status; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static int GetPaddingCount(TBase64Decoder decoder, ref byte ptrToLastElement) + where TBase64Decoder : IBase64Decoder + { + int padding = 0; + + if (decoder.IsValidPadding(ptrToLastElement)) + { + padding++; + } + + if (decoder.IsValidPadding(Unsafe.Subtract(ref ptrToLastElement, 1))) + { + padding++; + } + + return padding; + } + + private static OperationStatus DecodeWithWhiteSpaceFromUtf8InPlace(TBase64Decoder decoder, Span source, ref int destIndex, uint sourceIndex) + where TBase64Decoder : IBase64Decoder + { + int BlockSize = Math.Min(source.Length - (int)sourceIndex, 4); + Span buffer = stackalloc byte[BlockSize]; + + OperationStatus status = OperationStatus.Done; + int localDestIndex = destIndex; + bool hasPaddingBeenProcessed = false; + int localBytesWritten = 0; + + while (sourceIndex < (uint)source.Length) + { + int bufferIdx = 0; + + while (bufferIdx < BlockSize && sourceIndex < (uint)source.Length) + { + if (!IsWhiteSpace(source[(int)sourceIndex])) + { + buffer[bufferIdx] = source[(int)sourceIndex]; + bufferIdx++; + } + + sourceIndex++; + } + + if (bufferIdx == 0) + { + continue; + } + + if (bufferIdx != 4) + { + // Base64 require 4 bytes, for Base64Url it can be less than 4 bytes but not 1 byte. + if (decoder is Base64DecoderByte || bufferIdx == 1) + { + status = OperationStatus.InvalidData; + break; + } + else // For Base64Url fill empty slots in last block with padding + { + while (bufferIdx < BlockSize) // Can happen only for last block + { + Debug.Assert(source.Length == sourceIndex); + buffer[bufferIdx++] = (byte)EncodingPad; + } + } + } + + if (hasPaddingBeenProcessed) + { + // Padding has already been processed, a new valid block cannot be processed. + // Revert previous dest increment, since an invalid state followed. + localDestIndex -= localBytesWritten; + status = OperationStatus.InvalidData; + break; + } + + status = DecodeFromUtf8InPlace(decoder, buffer, out localBytesWritten, ignoreWhiteSpace: false); + localDestIndex += localBytesWritten; + hasPaddingBeenProcessed = localBytesWritten < 3; + + if (status != OperationStatus.Done) + { + break; + } + + // Write result to source span in place. + for (int i = 0; i < localBytesWritten; i++) + { + source[localDestIndex - localBytesWritten + i] = buffer[i]; + } + } + + destIndex = localDestIndex; + return status; + } + +#if NET + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx512BW))] + [CompExactlyDependsOn(typeof(Avx512Vbmi))] + private static unsafe void Avx512Decode(TBase64Decoder decoder, ref T* srcBytes, ref byte* destBytes, T* srcEnd, int sourceLength, int destLength, T* srcStart, byte* destStart) + where TBase64Decoder : IBase64Decoder + where T : unmanaged + { + // Reference for VBMI implementation : https://github.com/WojciechMula/base64simd/tree/master/decode + // If we have AVX512 support, pick off 64 bytes at a time for as long as we can, + // but make sure that we quit before seeing any == markers at the end of the + // string. Also, because we write 16 zeroes at the end of the output, ensure + // that there are at least 22 valid bytes of input data remaining to close the + // gap. 64 + 2 + 22 = 88 bytes. + T* src = srcBytes; + byte* dest = destBytes; + + // The JIT won't hoist these "constants", so help it + Vector512 vbmiLookup0 = Vector512.Create(decoder.VbmiLookup0).AsSByte(); + Vector512 vbmiLookup1 = Vector512.Create(decoder.VbmiLookup1).AsSByte(); + Vector512 vbmiPackedLanesControl = Vector512.Create( + 0x06000102, 0x090a0405, 0x0c0d0e08, 0x16101112, + 0x191a1415, 0x1c1d1e18, 0x26202122, 0x292a2425, + 0x2c2d2e28, 0x36303132, 0x393a3435, 0x3c3d3e38, + 0x00000000, 0x00000000, 0x00000000, 0x00000000).AsByte(); + + Vector512 mergeConstant0 = Vector512.Create(0x01400140).AsSByte(); + Vector512 mergeConstant1 = Vector512.Create(0x00011000).AsInt16(); + + // This algorithm requires AVX512VBMI support. + // Vbmi was first introduced in CannonLake and is available from IceLake on. + do + { + if (!decoder.TryLoadVector512(src, srcStart, sourceLength, out Vector512 str)) + { + break; + } + + // Step 1: Translate encoded Base64 input to their original indices + // This step also checks for invalid inputs and exits. + // After this, we have indices which are verified to have upper 2 bits set to 0 in each byte. + // origIndex = [...|00dddddd|00cccccc|00bbbbbb|00aaaaaa] + Vector512 origIndex = Avx512Vbmi.PermuteVar64x8x2(vbmiLookup0, str, vbmiLookup1); + Vector512 errorVec = (origIndex.AsInt32() | str.AsInt32()).AsSByte(); + if (errorVec.ExtractMostSignificantBits() != 0) + { + break; + } + + // Step 2: Now we need to reshuffle bits to remove the 0 bits. + // multiAdd1: [...|0000cccc|ccdddddd|0000aaaa|aabbbbbb] + Vector512 multiAdd1 = Avx512BW.MultiplyAddAdjacent(origIndex.AsByte(), mergeConstant0); + // multiAdd1: [...|00000000|aaaaaabb|bbbbcccc|ccdddddd] + Vector512 multiAdd2 = Avx512BW.MultiplyAddAdjacent(multiAdd1, mergeConstant1); + + // Step 3: Pack 48 bytes + str = Avx512Vbmi.PermuteVar64x8(multiAdd2.AsByte(), vbmiPackedLanesControl).AsSByte(); + + AssertWrite>(dest, destStart, destLength); + str.Store((sbyte*)dest); + src += 64; + dest += 48; + } + while (src <= srcEnd); + + srcBytes = src; + destBytes = dest; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx2))] + private static unsafe void Avx2Decode(TBase64Decoder decoder, ref T* srcBytes, ref byte* destBytes, T* srcEnd, int sourceLength, int destLength, T* srcStart, byte* destStart) + where TBase64Decoder : IBase64Decoder + where T : unmanaged + { + // If we have AVX2 support, pick off 32 bytes at a time for as long as we can, + // but make sure that we quit before seeing any == markers at the end of the + // string. Also, because we write 8 zeroes at the end of the output, ensure + // that there are at least 11 valid bytes of input data remaining to close the + // gap. 32 + 2 + 11 = 45 bytes. + + // See SSSE3-version below for an explanation of how the code works. + + // The JIT won't hoist these "constants", so help it + Vector256 lutHi = Vector256.Create(decoder.Avx2LutHigh); + + Vector256 lutLo = Vector256.Create(decoder.Avx2LutLow); + + Vector256 lutShift = Vector256.Create(decoder.Avx2LutShift); + + Vector256 packBytesInLaneMask = Vector256.Create( + 2, 1, 0, 6, + 5, 4, 10, 9, + 8, 14, 13, 12, + -1, -1, -1, -1, + 2, 1, 0, 6, + 5, 4, 10, 9, + 8, 14, 13, 12, + -1, -1, -1, -1); + + Vector256 packLanesControl = Vector256.Create( + 0, 0, 0, 0, + 1, 0, 0, 0, + 2, 0, 0, 0, + 4, 0, 0, 0, + 5, 0, 0, 0, + 6, 0, 0, 0, + -1, -1, -1, -1, + -1, -1, -1, -1).AsInt32(); + + Vector256 maskSlashOrUnderscore = Vector256.Create((sbyte)decoder.MaskSlashOrUnderscore); + Vector256 shiftForUnderscore = Vector256.Create((sbyte)33); + Vector256 mergeConstant0 = Vector256.Create(0x01400140).AsSByte(); + Vector256 mergeConstant1 = Vector256.Create(0x00011000).AsInt16(); + + T* src = srcBytes; + byte* dest = destBytes; + + //while (remaining >= 45) + do + { + if (!decoder.TryLoadAvxVector256(src, srcStart, sourceLength, out Vector256 str)) + { + break; + } + + Vector256 hiNibbles = Avx2.And(Avx2.ShiftRightLogical(str.AsInt32(), 4).AsSByte(), maskSlashOrUnderscore); + + if (!decoder.TryDecode256Core(str, hiNibbles, maskSlashOrUnderscore, lutLo, lutHi, lutShift, shiftForUnderscore, out str)) + { + break; + } + + // in, lower lane, bits, upper case are most significant bits, lower case are least significant bits: + // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ + // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG + // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD + // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA + + Vector256 merge_ab_and_bc = Avx2.MultiplyAddAdjacent(str.AsByte(), mergeConstant0); + // 0000kkkk LLllllll 0000JJJJ JJjjKKKK + // 0000hhhh IIiiiiii 0000GGGG GGggHHHH + // 0000eeee FFffffff 0000DDDD DDddEEEE + // 0000bbbb CCcccccc 0000AAAA AAaaBBBB + + Vector256 output = Avx2.MultiplyAddAdjacent(merge_ab_and_bc, mergeConstant1); + // 00000000 JJJJJJjj KKKKkkkk LLllllll + // 00000000 GGGGGGgg HHHHhhhh IIiiiiii + // 00000000 DDDDDDdd EEEEeeee FFffffff + // 00000000 AAAAAAaa BBBBbbbb CCcccccc + + // Pack bytes together in each lane: + output = Avx2.Shuffle(output.AsSByte(), packBytesInLaneMask).AsInt32(); + // 00000000 00000000 00000000 00000000 + // LLllllll KKKKkkkk JJJJJJjj IIiiiiii + // HHHHhhhh GGGGGGgg FFffffff EEEEeeee + // DDDDDDdd CCcccccc BBBBbbbb AAAAAAaa + + // Pack lanes + str = Avx2.PermuteVar8x32(output, packLanesControl).AsSByte(); + + AssertWrite>(dest, destStart, destLength); + Avx.Store(dest, str.AsByte()); + + src += 32; + dest += 24; + } + while (src <= srcEnd); + + srcBytes = src; + destBytes = dest; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + internal static Vector128 SimdShuffle(Vector128 left, Vector128 right, Vector128 mask8F) + { + Debug.Assert((Ssse3.IsSupported || AdvSimd.Arm64.IsSupported) && BitConverter.IsLittleEndian); + + if (AdvSimd.Arm64.IsSupported) + { + right &= mask8F; + } + + return Vector128.ShuffleUnsafe(left, right); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + private static unsafe void AdvSimdDecode(TBase64Decoder decoder, ref T* srcBytes, ref byte* destBytes, T* srcEnd, int sourceLength, int destLength, T* srcStart, byte* destStart) + where TBase64Decoder : IBase64Decoder + where T : unmanaged + { + // C# implementation of https://github.com/aklomp/base64/blob/3a5add8652076612a8407627a42c768736a4263f/lib/arch/neon64/dec_loop.c + // If we have AdvSimd support, pick off 64 bytes at a time for as long as we can, + // but make sure that we quit before seeing any == markers at the end of the + // string. 64 + 2 = 66 bytes. + + // In the decoding process, we want to map each byte, representing a Base64 value, to its 6-bit (0-63) representation. + // It uses the following mapping. Values outside the following groups are invalid and, we abort decoding when encounter one. + // + // # From To Char + // 1 [43] [62] + + // 2 [47] [63] / + // 3 [48..57] [52..61] 0..9 + // 4 [65..90] [0..25] A..Z + // 5 [97..122] [26..51] a..z + // + // To map an input value to its Base64 representation, we use look-up tables 'decLutOne' and 'decLutTwo'. + // 'decLutOne' helps to map groups 1, 2 and 3 while 'decLutTwo' maps groups 4 and 5 in the above list. + // After mapping, each value falls between 0-63. Consequently, the last six bits of each byte now hold a valid value. + // We then compress four such bytes (with valid 4 * 6 = 24 bits) to three UTF8 bytes (3 * 8 = 24 bits). + // For faster decoding, we use SIMD operations that allow the processing of multiple bytes together. + // However, the compress operation on adjacent values of a vector could be slower. Thus, we de-interleave while reading + // the input bytes that store adjacent bytes in separate vectors. This later simplifies the compress step with the help + // of logical operations. This requires interleaving while storing the decoded result. + + // Values in 'decLutOne' maps input values from 0 to 63. + // 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 + // 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255 + // 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 62, 255, 255, 255, 63 + // 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 255, 255, 255, 255, 255, 255 + var decLutOne = (Vector128.AllBitsSet, + Vector128.AllBitsSet, + Vector128.Create(decoder.AdvSimdLutOne3).AsByte(), + Vector128.Create(0x37363534, 0x3B3A3938, 0xFFFF3D3C, 0xFFFFFFFF).AsByte()); + + // Values in 'decLutTwo' maps input values from 63 to 127. + // 0, 255, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13 + // 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 255, 255, 255, 255 + // 255, 255, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39 + // 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 255, 255, 255, 255 + var decLutTwo = (Vector128.Create(0x0100FF00, 0x05040302, 0x09080706, 0x0D0C0B0A).AsByte(), + Vector128.Create(0x11100F0E, 0x15141312, 0x19181716, 0xFFFFFFFF).AsByte(), + Vector128.Create(decoder.AdvSimdLutTwo3Uint1, 0x1F1E1D1C, 0x23222120, 0x27262524).AsByte(), + Vector128.Create(0x2B2A2928, 0x2F2E2D2C, 0x33323130, 0xFFFFFFFF).AsByte()); + + T* src = srcBytes; + byte* dest = destBytes; + Vector128 offset = Vector128.Create(63); + + do + { + // Step 1: Load 64 bytes and de-interleave. + if (!decoder.TryLoadArmVector128x4(src, srcStart, sourceLength, + out Vector128 str1, out Vector128 str2, out Vector128 str3, out Vector128 str4)) + { + break; + } + + // Step 2: Map each valid input to its Base64 value. + // We use two look-ups to compute partial results and combine them later. + + // Step 2.1: Detect valid Base64 values from the first three groups. Maps input as, + // 0 to 63 (Invalid) => 255 + // 0 to 63 (Valid) => Their Base64 equivalent + // 64 to 255 => 0 + + // Each input value acts as an index in the look-up table 'decLutOne'. + // e.g., for group 1: index 43 maps to 62 (Base64 '+'). + // Group 4 and 5 values are out-of-range (>64), so they are mapped to zero. + // Other valid indices but invalid values are mapped to 255. + Vector128 decOne1 = AdvSimd.Arm64.VectorTableLookup(decLutOne, str1); + Vector128 decOne2 = AdvSimd.Arm64.VectorTableLookup(decLutOne, str2); + Vector128 decOne3 = AdvSimd.Arm64.VectorTableLookup(decLutOne, str3); + Vector128 decOne4 = AdvSimd.Arm64.VectorTableLookup(decLutOne, str4); + + // Step 2.2: Detect valid Base64 values from groups 4 and 5. Maps input as, + // 0 to 63 => 0 + // 64 to 122 (Valid) => Their Base64 equivalent + // 64 to 122 (Invalid) => 255 + // 123 to 255 => Remains unchanged + + // Subtract/offset each input value by 63 so that it can be used as a valid offset. + // Subtract saturate makes values from the first three groups set to zero that are + // then mapped to zero in the subsequent look-up. + Vector128 decTwo1 = AdvSimd.SubtractSaturate(str1, offset); + Vector128 decTwo2 = AdvSimd.SubtractSaturate(str2, offset); + Vector128 decTwo3 = AdvSimd.SubtractSaturate(str3, offset); + Vector128 decTwo4 = AdvSimd.SubtractSaturate(str4, offset); + + // We use VTBX to map values where out-of-range indices are unchanged. + decTwo1 = AdvSimd.Arm64.VectorTableLookupExtension(decTwo1, decLutTwo, decTwo1); + decTwo2 = AdvSimd.Arm64.VectorTableLookupExtension(decTwo2, decLutTwo, decTwo2); + decTwo3 = AdvSimd.Arm64.VectorTableLookupExtension(decTwo3, decLutTwo, decTwo3); + decTwo4 = AdvSimd.Arm64.VectorTableLookupExtension(decTwo4, decLutTwo, decTwo4); + + // Step 3: Combine the partial result. + // Each look-up above maps valid values to their Base64 equivalent or zero. + // Thus the intermediate results 'decOne' and 'decTwo' could be OR-ed to get final values. + str1 = (decOne1 | decTwo1); + str2 = (decOne2 | decTwo2); + str3 = (decOne3 | decTwo3); + str4 = (decOne4 | decTwo4); + + // Step 4: Detect an invalid input value. + // Invalid values < 122 are set to 255 while the ones above 122 are unchanged. + // Check for invalid input, any value larger than 63. + Vector128 classified = (Vector128.GreaterThan(str1, offset) + | Vector128.GreaterThan(str2, offset) + | Vector128.GreaterThan(str3, offset) + | Vector128.GreaterThan(str4, offset)); + + // Check that all bits are zero. + if (classified != Vector128.Zero) + { + break; + } + + // Step 5: Compress four bytes into three. + Vector128 res1 = ((str1 << 2) | (str2 >> 4)); + Vector128 res2 = ((str2 << 4) | (str3 >> 2)); + Vector128 res3 = ((str3 << 6) | str4); + + // Step 6: Interleave and store decoded results. + AssertWrite>(dest, destStart, destLength); + AdvSimd.Arm64.StoreVectorAndZip(dest, (res1, res2, res3)); + + src += 64; + dest += 48; + } + while (src <= srcEnd); + + srcBytes = src; + destBytes = dest; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + [CompExactlyDependsOn(typeof(Ssse3))] + private static unsafe void Vector128Decode(TBase64Decoder decoder, ref T* srcBytes, ref byte* destBytes, T* srcEnd, int sourceLength, int destLength, T* srcStart, byte* destStart) + where TBase64Decoder : IBase64Decoder + where T : unmanaged + { + Debug.Assert((Ssse3.IsSupported || AdvSimd.Arm64.IsSupported) && BitConverter.IsLittleEndian); + + // If we have Vector128 support, pick off 16 bytes at a time for as long as we can, + // but make sure that we quit before seeing any == markers at the end of the + // string. Also, because we write four zeroes at the end of the output, ensure + // that there are at least 6 valid bytes of input data remaining to close the + // gap. 16 + 2 + 6 = 24 bytes. + + // The input consists of six character sets in the Base64 alphabet, + // which we need to map back to the 6-bit values they represent. + // There are three ranges, two singles, and then there's the rest. + // + // # From To Add Characters + // 1 [43] [62] +19 + + // 2 [47] [63] +16 / + // 3 [48..57] [52..61] +4 0..9 + // 4 [65..90] [0..25] -65 A..Z + // 5 [97..122] [26..51] -71 a..z + // (6) Everything else => invalid input + + // We will use LUTS for character validation & offset computation + // Remember that 0x2X and 0x0X are the same index for _mm_shuffle_epi8, + // this allows to mask with 0x2F instead of 0x0F and thus save one constant declaration (register and/or memory access) + + // For offsets: + // Perfect hash for lut = ((src>>4)&0x2F)+((src==0x2F)?0xFF:0x00) + // 0000 = garbage + // 0001 = / + // 0010 = + + // 0011 = 0-9 + // 0100 = A-Z + // 0101 = A-Z + // 0110 = a-z + // 0111 = a-z + // 1000 >= garbage + + // For validation, here's the table. + // A character is valid if and only if the AND of the 2 lookups equals 0: + + // hi \ lo 0000 0001 0010 0011 0100 0101 0110 0111 1000 1001 1010 1011 1100 1101 1110 1111 + // LUT 0x15 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x11 0x13 0x1A 0x1B 0x1B 0x1B 0x1A + + // 0000 0X10 char NUL SOH STX ETX EOT ENQ ACK BEL BS HT LF VT FF CR SO SI + // andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 + + // 0001 0x10 char DLE DC1 DC2 DC3 DC4 NAK SYN ETB CAN EM SUB ESC FS GS RS US + // andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 + + // 0010 0x01 char ! " # $ % & ' ( ) * + , - . / + // andlut 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x01 0x00 0x01 0x01 0x01 0x00 + + // 0011 0x02 char 0 1 2 3 4 5 6 7 8 9 : ; < = > ? + // andlut 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x02 0x02 0x02 0x02 0x02 0x02 + + // 0100 0x04 char @ A B C D E F G H I J K L M N 0 + // andlut 0x04 0x00 0x00 0x00 0X00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 + + // 0101 0x08 char P Q R S T U V W X Y Z [ \ ] ^ _ + // andlut 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x08 0x08 0x08 0x08 0x08 + + // 0110 0x04 char ` a b c d e f g h i j k l m n o + // andlut 0x04 0x00 0x00 0x00 0X00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 + // 0111 0X08 char p q r s t u v w x y z { | } ~ + // andlut 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x00 0x08 0x08 0x08 0x08 0x08 + + // 1000 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 + // 1001 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 + // 1010 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 + // 1011 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 + // 1100 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 + // 1101 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 + // 1110 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 + // 1111 0x10 andlut 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 0x10 + + // The JIT won't hoist these "constants", so help it + Vector128 lutHi = Vector128.Create(decoder.Vector128LutHigh).AsByte(); + Vector128 lutLo = Vector128.Create(decoder.Vector128LutLow).AsByte(); + Vector128 lutShift = Vector128.Create(decoder.Vector128LutShift).AsSByte(); + Vector128 packBytesMask = Vector128.Create(0x06000102, 0x090A0405, 0x0C0D0E08, 0xffffffff).AsSByte(); + Vector128 mergeConstant0 = Vector128.Create(0x01400140).AsByte(); + Vector128 mergeConstant1 = Vector128.Create(0x00011000).AsInt16(); + Vector128 one = Vector128.Create((byte)1); + Vector128 mask2F = Vector128.Create(decoder.MaskSlashOrUnderscore); + Vector128 mask8F = Vector128.Create((byte)0x8F); + Vector128 shiftForUnderscore = Vector128.Create((byte)33); + T* src = srcBytes; + byte* dest = destBytes; + + //while (remaining >= 24) + do + { + if (!decoder.TryLoadVector128(src, srcStart, sourceLength, out Vector128 str)) + { + break; + } + + // lookup + Vector128 hiNibbles = Vector128.ShiftRightLogical(str.AsInt32(), 4).AsByte() & mask2F; + + if (!decoder.TryDecode128Core(str, hiNibbles, mask2F, mask8F, lutLo, lutHi, lutShift, shiftForUnderscore, out str)) + { + break; + } + + // in, bits, upper case are most significant bits, lower case are least significant bits + // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ + // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG + // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD + // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA + + Vector128 merge_ab_and_bc; + if (Ssse3.IsSupported) + { + merge_ab_and_bc = Ssse3.MultiplyAddAdjacent(str.AsByte(), mergeConstant0.AsSByte()); + } + else + { + Vector128 evens = AdvSimd.ShiftLeftLogicalWideningLower(AdvSimd.Arm64.UnzipEven(str, one).GetLower(), 6); + Vector128 odds = AdvSimd.Arm64.TransposeOdd(str, Vector128.Zero).AsUInt16(); + merge_ab_and_bc = Vector128.Add(evens, odds).AsInt16(); + } + // 0000kkkk LLllllll 0000JJJJ JJjjKKKK + // 0000hhhh IIiiiiii 0000GGGG GGggHHHH + // 0000eeee FFffffff 0000DDDD DDddEEEE + // 0000bbbb CCcccccc 0000AAAA AAaaBBBB + + Vector128 output; + if (Ssse3.IsSupported) + { + output = Sse2.MultiplyAddAdjacent(merge_ab_and_bc, mergeConstant1); + } + else + { + Vector128 ievens = AdvSimd.ShiftLeftLogicalWideningLower(AdvSimd.Arm64.UnzipEven(merge_ab_and_bc, one.AsInt16()).GetLower(), 12); + Vector128 iodds = AdvSimd.Arm64.TransposeOdd(merge_ab_and_bc, Vector128.Zero).AsInt32(); + output = Vector128.Add(ievens, iodds).AsInt32(); + } + // 00000000 JJJJJJjj KKKKkkkk LLllllll + // 00000000 GGGGGGgg HHHHhhhh IIiiiiii + // 00000000 DDDDDDdd EEEEeeee FFffffff + // 00000000 AAAAAAaa BBBBbbbb CCcccccc + + // Pack bytes together: + str = SimdShuffle(output.AsByte(), packBytesMask.AsByte(), mask8F); + // 00000000 00000000 00000000 00000000 + // LLllllll KKKKkkkk JJJJJJjj IIiiiiii + // HHHHhhhh GGGGGGgg FFffffff EEEEeeee + // DDDDDDdd CCcccccc BBBBbbbb AAAAAAaa + + AssertWrite>(dest, destStart, destLength); + str.Store(dest); + + src += 16; + dest += 12; + } + while (src <= srcEnd); + + srcBytes = src; + destBytes = dest; + } +#endif + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe void WriteThreeLowOrderBytes(byte* destination, int value) + { + destination[0] = (byte)(value >> 16); + destination[1] = (byte)(value >> 8); + destination[2] = (byte)value; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + internal static bool IsWhiteSpace(int value) + { + Debug.Assert(value >= 0 && value <= ushort.MaxValue); + uint charMinusLowUInt32; + return (int)((0xC8000100U << (short)(charMinusLowUInt32 = (ushort)(value - '\t'))) & (charMinusLowUInt32 - 32)) < 0; + } + + internal readonly struct Base64DecoderByte : IBase64Decoder + { + // Pre-computing this table using a custom string(s_characters) and GenerateDecodingMapAndVerify (found in tests) + public ReadOnlySpan DecodingMap => + [ + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 62, -1, -1, -1, 63, //62 is placed at index 43 (for +), 63 at index 47 (for /) + 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, -1, -1, -1, -1, -1, -1, //52-61 are placed at index 48-57 (for 0-9) + -1, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, + 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, -1, -1, -1, -1, -1, //0-25 are placed at index 65-90 (for A-Z) + -1, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, + 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, -1, -1, -1, -1, -1, //26-51 are placed at index 97-122 (for a-z) + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // Bytes over 122 ('z') are invalid and cannot be decoded + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, // Hence, padding the map with 255, which indicates invalid input + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, + ]; + + public ReadOnlySpan VbmiLookup0 => + [ + 0x80808080, 0x80808080, 0x80808080, 0x80808080, + 0x80808080, 0x80808080, 0x80808080, 0x80808080, + 0x80808080, 0x80808080, 0x3e808080, 0x3f808080, + 0x37363534, 0x3b3a3938, 0x80803d3c, 0x80808080 + ]; + + public ReadOnlySpan VbmiLookup1 => + [ + 0x02010080, 0x06050403, 0x0a090807, 0x0e0d0c0b, + 0x1211100f, 0x16151413, 0x80191817, 0x80808080, + 0x1c1b1a80, 0x201f1e1d, 0x24232221, 0x28272625, + 0x2c2b2a29, 0x302f2e2d, 0x80333231, 0x80808080 + ]; + + public ReadOnlySpan Avx2LutHigh => + [ + 0x10, 0x10, 0x01, 0x02, + 0x04, 0x08, 0x04, 0x08, + 0x10, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x01, 0x02, + 0x04, 0x08, 0x04, 0x08, + 0x10, 0x10, 0x10, 0x10, + 0x10, 0x10, 0x10, 0x10 + ]; + + public ReadOnlySpan Avx2LutLow => + [ + 0x15, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x13, 0x1A, + 0x1B, 0x1B, 0x1B, 0x1A, + 0x15, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x11, 0x11, + 0x11, 0x11, 0x13, 0x1A, + 0x1B, 0x1B, 0x1B, 0x1A + ]; + + public ReadOnlySpan Avx2LutShift => + [ + 0, 16, 19, 4, + -65, -65, -71, -71, + 0, 0, 0, 0, + 0, 0, 0, 0, + 0, 16, 19, 4, + -65, -65, -71, -71, + 0, 0, 0, 0, + 0, 0, 0, 0 + ]; + + public byte MaskSlashOrUnderscore => (byte)'/'; + + public ReadOnlySpan Vector128LutHigh => [0x02011010, 0x08040804, 0x10101010, 0x10101010]; + + public ReadOnlySpan Vector128LutLow => [0x11111115, 0x11111111, 0x1A131111, 0x1A1B1B1B]; + + public ReadOnlySpan Vector128LutShift => [0x04131000, 0xb9b9bfbf, 0x00000000, 0x00000000]; + + public ReadOnlySpan AdvSimdLutOne3 => [0xFFFFFFFF, 0xFFFFFFFF, 0x3EFFFFFF, 0x3FFFFFFF]; + + public uint AdvSimdLutTwo3Uint1 => 0x1B1AFFFF; + + public int GetMaxDecodedLength(int utf8Length) => Base64.GetMaxDecodedFromUtf8Length(utf8Length); + + public bool IsInvalidLength(int bufferLength) => bufferLength % 4 != 0; // only decode input if it is a multiple of 4 + + public bool IsValidPadding(uint padChar) => padChar == EncodingPad; + + public int SrcLength(bool _, int utf8Length) => utf8Length & ~0x3; // only decode input up to the closest multiple of 4. + +#if NET + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + [CompExactlyDependsOn(typeof(Ssse3))] + public bool TryDecode128Core( + Vector128 str, + Vector128 hiNibbles, + Vector128 maskSlashOrUnderscore, + Vector128 mask8F, + Vector128 lutLow, + Vector128 lutHigh, + Vector128 lutShift, + Vector128 _, + out Vector128 result) + { + Vector128 loNibbles = str & maskSlashOrUnderscore; + Vector128 hi = SimdShuffle(lutHigh, hiNibbles, mask8F); + Vector128 lo = SimdShuffle(lutLow, loNibbles, mask8F); + + // Check for invalid input: if any "and" values from lo and hi are not zero, + // fall back on bytewise code to do error checking and reporting: + if ((lo & hi) != Vector128.Zero) + { + result = default; + return false; + } + + Vector128 eq2F = Vector128.Equals(str, maskSlashOrUnderscore); + Vector128 shift = SimdShuffle(lutShift.AsByte(), (eq2F + hiNibbles), mask8F); + + // Now simply add the delta values to the input: + result = str + shift; + + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx2))] + public bool TryDecode256Core( + Vector256 str, + Vector256 hiNibbles, + Vector256 maskSlashOrUnderscore, + Vector256 lutLow, + Vector256 lutHigh, + Vector256 lutShift, + Vector256 _, + out Vector256 result) + { + Vector256 loNibbles = Avx2.And(str, maskSlashOrUnderscore); + Vector256 hi = Avx2.Shuffle(lutHigh, hiNibbles); + Vector256 lo = Avx2.Shuffle(lutLow, loNibbles); + + if (!Avx.TestZ(lo, hi)) + { + result = default; + return false; + } + + Vector256 eq2F = Avx2.CompareEqual(str, maskSlashOrUnderscore); + Vector256 shift = Avx2.Shuffle(lutShift, Avx2.Add(eq2F, hiNibbles)); + + result = Avx2.Add(str, shift); + + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public unsafe bool TryLoadVector512(byte* src, byte* srcStart, int sourceLength, out Vector512 str) + { + AssertRead>(src, srcStart, sourceLength); + str = Vector512.Load(src).AsSByte(); + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx2))] + public unsafe bool TryLoadAvxVector256(byte* src, byte* srcStart, int sourceLength, out Vector256 str) + { + AssertRead>(src, srcStart, sourceLength); + str = Avx.LoadVector256(src).AsSByte(); + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public unsafe bool TryLoadVector128(byte* src, byte* srcStart, int sourceLength, out Vector128 str) + { + AssertRead>(src, srcStart, sourceLength); + str = Vector128.LoadUnsafe(ref *src); + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + public unsafe bool TryLoadArmVector128x4(byte* src, byte* srcStart, int sourceLength, + out Vector128 str1, out Vector128 str2, out Vector128 str3, out Vector128 str4) + { + AssertRead>(src, srcStart, sourceLength); + (str1, str2, str3, str4) = AdvSimd.Arm64.Load4xVector128AndUnzip(src); + + return true; + } +#endif + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public unsafe int DecodeFourElements(byte* source, ref sbyte decodingMap) + { + // The 'source' span expected to have at least 4 elements, and the 'decodingMap' consists 256 sbytes + uint t0 = source[0]; + uint t1 = source[1]; + uint t2 = source[2]; + uint t3 = source[3]; + + int i0 = Unsafe.Add(ref decodingMap, (int)t0); + int i1 = Unsafe.Add(ref decodingMap, (int)t1); + int i2 = Unsafe.Add(ref decodingMap, (int)t2); + int i3 = Unsafe.Add(ref decodingMap, (int)t3); + + i0 <<= 18; + i1 <<= 12; + i2 <<= 6; + + i0 |= i3; + i1 |= i2; + + i0 |= i1; + return i0; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public unsafe int DecodeRemaining(byte* srcEnd, ref sbyte decodingMap, long remaining, out uint t2, out uint t3) + { + uint t0; + uint t1; + t2 = EncodingPad; + t3 = EncodingPad; + switch (remaining) + { + case 2: + t0 = srcEnd[-2]; + t1 = srcEnd[-1]; + break; + case 3: + t0 = srcEnd[-3]; + t1 = srcEnd[-2]; + t2 = srcEnd[-1]; + break; + case 4: + t0 = srcEnd[-4]; + t1 = srcEnd[-3]; + t2 = srcEnd[-2]; + t3 = srcEnd[-1]; + break; + default: + return -1; + } + + int i0 = Unsafe.Add(ref decodingMap, (IntPtr)t0); + int i1 = Unsafe.Add(ref decodingMap, (IntPtr)t1); + + i0 <<= 18; + i1 <<= 12; + + i0 |= i1; + return i0; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int IndexOfAnyExceptWhiteSpace(ReadOnlySpan span) + { + for (int i = 0; i < span.Length; i++) + { + if (!IsWhiteSpace(span[i])) + { + return i; + } + } + + return -1; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public OperationStatus DecodeWithWhiteSpaceBlockwiseWrapper(TBase64Decoder decoder, ReadOnlySpan utf8, + Span bytes, ref int bytesConsumed, ref int bytesWritten, bool isFinalBlock = true) + where TBase64Decoder : IBase64Decoder => + DecodeWithWhiteSpaceBlockwise(decoder, utf8, bytes, ref bytesConsumed, ref bytesWritten, isFinalBlock); + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Helper/Base64EncoderHelper.cs b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Helper/Base64EncoderHelper.cs new file mode 100644 index 00000000000000..252b892bed0ca0 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Helper/Base64EncoderHelper.cs @@ -0,0 +1,789 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +#if NET +using System.Runtime.Intrinsics; +using System.Runtime.Intrinsics.Arm; +using System.Runtime.Intrinsics.X86; +#endif + +namespace System.Buffers.Text +{ + internal static partial class Base64Helper + { + internal static unsafe OperationStatus EncodeTo(TBase64Encoder encoder, ReadOnlySpan source, + Span destination, out int bytesConsumed, out int bytesWritten, bool isFinalBlock = true) + where TBase64Encoder : IBase64Encoder + where T : unmanaged + { + if (source.IsEmpty) + { + bytesConsumed = 0; + bytesWritten = 0; + return OperationStatus.Done; + } + + fixed (byte* srcBytes = &MemoryMarshal.GetReference(source)) + fixed (T* destBytes = &MemoryMarshal.GetReference(destination)) + { + int srcLength = source.Length; + int destLength = destination.Length; + int maxSrcLength = encoder.GetMaxSrcLength(srcLength, destLength); + + byte* src = srcBytes; + T* dest = destBytes; + byte* srcEnd = srcBytes + (uint)srcLength; + byte* srcMax = srcBytes + (uint)maxSrcLength; + +#if NET + if (maxSrcLength >= 16) + { + byte* end = srcMax - 64; + if (Vector512.IsHardwareAccelerated && Avx512Vbmi.IsSupported && (end >= src)) + { + Avx512Encode(encoder, ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); + + if (src == srcEnd) + goto DoneExit; + } + + end = srcMax - 32; + if (Avx2.IsSupported && (end >= src)) + { + Avx2Encode(encoder, ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); + + if (src == srcEnd) + goto DoneExit; + } + + end = srcMax - 48; + if (AdvSimd.Arm64.IsSupported && (end >= src)) + { + AdvSimdEncode(encoder, ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); + + if (src == srcEnd) + goto DoneExit; + } + + end = srcMax - 16; + if ((Ssse3.IsSupported || AdvSimd.Arm64.IsSupported) && BitConverter.IsLittleEndian && (end >= src)) + { + Vector128Encode(encoder, ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes); + + if (src == srcEnd) + goto DoneExit; + } + } +#endif + ref byte encodingMap = ref MemoryMarshal.GetReference(encoder.EncodingMap); + + srcMax -= 2; + while (src < srcMax) + { + encoder.EncodeThreeAndWrite(src, dest, ref encodingMap); + src += 3; + dest += 4; + } + + if (srcMax + 2 != srcEnd) + goto DestinationTooSmallExit; + + if (!isFinalBlock) + { + if (src == srcEnd) + goto DoneExit; + + goto NeedMoreData; + } + + if (src + 1 == srcEnd) + { + encoder.EncodeOneOptionallyPadTwo(src, dest, ref encodingMap); + src += 1; + dest += encoder.IncrementPadTwo; + } + else if (src + 2 == srcEnd) + { + encoder.EncodeTwoOptionallyPadOne(src, dest, ref encodingMap); + src += 2; + dest += encoder.IncrementPadOne; + } + + DoneExit: + bytesConsumed = (int)(src - srcBytes); + bytesWritten = (int)(dest - destBytes); + return OperationStatus.Done; + + DestinationTooSmallExit: + bytesConsumed = (int)(src - srcBytes); + bytesWritten = (int)(dest - destBytes); + return OperationStatus.DestinationTooSmall; + + NeedMoreData: + bytesConsumed = (int)(src - srcBytes); + bytesWritten = (int)(dest - destBytes); + return OperationStatus.NeedMoreData; + } + } + +#if NET + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx512BW))] + [CompExactlyDependsOn(typeof(Avx512Vbmi))] + private static unsafe void Avx512Encode(TBase64Encoder encoder, ref byte* srcBytes, ref T* destBytes, byte* srcEnd, int sourceLength, int destLength, byte* srcStart, T* destStart) + where TBase64Encoder : IBase64Encoder + where T : unmanaged + { + // Reference for VBMI implementation : https://github.com/WojciechMula/base64simd/tree/master/encode + // If we have AVX512 support, pick off 48 bytes at a time for as long as we can. + // But because we read 64 bytes at a time, ensure we have enough room to do a + // full 64-byte read without segfaulting. + + byte* src = srcBytes; + T* dest = destBytes; + + // The JIT won't hoist these "constants", so help it + Vector512 shuffleVecVbmi = Vector512.Create( + 0x01020001, 0x04050304, 0x07080607, 0x0a0b090a, + 0x0d0e0c0d, 0x10110f10, 0x13141213, 0x16171516, + 0x191a1819, 0x1c1d1b1c, 0x1f201e1f, 0x22232122, + 0x25262425, 0x28292728, 0x2b2c2a2b, 0x2e2f2d2e).AsSByte(); + Vector512 vbmiLookup = Vector512.Create(encoder.EncodingMap).AsSByte(); + + Vector512 maskAC = Vector512.Create((uint)0x0fc0fc00).AsUInt16(); + Vector512 maskBB = Vector512.Create((uint)0x3f003f00); + Vector512 shiftAC = Vector512.Create((uint)0x0006000a).AsUInt16(); + Vector512 shiftBB = Vector512.Create((uint)0x00080004).AsUInt16(); + + AssertRead>(src, srcStart, sourceLength); + + // This algorithm requires AVX512VBMI support. + // Vbmi was first introduced in CannonLake and is available from IceLake on. + + // str = [...|PONM|LKJI|HGFE|DCBA] + Vector512 str = Vector512.Load(src).AsSByte(); + + while (true) + { + // Step 1 : Split 48 bytes into 64 bytes with each byte using 6-bits from input + // str = [...|KLJK|HIGH|EFDE|BCAB] + str = Avx512Vbmi.PermuteVar64x8(str, shuffleVecVbmi); + + // TO-DO- This can be achieved faster with multishift + // Consider the first 4 bytes - BCAB + // temp1 = [...|0000cccc|cc000000|aaaaaa00|00000000] + Vector512 temp1 = (str.AsUInt16() & maskAC); + + // temp2 = [...|00000000|00cccccc|00000000|00aaaaaa] + Vector512 temp2 = Avx512BW.ShiftRightLogicalVariable(temp1, shiftAC).AsUInt16(); + + // temp3 = [...|ccdddddd|00000000|aabbbbbb|cccc0000] + Vector512 temp3 = Avx512BW.ShiftLeftLogicalVariable(str.AsUInt16(), shiftBB).AsUInt16(); + + // str = [...|00dddddd|00cccccc|00bbbbbb|00aaaaaa] + str = Vector512.ConditionalSelect(maskBB, temp3.AsUInt32(), temp2.AsUInt32()).AsSByte(); + + // Step 2: Now we have the indices calculated. Next step is to use these indices to translate. + str = Avx512Vbmi.PermuteVar64x8(vbmiLookup, str); + + encoder.StoreVector512ToDestination(dest, destStart, destLength, str.AsByte()); + + src += 48; + dest += 64; + + if (src > srcEnd) + break; + + AssertRead>(src, srcStart, sourceLength); + str = Vector512.Load(src).AsSByte(); + } + + srcBytes = src; + destBytes = dest; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx2))] + private static unsafe void Avx2Encode(TBase64Encoder encoder, ref byte* srcBytes, ref T* destBytes, byte* srcEnd, int sourceLength, int destLength, byte* srcStart, T* destStart) + where TBase64Encoder : IBase64Encoder + where T : unmanaged + { + // If we have AVX2 support, pick off 24 bytes at a time for as long as we can. + // But because we read 32 bytes at a time, ensure we have enough room to do a + // full 32-byte read without segfaulting. + + // translation from SSSE3 into AVX2 of procedure + // This one works with shifted (4 bytes) input in order to + // be able to work efficiently in the 2 128-bit lanes + + // srcBytes, bytes MSB to LSB: + // 0 0 0 0 x w v u t s r q p o n m + // l k j i h g f e d c b a 0 0 0 0 + + // The JIT won't hoist these "constants", so help it + Vector256 shuffleVec = Vector256.Create( + 5, 4, 6, 5, + 8, 7, 9, 8, + 11, 10, 12, 11, + 14, 13, 15, 14, + 1, 0, 2, 1, + 4, 3, 5, 4, + 7, 6, 8, 7, + 10, 9, 11, 10); + + Vector256 lut = Vector256.Create( + 65, 71, -4, -4, + -4, -4, -4, -4, + -4, -4, -4, -4, + encoder.Avx2LutChar62, encoder.Avx2LutChar63, 0, 0, + 65, 71, -4, -4, + -4, -4, -4, -4, + -4, -4, -4, -4, + encoder.Avx2LutChar62, encoder.Avx2LutChar63, 0, 0); + + Vector256 maskAC = Vector256.Create(0x0fc0fc00).AsSByte(); + Vector256 maskBB = Vector256.Create(0x003f03f0).AsSByte(); + Vector256 shiftAC = Vector256.Create(0x04000040).AsUInt16(); + Vector256 shiftBB = Vector256.Create(0x01000010).AsInt16(); + Vector256 const51 = Vector256.Create((byte)51); + Vector256 const25 = Vector256.Create((sbyte)25); + + byte* src = srcBytes; + T* dest = destBytes; + + // first load is done at c-0 not to get a segfault + AssertRead>(src, srcStart, sourceLength); + Vector256 str = Avx.LoadVector256(src).AsSByte(); + + // shift by 4 bytes, as required by Reshuffle + str = Avx2.PermuteVar8x32(str.AsInt32(), Vector256.Create( + 0, 0, 0, 0, + 0, 0, 0, 0, + 1, 0, 0, 0, + 2, 0, 0, 0, + 3, 0, 0, 0, + 4, 0, 0, 0, + 5, 0, 0, 0, + 6, 0, 0, 0).AsInt32()).AsSByte(); + + // Next loads are done at src-4, as required by Reshuffle, so shift it once + src -= 4; + + while (true) + { + // Reshuffle + str = Avx2.Shuffle(str, shuffleVec); + // str, bytes MSB to LSB: + // w x v w + // t u s t + // q r p q + // n o m n + // k l j k + // h i g h + // e f d e + // b c a b + + Vector256 t0 = Avx2.And(str, maskAC); + // bits, upper case are most significant bits, lower case are least significant bits. + // 0000wwww XX000000 VVVVVV00 00000000 + // 0000tttt UU000000 SSSSSS00 00000000 + // 0000qqqq RR000000 PPPPPP00 00000000 + // 0000nnnn OO000000 MMMMMM00 00000000 + // 0000kkkk LL000000 JJJJJJ00 00000000 + // 0000hhhh II000000 GGGGGG00 00000000 + // 0000eeee FF000000 DDDDDD00 00000000 + // 0000bbbb CC000000 AAAAAA00 00000000 + + Vector256 t2 = Avx2.And(str, maskBB); + // 00000000 00xxxxxx 000000vv WWWW0000 + // 00000000 00uuuuuu 000000ss TTTT0000 + // 00000000 00rrrrrr 000000pp QQQQ0000 + // 00000000 00oooooo 000000mm NNNN0000 + // 00000000 00llllll 000000jj KKKK0000 + // 00000000 00iiiiii 000000gg HHHH0000 + // 00000000 00ffffff 000000dd EEEE0000 + // 00000000 00cccccc 000000aa BBBB0000 + + Vector256 t1 = Avx2.MultiplyHigh(t0.AsUInt16(), shiftAC); + // 00000000 00wwwwXX 00000000 00VVVVVV + // 00000000 00ttttUU 00000000 00SSSSSS + // 00000000 00qqqqRR 00000000 00PPPPPP + // 00000000 00nnnnOO 00000000 00MMMMMM + // 00000000 00kkkkLL 00000000 00JJJJJJ + // 00000000 00hhhhII 00000000 00GGGGGG + // 00000000 00eeeeFF 00000000 00DDDDDD + // 00000000 00bbbbCC 00000000 00AAAAAA + + Vector256 t3 = Avx2.MultiplyLow(t2.AsInt16(), shiftBB); + // 00xxxxxx 00000000 00vvWWWW 00000000 + // 00uuuuuu 00000000 00ssTTTT 00000000 + // 00rrrrrr 00000000 00ppQQQQ 00000000 + // 00oooooo 00000000 00mmNNNN 00000000 + // 00llllll 00000000 00jjKKKK 00000000 + // 00iiiiii 00000000 00ggHHHH 00000000 + // 00ffffff 00000000 00ddEEEE 00000000 + // 00cccccc 00000000 00aaBBBB 00000000 + + str = Avx2.Or(t1.AsSByte(), t3.AsSByte()); + // 00xxxxxx 00wwwwXX 00vvWWWW 00VVVVVV + // 00uuuuuu 00ttttUU 00ssTTTT 00SSSSSS + // 00rrrrrr 00qqqqRR 00ppQQQQ 00PPPPPP + // 00oooooo 00nnnnOO 00mmNNNN 00MMMMMM + // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ + // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG + // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD + // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA + + // Translation + // LUT contains Absolute offset for all ranges: + // Translate values 0..63 to the Base64 alphabet. There are five sets: + // # From To Abs Index Characters + // 0 [0..25] [65..90] +65 0 ABCDEFGHIJKLMNOPQRSTUVWXYZ + // 1 [26..51] [97..122] +71 1 abcdefghijklmnopqrstuvwxyz + // 2 [52..61] [48..57] -4 [2..11] 0123456789 + // 3 [62] [43] -19 12 + + // 4 [63] [47] -16 13 / + + // Create LUT indices from input: + // the index for range #0 is right, others are 1 less than expected: + Vector256 indices = Avx2.SubtractSaturate(str.AsByte(), const51); + + // mask is 0xFF (-1) for range #[1..4] and 0x00 for range #0: + Vector256 mask = Avx2.CompareGreaterThan(str, const25); + + // subtract -1, so add 1 to indices for range #[1..4], All indices are now correct: + Vector256 tmp = Avx2.Subtract(indices.AsSByte(), mask); + + // Add offsets to input values: + str = Avx2.Add(str, Avx2.Shuffle(lut, tmp)); + + encoder.StoreVector256ToDestination(dest, destStart, destLength, str.AsByte()); + + src += 24; + dest += 32; + + if (src > srcEnd) + break; + + // Load at src-4, as required by Reshuffle (already shifted by -4) + AssertRead>(src, srcStart, sourceLength); + str = Avx.LoadVector256(src).AsSByte(); + } + + srcBytes = src + 4; + destBytes = dest; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + private static unsafe void AdvSimdEncode(TBase64Encoder encoder, ref byte* srcBytes, ref T* destBytes, byte* srcEnd, int sourceLength, int destLength, byte* srcStart, T* destStart) + where TBase64Encoder : IBase64Encoder + where T : unmanaged + { + // C# implementation of https://github.com/aklomp/base64/blob/3a5add8652076612a8407627a42c768736a4263f/lib/arch/neon64/enc_loop.c + Vector128 str1; + Vector128 str2; + Vector128 str3; + Vector128 res1; + Vector128 res2; + Vector128 res3; + Vector128 res4; + Vector128 tblEnc1 = Vector128.Create("ABCDEFGHIJKLMNOP"u8).AsByte(); + Vector128 tblEnc2 = Vector128.Create("QRSTUVWXYZabcdef"u8).AsByte(); + Vector128 tblEnc3 = Vector128.Create("ghijklmnopqrstuv"u8).AsByte(); + Vector128 tblEnc4 = Vector128.Create(encoder.AdvSimdLut4).AsByte(); + byte* src = srcBytes; + T* dest = destBytes; + + // If we have Neon support, pick off 48 bytes at a time for as long as we can. + do + { + // Load 48 bytes and deinterleave: + AssertRead>(src, srcStart, sourceLength); + (str1, str2, str3) = AdvSimd.Arm64.Load3xVector128AndUnzip(src); + + // Divide bits of three input bytes over four output bytes: + res1 = AdvSimd.ShiftRightLogical(str1, 2); + res2 = AdvSimd.ShiftRightLogical(str2, 4); + res3 = AdvSimd.ShiftRightLogical(str3, 6); + res2 = AdvSimd.ShiftLeftAndInsert(res2, str1, 4); + res3 = AdvSimd.ShiftLeftAndInsert(res3, str2, 2); + + // Clear top two bits: + res2 &= AdvSimd.DuplicateToVector128((byte)0x3F); + res3 &= AdvSimd.DuplicateToVector128((byte)0x3F); + res4 = str3 & AdvSimd.DuplicateToVector128((byte)0x3F); + + // The bits have now been shifted to the right locations; + // translate their values 0..63 to the Base64 alphabet. + // Use a 64-byte table lookup: + res1 = AdvSimd.Arm64.VectorTableLookup((tblEnc1, tblEnc2, tblEnc3, tblEnc4), res1); + res2 = AdvSimd.Arm64.VectorTableLookup((tblEnc1, tblEnc2, tblEnc3, tblEnc4), res2); + res3 = AdvSimd.Arm64.VectorTableLookup((tblEnc1, tblEnc2, tblEnc3, tblEnc4), res3); + res4 = AdvSimd.Arm64.VectorTableLookup((tblEnc1, tblEnc2, tblEnc3, tblEnc4), res4); + + // Interleave and store result: + encoder.StoreArmVector128x4ToDestination(dest, destStart, destLength, res1, res2, res3, res4); + + src += 48; + dest += 64; + } while (src <= srcEnd); + + srcBytes = src; + destBytes = dest; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Ssse3))] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + private static unsafe void Vector128Encode(TBase64Encoder encoder, ref byte* srcBytes, ref T* destBytes, byte* srcEnd, int sourceLength, int destLength, byte* srcStart, T* destStart) + where TBase64Encoder : IBase64Encoder + where T : unmanaged + { + // If we have SSSE3 support, pick off 12 bytes at a time for as long as we can. + // But because we read 16 bytes at a time, ensure we have enough room to do a + // full 16-byte read without segfaulting. + + // srcBytes, bytes MSB to LSB: + // 0 0 0 0 l k j i h g f e d c b a + + // The JIT won't hoist these "constants", so help it + Vector128 shuffleVec = Vector128.Create(0x01020001, 0x04050304, 0x07080607, 0x0A0B090A).AsByte(); + Vector128 lut = Vector128.Create(0xFCFC4741, 0xFCFCFCFC, 0xFCFCFCFC, encoder.Ssse3AdvSimdLutE3).AsByte(); + Vector128 maskAC = Vector128.Create(0x0fc0fc00).AsByte(); + Vector128 maskBB = Vector128.Create(0x003f03f0).AsByte(); + Vector128 shiftAC = Vector128.Create(0x04000040).AsUInt16(); + Vector128 shiftBB = Vector128.Create(0x01000010).AsInt16(); + Vector128 const51 = Vector128.Create((byte)51); + Vector128 const25 = Vector128.Create((sbyte)25); + Vector128 mask8F = Vector128.Create((byte)0x8F); + + byte* src = srcBytes; + T* dest = destBytes; + + //while (remaining >= 16) + do + { + AssertRead>(src, srcStart, sourceLength); + Vector128 str = Vector128.LoadUnsafe(ref *src); + + // Reshuffle + str = SimdShuffle(str, shuffleVec, mask8F); + // str, bytes MSB to LSB: + // k l j k + // h i g h + // e f d e + // b c a b + + Vector128 t0 = str & maskAC; + // bits, upper case are most significant bits, lower case are least significant bits + // 0000kkkk LL000000 JJJJJJ00 00000000 + // 0000hhhh II000000 GGGGGG00 00000000 + // 0000eeee FF000000 DDDDDD00 00000000 + // 0000bbbb CC000000 AAAAAA00 00000000 + + Vector128 t2 = str & maskBB; + // 00000000 00llllll 000000jj KKKK0000 + // 00000000 00iiiiii 000000gg HHHH0000 + // 00000000 00ffffff 000000dd EEEE0000 + // 00000000 00cccccc 000000aa BBBB0000 + + Vector128 t1; + if (Ssse3.IsSupported) + { + t1 = Sse2.MultiplyHigh(t0.AsUInt16(), shiftAC); + } + else + { + Vector128 odd = Vector128.ShiftRightLogical(AdvSimd.Arm64.UnzipOdd(t0.AsUInt16(), t0.AsUInt16()), 6); + Vector128 even = Vector128.ShiftRightLogical(AdvSimd.Arm64.UnzipEven(t0.AsUInt16(), t0.AsUInt16()), 10); + t1 = AdvSimd.Arm64.ZipLow(even, odd); + } + // 00000000 00kkkkLL 00000000 00JJJJJJ + // 00000000 00hhhhII 00000000 00GGGGGG + // 00000000 00eeeeFF 00000000 00DDDDDD + // 00000000 00bbbbCC 00000000 00AAAAAA + + Vector128 t3 = t2.AsInt16() * shiftBB; + // 00llllll 00000000 00jjKKKK 00000000 + // 00iiiiii 00000000 00ggHHHH 00000000 + // 00ffffff 00000000 00ddEEEE 00000000 + // 00cccccc 00000000 00aaBBBB 00000000 + + str = t1.AsByte() | t3.AsByte(); + // 00llllll 00kkkkLL 00jjKKKK 00JJJJJJ + // 00iiiiii 00hhhhII 00ggHHHH 00GGGGGG + // 00ffffff 00eeeeFF 00ddEEEE 00DDDDDD + // 00cccccc 00bbbbCC 00aaBBBB 00AAAAAA + + // Translation + // LUT contains Absolute offset for all ranges: + // Translate values 0..63 to the Base64 alphabet. There are five sets: + // # From To Abs Index Characters + // 0 [0..25] [65..90] +65 0 ABCDEFGHIJKLMNOPQRSTUVWXYZ + // 1 [26..51] [97..122] +71 1 abcdefghijklmnopqrstuvwxyz + // 2 [52..61] [48..57] -4 [2..11] 0123456789 + // 3 [62] [43] -19 12 + + // 4 [63] [47] -16 13 / + + // Create LUT indices from input: + // the index for range #0 is right, others are 1 less than expected: + Vector128 indices; + if (Ssse3.IsSupported) + { + indices = Sse2.SubtractSaturate(str.AsByte(), const51); + } + else + { + indices = AdvSimd.SubtractSaturate(str.AsByte(), const51); + } + + // mask is 0xFF (-1) for range #[1..4] and 0x00 for range #0: + Vector128 mask = Vector128.GreaterThan(str.AsSByte(), const25); + + // subtract -1, so add 1 to indices for range #[1..4], All indices are now correct: + Vector128 tmp = indices.AsSByte() - mask; + + // Add offsets to input values: + str += SimdShuffle(lut, tmp.AsByte(), mask8F); + + encoder.StoreVector128ToDestination(dest, destStart, destLength, str); + + src += 12; + dest += 16; + } + while (src <= srcEnd); + + srcBytes = src; + destBytes = dest; + } +#endif + + internal static unsafe OperationStatus EncodeToUtf8InPlace(TBase64Encoder encoder, Span buffer, int dataLength, out int bytesWritten) + where TBase64Encoder : IBase64Encoder + { + if (buffer.IsEmpty) + { + bytesWritten = 0; + return OperationStatus.Done; + } + + fixed (byte* bufferBytes = &MemoryMarshal.GetReference(buffer)) + { + int encodedLength = encoder.GetMaxEncodedLength(dataLength); + if (buffer.Length < encodedLength) + { + bytesWritten = 0; + return OperationStatus.DestinationTooSmall; + } + + int leftover = (int)((uint)dataLength % 3); // how many bytes after packs of 3 + + uint destinationIndex = encoder.GetInPlaceDestinationLength(encodedLength, leftover); + uint sourceIndex = (uint)(dataLength - leftover); + ref byte encodingMap = ref MemoryMarshal.GetReference(encoder.EncodingMap); + + // encode last pack to avoid conditional in the main loop + if (leftover != 0) + { + if (leftover == 1) + { + encoder.EncodeOneOptionallyPadTwo(bufferBytes + sourceIndex, bufferBytes + destinationIndex, ref encodingMap); + } + else + { + encoder.EncodeTwoOptionallyPadOne(bufferBytes + sourceIndex, bufferBytes + destinationIndex, ref encodingMap); + } + + destinationIndex -= 4; + } + + sourceIndex -= 3; + while ((int)sourceIndex >= 0) + { + uint result = Encode(bufferBytes + sourceIndex, ref encodingMap); + Unsafe.WriteUnaligned(bufferBytes + destinationIndex, result); + destinationIndex -= 4; + sourceIndex -= 3; + } + + bytesWritten = encodedLength; + return OperationStatus.Done; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static unsafe uint Encode(byte* threeBytes, ref byte encodingMap) + { + uint t0 = threeBytes[0]; + uint t1 = threeBytes[1]; + uint t2 = threeBytes[2]; + + uint i = (t0 << 16) | (t1 << 8) | t2; + + uint i0 = Unsafe.Add(ref encodingMap, (IntPtr)(i >> 18)); + uint i1 = Unsafe.Add(ref encodingMap, (IntPtr)((i >> 12) & 0x3F)); + uint i2 = Unsafe.Add(ref encodingMap, (IntPtr)((i >> 6) & 0x3F)); + uint i3 = Unsafe.Add(ref encodingMap, (IntPtr)(i & 0x3F)); + + if (BitConverter.IsLittleEndian) + { + return i0 | (i1 << 8) | (i2 << 16) | (i3 << 24); + } + else + { + return (i0 << 24) | (i1 << 16) | (i2 << 8) | i3; + } + } + + internal const uint EncodingPad = '='; // '=', for padding + + internal const int MaximumEncodeLength = (int.MaxValue / 4) * 3; // 1610612733 + + internal readonly struct Base64EncoderByte : IBase64Encoder + { + public ReadOnlySpan EncodingMap => "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"u8; + + public sbyte Avx2LutChar62 => -19; // char '+' diff + + public sbyte Avx2LutChar63 => -16; // char '/' diff + + public ReadOnlySpan AdvSimdLut4 => "wxyz0123456789+/"u8; + + public uint Ssse3AdvSimdLutE3 => 0x0000F0ED; + + public int IncrementPadTwo => 4; + + public int IncrementPadOne => 4; + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public int GetMaxSrcLength(int srcLength, int destLength) => + srcLength <= MaximumEncodeLength && destLength >= Base64.GetMaxEncodedToUtf8Length(srcLength) ? + srcLength : (destLength >> 2) * 3; + + public uint GetInPlaceDestinationLength(int encodedLength, int _) => (uint)(encodedLength - 4); + + public int GetMaxEncodedLength(int srcLength) => Base64.GetMaxEncodedToUtf8Length(srcLength); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public unsafe void EncodeOneOptionallyPadTwo(byte* oneByte, byte* dest, ref byte encodingMap) + { + uint t0 = oneByte[0]; + + uint i = t0 << 8; + + uint i0 = Unsafe.Add(ref encodingMap, (IntPtr)(i >> 10)); + uint i1 = Unsafe.Add(ref encodingMap, (IntPtr)((i >> 4) & 0x3F)); + + if (BitConverter.IsLittleEndian) + { + dest[0] = (byte)i0; + dest[1] = (byte)i1; + dest[2] = (byte)EncodingPad; + dest[3] = (byte)EncodingPad; + } + else + { + dest[3] = (byte)i0; + dest[2] = (byte)i1; + dest[1] = (byte)EncodingPad; + dest[0] = (byte)EncodingPad; + } + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public unsafe void EncodeTwoOptionallyPadOne(byte* twoBytes, byte* dest, ref byte encodingMap) + { + uint t0 = twoBytes[0]; + uint t1 = twoBytes[1]; + + uint i = (t0 << 16) | (t1 << 8); + + uint i0 = Unsafe.Add(ref encodingMap, (IntPtr)(i >> 18)); + uint i1 = Unsafe.Add(ref encodingMap, (IntPtr)((i >> 12) & 0x3F)); + uint i2 = Unsafe.Add(ref encodingMap, (IntPtr)((i >> 6) & 0x3F)); + + if (BitConverter.IsLittleEndian) + { + dest[0] = (byte)i0; + dest[1] = (byte)i1; + dest[2] = (byte)i2; + dest[3] = (byte)EncodingPad; + } + else + { + dest[3] = (byte)i0; + dest[2] = (byte)i1; + dest[1] = (byte)i2; + dest[0] = (byte)EncodingPad; + } + } + +#if NET + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public unsafe void StoreVector512ToDestination(byte* dest, byte* destStart, int destLength, Vector512 str) + { + AssertWrite>(dest, destStart, destLength); + str.Store(dest); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(Avx2))] + public unsafe void StoreVector256ToDestination(byte* dest, byte* destStart, int destLength, Vector256 str) + { + AssertWrite>(dest, destStart, destLength); + Avx.Store(dest, str.AsByte()); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public unsafe void StoreVector128ToDestination(byte* dest, byte* destStart, int destLength, Vector128 str) + { + AssertWrite>(dest, destStart, destLength); + str.Store(dest); + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + public unsafe void StoreArmVector128x4ToDestination(byte* dest, byte* destStart, int destLength, + Vector128 res1, Vector128 res2, Vector128 res3, Vector128 res4) + { + AssertWrite>(dest, destStart, destLength); + AdvSimd.Arm64.StoreVectorAndZip(dest, (res1, res2, res3, res4)); + } +#endif + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public unsafe void EncodeThreeAndWrite(byte* threeBytes, byte* destination, ref byte encodingMap) + { + uint t0 = threeBytes[0]; + uint t1 = threeBytes[1]; + uint t2 = threeBytes[2]; + + uint i = (t0 << 16) | (t1 << 8) | t2; + + byte i0 = Unsafe.Add(ref encodingMap, (IntPtr)(i >> 18)); + byte i1 = Unsafe.Add(ref encodingMap, (IntPtr)((i >> 12) & 0x3F)); + byte i2 = Unsafe.Add(ref encodingMap, (IntPtr)((i >> 6) & 0x3F)); + byte i3 = Unsafe.Add(ref encodingMap, (IntPtr)(i & 0x3F)); + + if (BitConverter.IsLittleEndian) + { + destination[0] = i0; + destination[1] = i1; + destination[2] = i2; + destination[3] = i3; + } + else + { + destination[3] = i0; + destination[2] = i1; + destination[1] = i2; + destination[0] = i3; + } + } + } + + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64.cs b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Helper/Base64Helper.cs similarity index 51% rename from src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64.cs rename to src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Helper/Base64Helper.cs index c362220d308049..b86600c74db27a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Helper/Base64Helper.cs @@ -3,11 +3,13 @@ using System.Diagnostics; using System.Runtime.CompilerServices; +#if NET using System.Runtime.Intrinsics; +#endif namespace System.Buffers.Text { - public static partial class Base64 + internal static partial class Base64Helper { [Conditional("DEBUG")] internal static unsafe void AssertRead(byte* src, byte* srcStart, int srcLength) @@ -67,45 +69,48 @@ internal static unsafe void AssertWrite(ushort* dest, ushort* destStart internal interface IBase64Encoder where T : unmanaged { - static abstract ReadOnlySpan EncodingMap { get; } - static abstract sbyte Avx2LutChar62 { get; } - static abstract sbyte Avx2LutChar63 { get; } - static abstract ReadOnlySpan AdvSimdLut4 { get; } - static abstract uint Ssse3AdvSimdLutE3 { get; } - static abstract int GetMaxSrcLength(int srcLength, int destLength); - static abstract int GetMaxEncodedLength(int srcLength); - static abstract uint GetInPlaceDestinationLength(int encodedLength, int leftOver); - static abstract unsafe void EncodeOneOptionallyPadTwo(byte* oneByte, T* dest, ref byte encodingMap); - static abstract unsafe void EncodeTwoOptionallyPadOne(byte* oneByte, T* dest, ref byte encodingMap); - static abstract unsafe void EncodeThreeAndWrite(byte* threeBytes, T* destination, ref byte encodingMap); - static abstract int IncrementPadTwo { get; } - static abstract int IncrementPadOne { get; } - static abstract unsafe void StoreVector512ToDestination(T* dest, T* destStart, int destLength, Vector512 str); - static abstract unsafe void StoreVector256ToDestination(T* dest, T* destStart, int destLength, Vector256 str); - static abstract unsafe void StoreVector128ToDestination(T* dest, T* destStart, int destLength, Vector128 str); - static abstract unsafe void StoreArmVector128x4ToDestination(T* dest, T* destStart, int destLength, Vector128 res1, + ReadOnlySpan EncodingMap { get; } + sbyte Avx2LutChar62 { get; } + sbyte Avx2LutChar63 { get; } + ReadOnlySpan AdvSimdLut4 { get; } + uint Ssse3AdvSimdLutE3 { get; } + int GetMaxSrcLength(int srcLength, int destLength); + int GetMaxEncodedLength(int srcLength); + uint GetInPlaceDestinationLength(int encodedLength, int leftOver); + unsafe void EncodeOneOptionallyPadTwo(byte* oneByte, T* dest, ref byte encodingMap); + unsafe void EncodeTwoOptionallyPadOne(byte* oneByte, T* dest, ref byte encodingMap); + unsafe void EncodeThreeAndWrite(byte* threeBytes, T* destination, ref byte encodingMap); + int IncrementPadTwo { get; } + int IncrementPadOne { get; } +#if NET + unsafe void StoreVector512ToDestination(T* dest, T* destStart, int destLength, Vector512 str); + unsafe void StoreVector256ToDestination(T* dest, T* destStart, int destLength, Vector256 str); + unsafe void StoreVector128ToDestination(T* dest, T* destStart, int destLength, Vector128 str); + unsafe void StoreArmVector128x4ToDestination(T* dest, T* destStart, int destLength, Vector128 res1, Vector128 res2, Vector128 res3, Vector128 res4); +#endif } internal interface IBase64Decoder where T : unmanaged { - static abstract ReadOnlySpan DecodingMap { get; } - static abstract ReadOnlySpan VbmiLookup0 { get; } - static abstract ReadOnlySpan VbmiLookup1 { get; } - static abstract ReadOnlySpan Avx2LutHigh { get; } - static abstract ReadOnlySpan Avx2LutLow { get; } - static abstract ReadOnlySpan Avx2LutShift { get; } - static abstract byte MaskSlashOrUnderscore { get; } - static abstract ReadOnlySpan Vector128LutHigh { get; } - static abstract ReadOnlySpan Vector128LutLow { get; } - static abstract ReadOnlySpan Vector128LutShift { get; } - static abstract ReadOnlySpan AdvSimdLutOne3 { get; } - static abstract uint AdvSimdLutTwo3Uint1 { get; } - static abstract int SrcLength(bool isFinalBlock, int sourceLength); - static abstract int GetMaxDecodedLength(int sourceLength); - static abstract bool IsInvalidLength(int bufferLength); - static abstract bool IsValidPadding(uint padChar); - static abstract bool TryDecode128Core( + ReadOnlySpan DecodingMap { get; } + ReadOnlySpan VbmiLookup0 { get; } + ReadOnlySpan VbmiLookup1 { get; } + ReadOnlySpan Avx2LutHigh { get; } + ReadOnlySpan Avx2LutLow { get; } + ReadOnlySpan Avx2LutShift { get; } + byte MaskSlashOrUnderscore { get; } + ReadOnlySpan Vector128LutHigh { get; } + ReadOnlySpan Vector128LutLow { get; } + ReadOnlySpan Vector128LutShift { get; } + ReadOnlySpan AdvSimdLutOne3 { get; } + uint AdvSimdLutTwo3Uint1 { get; } + int SrcLength(bool isFinalBlock, int sourceLength); + int GetMaxDecodedLength(int sourceLength); + bool IsInvalidLength(int bufferLength); + bool IsValidPadding(uint padChar); +#if NET + bool TryDecode128Core( Vector128 str, Vector128 hiNibbles, Vector128 maskSlashOrUnderscore, @@ -115,7 +120,7 @@ static abstract bool TryDecode128Core( Vector128 lutShift, Vector128 shiftForUnderscore, out Vector128 result); - static abstract bool TryDecode256Core( + bool TryDecode256Core( Vector256 str, Vector256 hiNibbles, Vector256 maskSlashOrUnderscore, @@ -124,17 +129,18 @@ static abstract bool TryDecode256Core( Vector256 lutShift, Vector256 shiftForUnderscore, out Vector256 result); - static abstract unsafe int DecodeFourElements(T* source, ref sbyte decodingMap); - static abstract unsafe int DecodeRemaining(T* srcEnd, ref sbyte decodingMap, long remaining, out uint t2, out uint t3); - static abstract int IndexOfAnyExceptWhiteSpace(ReadOnlySpan span); - static abstract OperationStatus DecodeWithWhiteSpaceBlockwiseWrapper(ReadOnlySpan source, + unsafe bool TryLoadVector512(T* src, T* srcStart, int sourceLength, out Vector512 str); + unsafe bool TryLoadAvxVector256(T* src, T* srcStart, int sourceLength, out Vector256 str); + unsafe bool TryLoadVector128(T* src, T* srcStart, int sourceLength, out Vector128 str); + unsafe bool TryLoadArmVector128x4(T* src, T* srcStart, int sourceLength, + out Vector128 str1, out Vector128 str2, out Vector128 str3, out Vector128 str4); +#endif + unsafe int DecodeFourElements(T* source, ref sbyte decodingMap); + unsafe int DecodeRemaining(T* srcEnd, ref sbyte decodingMap, long remaining, out uint t2, out uint t3); + int IndexOfAnyExceptWhiteSpace(ReadOnlySpan span); + OperationStatus DecodeWithWhiteSpaceBlockwiseWrapper(TTBase64Decoder decoder, ReadOnlySpan source, Span bytes, ref int bytesConsumed, ref int bytesWritten, bool isFinalBlock = true) where TTBase64Decoder : IBase64Decoder; - static abstract unsafe bool TryLoadVector512(T* src, T* srcStart, int sourceLength, out Vector512 str); - static abstract unsafe bool TryLoadAvxVector256(T* src, T* srcStart, int sourceLength, out Vector256 str); - static abstract unsafe bool TryLoadVector128(T* src, T* srcStart, int sourceLength, out Vector128 str); - static abstract unsafe bool TryLoadArmVector128x4(T* src, T* srcStart, int sourceLength, - out Vector128 str1, out Vector128 str2, out Vector128 str3, out Vector128 str4); } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Helper/Base64ValidatorHelper.cs b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Helper/Base64ValidatorHelper.cs new file mode 100644 index 00000000000000..0e26aa4995d706 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Helper/Base64ValidatorHelper.cs @@ -0,0 +1,186 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.CompilerServices; + +namespace System.Buffers.Text +{ + internal static partial class Base64Helper + { + internal static bool IsValid(TBase64Validatable validatable, ReadOnlySpan base64Text, out int decodedLength) + where TBase64Validatable : IBase64Validatable + { + int length = 0, paddingCount = 0; + + if (!base64Text.IsEmpty) + { +#if NET + while (true) + { + + int index = validatable.IndexOfAnyExcept(base64Text); + if ((uint)index >= (uint)base64Text.Length) + { + length += base64Text.Length; + break; + } + + length += index; + + T charToValidate = base64Text[index]; + base64Text = base64Text.Slice(index + 1); + + if (validatable.IsWhiteSpace(charToValidate)) + { + // It's common if there's whitespace for there to be multiple whitespace characters in a row, + // e.g. \r\n. Optimize for that case by looping here. + while (!base64Text.IsEmpty && validatable.IsWhiteSpace(base64Text[0])) + { + base64Text = base64Text.Slice(1); + } + continue; + } + + if (!validatable.IsEncodingPad(charToValidate)) + { + // Invalid char was found. + goto Fail; + } + + // Encoding pad found. Determine if padding is valid, then stop processing. + paddingCount = 1; + foreach (T charToValidateInPadding in base64Text) + { +#else + for (int i = 0; i < base64Text.Length; i++) + { + T charToValidate = base64Text[i]; + int value = validatable.DecodeValue(charToValidate); + if (value == -2) + { + // Not an Ascii char + goto Fail; + } + + if (value >= 0) // valid char + { + length++; + continue; + } + if (validatable.IsWhiteSpace(charToValidate)) + { + continue; + } + + if (!validatable.IsEncodingPad(charToValidate)) + { + // Invalid char was found. + goto Fail; + } + + // Encoding pad found. Determine if padding is valid, then stop processing. + paddingCount = 1; + for (i++; i < base64Text.Length; i++) + { + T charToValidateInPadding = base64Text[i]; +#endif + if (validatable.IsEncodingPad(charToValidateInPadding)) + { + // There can be at most 2 padding chars. + if (paddingCount >= 2) + { + goto Fail; + } + + paddingCount++; + } + else if (!validatable.IsWhiteSpace(charToValidateInPadding)) + { + // Invalid char was found. + goto Fail; + } + } + + length += paddingCount; + break; + } + + if (!validatable.ValidateAndDecodeLength(length, paddingCount, out decodedLength)) + { + goto Fail; + } + + return true; + } + + decodedLength = 0; + return true; + + Fail: + decodedLength = 0; + return false; + } + + internal interface IBase64Validatable + { +#if NET + int IndexOfAnyExcept(ReadOnlySpan span); +#else + int DecodeValue(T value); +#endif + bool IsWhiteSpace(T value); + bool IsEncodingPad(T value); + bool ValidateAndDecodeLength(int length, int paddingCount, out int decodedLength); + } + + internal readonly struct Base64CharValidatable : IBase64Validatable + { +#if NET + private static readonly SearchValues s_validBase64Chars = SearchValues.Create("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"); + + public int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExcept(s_validBase64Chars); +#else + public int DecodeValue(char value) + { + if (value > byte.MaxValue) + { + // Invalid char was found. + return -2; + } + + return default(Base64DecoderByte).DecodingMap[value]; + } +#endif + public bool IsWhiteSpace(char value) => Base64Helper.IsWhiteSpace(value); + public bool IsEncodingPad(char value) => value == EncodingPad; + public bool ValidateAndDecodeLength(int length, int paddingCount, out int decodedLength) => + default(Base64ByteValidatable).ValidateAndDecodeLength(length, paddingCount, out decodedLength); + } + + internal readonly struct Base64ByteValidatable : IBase64Validatable + { +#if NET + private static readonly SearchValues s_validBase64Chars = SearchValues.Create(default(Base64EncoderByte).EncodingMap); + + public int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExcept(s_validBase64Chars); +#else + public int DecodeValue(byte value) => default(Base64DecoderByte).DecodingMap[value]; +#endif + public bool IsWhiteSpace(byte value) => Base64Helper.IsWhiteSpace(value); + public bool IsEncodingPad(byte value) => value == EncodingPad; + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public bool ValidateAndDecodeLength(int length, int paddingCount, out int decodedLength) + { + if (length % 4 == 0) + { + // Remove padding to get exact length. + decodedLength = (int)((uint)length / 4 * 3) - paddingCount; + return true; + } + + decodedLength = 0; + return false; + } + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Url/Base64UrlDecoder.cs b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Url/Base64UrlDecoder.cs index 65ba4624be53d9..ac03ef5847e39a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Url/Base64UrlDecoder.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Url/Base64UrlDecoder.cs @@ -4,11 +4,13 @@ using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +#if NET using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; +#endif using System.Text; -using static System.Buffers.Text.Base64; +using static System.Buffers.Text.Base64Helper; namespace System.Buffers.Text { @@ -25,11 +27,22 @@ public static partial class Base64Url /// public static int GetMaxDecodedLength(int base64Length) { +#if NET ArgumentOutOfRangeException.ThrowIfNegative(base64Length); (uint whole, uint remainder) = uint.DivRem((uint)base64Length, 4); return (int)(whole * 3 + (remainder > 0 ? remainder - 1 : 0)); +#else + if (base64Length < 0) + { + throw new ArgumentOutOfRangeException(nameof(base64Length)); + } + + int remainder = (int)((uint)base64Length % 4); + + return (base64Length >> 2) * 3 + (remainder > 0 ? remainder - 1 : 0); +#endif } /// @@ -50,7 +63,7 @@ public static int GetMaxDecodedLength(int base64Length) /// - Remainder of 1 byte - will cause OperationStatus.InvalidData result. /// public static OperationStatus DecodeFromUtf8(ReadOnlySpan source, Span destination, out int bytesConsumed, out int bytesWritten, bool isFinalBlock = true) => - DecodeFrom(source, destination, out bytesConsumed, out bytesWritten, isFinalBlock, ignoreWhiteSpace: true); + DecodeFrom(default(Base64UrlDecoderByte), source, destination, out bytesConsumed, out bytesWritten, isFinalBlock, ignoreWhiteSpace: true); /// /// Decodes the span of UTF-8 encoded text in Base64Url into binary data, in-place. @@ -69,7 +82,7 @@ public static OperationStatus DecodeFromUtf8(ReadOnlySpan source, Span public static int DecodeFromUtf8InPlace(Span buffer) { - OperationStatus status = DecodeFromUtf8InPlace(buffer, out int bytesWritten, ignoreWhiteSpace: true); + OperationStatus status = DecodeFromUtf8InPlace(default, buffer, out int bytesWritten, ignoreWhiteSpace: true); // Base64.DecodeFromUtf8InPlace returns OperationStatus, therefore doesn't throw. // For Base64Url, this is not an OperationStatus API and thus throws. @@ -185,9 +198,11 @@ public static byte[] DecodeFromUtf8(ReadOnlySpan source) /// public static OperationStatus DecodeFromChars(ReadOnlySpan source, Span destination, out int charsConsumed, out int bytesWritten, bool isFinalBlock = true) => - DecodeFrom(MemoryMarshal.Cast(source), destination, out charsConsumed, out bytesWritten, isFinalBlock, ignoreWhiteSpace: true); + DecodeFrom(default(Base64UrlDecoderChar), MemoryMarshal.Cast(source), destination, + out charsConsumed, out bytesWritten, isFinalBlock, ignoreWhiteSpace: true); - private static OperationStatus DecodeWithWhiteSpaceBlockwise(ReadOnlySpan source, Span bytes, ref int bytesConsumed, ref int bytesWritten, bool isFinalBlock = true) + private static OperationStatus DecodeWithWhiteSpaceBlockwise(TBase64Decoder decoder, + ReadOnlySpan source, Span bytes, ref int bytesConsumed, ref int bytesWritten, bool isFinalBlock = true) where TBase64Decoder : IBase64Decoder { const int BlockSize = 4; @@ -221,13 +236,23 @@ private static OperationStatus DecodeWithWhiteSpaceBlockwise(Rea continue; } - bool hasAnotherBlock = source.Length >= BlockSize && bufferIdx == BlockSize; + bool hasAnotherBlock; + + if (decoder is Base64DecoderByte) + { + hasAnotherBlock = source.Length >= BlockSize; + } + else + { + hasAnotherBlock = source.Length > 1; + } + bool localIsFinalBlock = !hasAnotherBlock; // If this block contains padding and there's another block, then only whitespace may follow for being valid. if (hasAnotherBlock) { - int paddingCount = GetPaddingCount(ref buffer[^1]); + int paddingCount = GetPaddingCount(decoder, ref buffer[BlockSize - 1]); if (paddingCount > 0) { hasAnotherBlock = false; @@ -240,7 +265,7 @@ private static OperationStatus DecodeWithWhiteSpaceBlockwise(Rea localIsFinalBlock = false; } - status = DecodeFrom(buffer.Slice(0, bufferIdx), bytes, out int localConsumed, out int localWritten, localIsFinalBlock, ignoreWhiteSpace: false); + status = DecodeFrom(decoder, buffer.Slice(0, bufferIdx), bytes, out int localConsumed, out int localWritten, localIsFinalBlock, ignoreWhiteSpace: false); bytesConsumed += localConsumed; bytesWritten += localWritten; @@ -276,17 +301,17 @@ private static OperationStatus DecodeWithWhiteSpaceBlockwise(Rea } [MethodImpl(MethodImplOptions.AggressiveInlining)] - private static int GetPaddingCount(ref ushort ptrToLastElement) + private static int GetPaddingCount(TBase64Decoder decoder, ref ushort ptrToLastElement) where TBase64Decoder : IBase64Decoder { int padding = 0; - if (TBase64Decoder.IsValidPadding(ptrToLastElement)) + if (decoder.IsValidPadding(ptrToLastElement)) { padding++; } - if (TBase64Decoder.IsValidPadding(Unsafe.Subtract(ref ptrToLastElement, 1))) + if (decoder.IsValidPadding(Unsafe.Subtract(ref ptrToLastElement, 1))) { padding++; } @@ -371,7 +396,7 @@ public static byte[] DecodeFromChars(ReadOnlySpan source) private readonly struct Base64UrlDecoderByte : IBase64Decoder { - public static ReadOnlySpan DecodingMap => + public ReadOnlySpan DecodingMap => [ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, @@ -391,7 +416,7 @@ public static byte[] DecodeFromChars(ReadOnlySpan source) -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, ]; - public static ReadOnlySpan VbmiLookup0 => + public ReadOnlySpan VbmiLookup0 => [ 0x80808080, 0x80808080, 0x80808080, 0x80808080, 0x80808080, 0x80808080, 0x80808080, 0x80808080, @@ -399,7 +424,7 @@ public static byte[] DecodeFromChars(ReadOnlySpan source) 0x37363534, 0x3b3a3938, 0x80803d3c, 0x80808080 ]; - public static ReadOnlySpan VbmiLookup1 => + public ReadOnlySpan VbmiLookup1 => [ 0x02010080, 0x06050403, 0x0a090807, 0x0e0d0c0b, 0x1211100f, 0x16151413, 0x80191817, 0x3f808080, @@ -407,7 +432,7 @@ public static byte[] DecodeFromChars(ReadOnlySpan source) 0x2c2b2a29, 0x302f2e2d, 0x80333231, 0x80808080 ]; - public static ReadOnlySpan Avx2LutHigh => + public ReadOnlySpan Avx2LutHigh => [ 0x00, 0x00, 0x2d, 0x39, 0x4f, 0x5a, 0x6f, 0x7a, @@ -419,7 +444,7 @@ public static byte[] DecodeFromChars(ReadOnlySpan source) 0x00, 0x00, 0x00, 0x00 ]; - public static ReadOnlySpan Avx2LutLow => + public ReadOnlySpan Avx2LutLow => [ 0x01, 0x01, 0x2d, 0x30, 0x41, 0x50, 0x61, 0x70, @@ -431,7 +456,7 @@ public static byte[] DecodeFromChars(ReadOnlySpan source) 0x01, 0x01, 0x01, 0x01 ]; - public static ReadOnlySpan Avx2LutShift => + public ReadOnlySpan Avx2LutShift => [ 0, 0, 17, 4, -65, -65, -71, -71, @@ -443,30 +468,31 @@ public static byte[] DecodeFromChars(ReadOnlySpan source) 0, 0, 0, 0 ]; - public static byte MaskSlashOrUnderscore => (byte)'_'; // underscore + public byte MaskSlashOrUnderscore => (byte)'_'; // underscore - public static ReadOnlySpan Vector128LutHigh => [0x392d0000, 0x7a6f5a4f, 0x00000000, 0x00000000]; + public ReadOnlySpan Vector128LutHigh => [0x392d0000, 0x7a6f5a4f, 0x00000000, 0x00000000]; - public static ReadOnlySpan Vector128LutLow => [0x302d0101, 0x70615041, 0x01010101, 0x01010101]; + public ReadOnlySpan Vector128LutLow => [0x302d0101, 0x70615041, 0x01010101, 0x01010101]; - public static ReadOnlySpan Vector128LutShift => [0x04110000, 0xb9b9bfbf, 0x00000000, 0x00000000]; + public ReadOnlySpan Vector128LutShift => [0x04110000, 0xb9b9bfbf, 0x00000000, 0x00000000]; - public static ReadOnlySpan AdvSimdLutOne3 => [0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFF3EFF]; + public ReadOnlySpan AdvSimdLutOne3 => [0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFF3EFF]; - public static uint AdvSimdLutTwo3Uint1 => 0x1B1AFF3F; + public uint AdvSimdLutTwo3Uint1 => 0x1B1AFF3F; - public static int GetMaxDecodedLength(int sourceLength) => Base64Url.GetMaxDecodedLength(sourceLength); + public int GetMaxDecodedLength(int sourceLength) => Base64Url.GetMaxDecodedLength(sourceLength); - public static bool IsInvalidLength(int bufferLength) => (bufferLength & 3) == 1; // One byte cannot be decoded completely + public bool IsInvalidLength(int bufferLength) => (bufferLength & 3) == 1; // One byte cannot be decoded completely - public static bool IsValidPadding(uint padChar) => padChar == EncodingPad || padChar == UrlEncodingPad; + public bool IsValidPadding(uint padChar) => padChar is EncodingPad or UrlEncodingPad; - public static int SrcLength(bool isFinalBlock, int sourceLength) => isFinalBlock ? sourceLength : sourceLength & ~0x3; + public int SrcLength(bool isFinalBlock, int sourceLength) => isFinalBlock ? sourceLength : sourceLength & ~0x3; +#if NET [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] [CompExactlyDependsOn(typeof(Ssse3))] - public static bool TryDecode128Core( + public bool TryDecode128Core( Vector128 str, Vector128 hiNibbles, Vector128 maskSlashOrUnderscore, @@ -502,7 +528,7 @@ public static bool TryDecode128Core( [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Avx2))] - public static bool TryDecode256Core( + public bool TryDecode256Core( Vector256 str, Vector256 hiNibbles, Vector256 maskSlashOrUnderscore, @@ -536,90 +562,167 @@ public static bool TryDecode256Core( } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe int DecodeFourElements(byte* source, ref sbyte decodingMap) => - Base64DecoderByte.DecodeFourElements(source, ref decodingMap); + public unsafe bool TryLoadVector512(byte* src, byte* srcStart, int sourceLength, out Vector512 str) => + default(Base64DecoderByte).TryLoadVector512(src, srcStart, sourceLength, out str); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe int DecodeRemaining(byte* srcEnd, ref sbyte decodingMap, long remaining, out uint t2, out uint t3) - => Base64DecoderByte.DecodeRemaining(srcEnd, ref decodingMap, remaining, out t2, out t3); + [CompExactlyDependsOn(typeof(Avx2))] + public unsafe bool TryLoadAvxVector256(byte* src, byte* srcStart, int sourceLength, out Vector256 str) => + default(Base64DecoderByte).TryLoadAvxVector256(src, srcStart, sourceLength, out str); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int IndexOfAnyExceptWhiteSpace(ReadOnlySpan span) => Base64DecoderByte.IndexOfAnyExceptWhiteSpace(span); + public unsafe bool TryLoadVector128(byte* src, byte* srcStart, int sourceLength, out Vector128 str) => + default(Base64DecoderByte).TryLoadVector128(src, srcStart, sourceLength, out str); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static OperationStatus DecodeWithWhiteSpaceBlockwiseWrapper(ReadOnlySpan utf8, Span bytes, - ref int bytesConsumed, ref int bytesWritten, bool isFinalBlock = true) where TBase64Decoder : IBase64Decoder => - Base64.DecodeWithWhiteSpaceBlockwise(utf8, bytes, ref bytesConsumed, ref bytesWritten, isFinalBlock); + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + public unsafe bool TryLoadArmVector128x4(byte* src, byte* srcStart, int sourceLength, + out Vector128 str1, out Vector128 str2, out Vector128 str3, out Vector128 str4) => + default(Base64DecoderByte).TryLoadArmVector128x4(src, srcStart, sourceLength, out str1, out str2, out str3, out str4); +#endif [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe bool TryLoadVector512(byte* src, byte* srcStart, int sourceLength, out Vector512 str) => - Base64DecoderByte.TryLoadVector512(src, srcStart, sourceLength, out str); + public unsafe int DecodeFourElements(byte* source, ref sbyte decodingMap) => + default(Base64DecoderByte).DecodeFourElements(source, ref decodingMap); [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Avx2))] - public static unsafe bool TryLoadAvxVector256(byte* src, byte* srcStart, int sourceLength, out Vector256 str) => - Base64DecoderByte.TryLoadAvxVector256(src, srcStart, sourceLength, out str); + public unsafe int DecodeRemaining(byte* srcEnd, ref sbyte decodingMap, long remaining, out uint t2, out uint t3) => + default(Base64DecoderByte).DecodeRemaining(srcEnd, ref decodingMap, remaining, out t2, out t3); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe bool TryLoadVector128(byte* src, byte* srcStart, int sourceLength, out Vector128 str) => - Base64DecoderByte.TryLoadVector128(src, srcStart, sourceLength, out str); + public int IndexOfAnyExceptWhiteSpace(ReadOnlySpan span) => default(Base64DecoderByte).IndexOfAnyExceptWhiteSpace(span); [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] - public static unsafe bool TryLoadArmVector128x4(byte* src, byte* srcStart, int sourceLength, - out Vector128 str1, out Vector128 str2, out Vector128 str3, out Vector128 str4) => - Base64DecoderByte.TryLoadArmVector128x4(src, srcStart, sourceLength, out str1, out str2, out str3, out str4); + public OperationStatus DecodeWithWhiteSpaceBlockwiseWrapper(TBase64Decoder decoder, ReadOnlySpan utf8, Span bytes, + ref int bytesConsumed, ref int bytesWritten, bool isFinalBlock = true) where TBase64Decoder : IBase64Decoder => + Base64Helper.DecodeWithWhiteSpaceBlockwise(decoder, utf8, bytes, ref bytesConsumed, ref bytesWritten, isFinalBlock); } private readonly struct Base64UrlDecoderChar : IBase64Decoder { - public static ReadOnlySpan DecodingMap => Base64UrlDecoderByte.DecodingMap; + public ReadOnlySpan DecodingMap => default(Base64UrlDecoderByte).DecodingMap; - public static ReadOnlySpan VbmiLookup0 => Base64UrlDecoderByte.VbmiLookup0; + public ReadOnlySpan VbmiLookup0 => default(Base64UrlDecoderByte).VbmiLookup0; - public static ReadOnlySpan VbmiLookup1 => Base64UrlDecoderByte.VbmiLookup1; + public ReadOnlySpan VbmiLookup1 => default(Base64UrlDecoderByte).VbmiLookup1; - public static ReadOnlySpan Avx2LutHigh => Base64UrlDecoderByte.Avx2LutHigh; + public ReadOnlySpan Avx2LutHigh => default(Base64UrlDecoderByte).Avx2LutHigh; - public static ReadOnlySpan Avx2LutLow => Base64UrlDecoderByte.Avx2LutLow; + public ReadOnlySpan Avx2LutLow => default(Base64UrlDecoderByte).Avx2LutLow; - public static ReadOnlySpan Avx2LutShift => Base64UrlDecoderByte.Avx2LutShift; + public ReadOnlySpan Avx2LutShift => default(Base64UrlDecoderByte).Avx2LutShift; - public static byte MaskSlashOrUnderscore => Base64UrlDecoderByte.MaskSlashOrUnderscore; + public byte MaskSlashOrUnderscore => default(Base64UrlDecoderByte).MaskSlashOrUnderscore; - public static ReadOnlySpan Vector128LutHigh => Base64UrlDecoderByte.Vector128LutHigh; + public ReadOnlySpan Vector128LutHigh => default(Base64UrlDecoderByte).Vector128LutHigh; - public static ReadOnlySpan Vector128LutLow => Base64UrlDecoderByte.Vector128LutLow; + public ReadOnlySpan Vector128LutLow => default(Base64UrlDecoderByte).Vector128LutLow; - public static ReadOnlySpan Vector128LutShift => Base64UrlDecoderByte.Vector128LutShift; + public ReadOnlySpan Vector128LutShift => default(Base64UrlDecoderByte).Vector128LutShift; - public static ReadOnlySpan AdvSimdLutOne3 => Base64UrlDecoderByte.AdvSimdLutOne3; + public ReadOnlySpan AdvSimdLutOne3 => default(Base64UrlDecoderByte).AdvSimdLutOne3; - public static uint AdvSimdLutTwo3Uint1 => Base64UrlDecoderByte.AdvSimdLutTwo3Uint1; + public uint AdvSimdLutTwo3Uint1 => default(Base64UrlDecoderByte).AdvSimdLutTwo3Uint1; - public static int GetMaxDecodedLength(int sourceLength) => Base64UrlDecoderByte.GetMaxDecodedLength(sourceLength); + public int GetMaxDecodedLength(int sourceLength) => default(Base64UrlDecoderByte).GetMaxDecodedLength(sourceLength); - public static bool IsInvalidLength(int bufferLength) => Base64DecoderByte.IsInvalidLength(bufferLength); + public bool IsInvalidLength(int bufferLength) => default(Base64UrlDecoderByte).IsInvalidLength(bufferLength); - public static bool IsValidPadding(uint padChar) => Base64UrlDecoderByte.IsValidPadding(padChar); + public bool IsValidPadding(uint padChar) => default(Base64UrlDecoderByte).IsValidPadding(padChar); - public static int SrcLength(bool isFinalBlock, int sourceLength) => Base64UrlDecoderByte.SrcLength(isFinalBlock, sourceLength); + public int SrcLength(bool isFinalBlock, int sourceLength) => default(Base64UrlDecoderByte).SrcLength(isFinalBlock, sourceLength); +#if NET [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] [CompExactlyDependsOn(typeof(Ssse3))] - public static bool TryDecode128Core(Vector128 str, Vector128 hiNibbles, Vector128 maskSlashOrUnderscore, Vector128 mask8F, + public bool TryDecode128Core(Vector128 str, Vector128 hiNibbles, Vector128 maskSlashOrUnderscore, Vector128 mask8F, Vector128 lutLow, Vector128 lutHigh, Vector128 lutShift, Vector128 shiftForUnderscore, out Vector128 result) => - Base64UrlDecoderByte.TryDecode128Core(str, hiNibbles, maskSlashOrUnderscore, mask8F, lutLow, lutHigh, lutShift, shiftForUnderscore, out result); + default(Base64UrlDecoderByte).TryDecode128Core(str, hiNibbles, maskSlashOrUnderscore, mask8F, lutLow, lutHigh, lutShift, shiftForUnderscore, out result); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Avx2))] - public static bool TryDecode256Core(Vector256 str, Vector256 hiNibbles, Vector256 maskSlashOrUnderscore, Vector256 lutLow, + public bool TryDecode256Core(Vector256 str, Vector256 hiNibbles, Vector256 maskSlashOrUnderscore, Vector256 lutLow, Vector256 lutHigh, Vector256 lutShift, Vector256 shiftForUnderscore, out Vector256 result) => - Base64UrlDecoderByte.TryDecode256Core(str, hiNibbles, maskSlashOrUnderscore, lutLow, lutHigh, lutShift, shiftForUnderscore, out result); + default(Base64UrlDecoderByte).TryDecode256Core(str, hiNibbles, maskSlashOrUnderscore, lutLow, lutHigh, lutShift, shiftForUnderscore, out result); + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public unsafe bool TryLoadVector512(ushort* src, ushort* srcStart, int sourceLength, out Vector512 str) + { + AssertRead>(src, srcStart, sourceLength); + Vector512 utf16VectorLower = Vector512.Load(src); + Vector512 utf16VectorUpper = Vector512.Load(src + 32); + + if (Ascii.VectorContainsNonAsciiChar(utf16VectorLower | utf16VectorUpper)) + { + str = default; + return false; + } + + str = Vector512.Narrow(utf16VectorLower, utf16VectorUpper).AsSByte(); + return true; + } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe int DecodeFourElements(ushort* source, ref sbyte decodingMap) + [CompExactlyDependsOn(typeof(Avx2))] + public unsafe bool TryLoadAvxVector256(ushort* src, ushort* srcStart, int sourceLength, out Vector256 str) + { + AssertRead>(src, srcStart, sourceLength); + Vector256 utf16VectorLower = Avx.LoadVector256(src); + Vector256 utf16VectorUpper = Avx.LoadVector256(src + 16); + + if (Ascii.VectorContainsNonAsciiChar(utf16VectorLower | utf16VectorUpper)) + { + str = default; + return false; + } + + str = Vector256.Narrow(utf16VectorLower, utf16VectorUpper).AsSByte(); + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public unsafe bool TryLoadVector128(ushort* src, ushort* srcStart, int sourceLength, out Vector128 str) + { + AssertRead>(src, srcStart, sourceLength); + Vector128 utf16VectorLower = Vector128.LoadUnsafe(ref *src); + Vector128 utf16VectorUpper = Vector128.LoadUnsafe(ref *src, 8); + if (Ascii.VectorContainsNonAsciiChar(utf16VectorLower | utf16VectorUpper)) + { + str = default; + return false; + } + + str = Ascii.ExtractAsciiVector(utf16VectorLower, utf16VectorUpper); + return true; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] + public unsafe bool TryLoadArmVector128x4(ushort* src, ushort* srcStart, int sourceLength, + out Vector128 str1, out Vector128 str2, out Vector128 str3, out Vector128 str4) + { + AssertRead>(src, srcStart, sourceLength); + var (s11, s12, s21, s22) = AdvSimd.Arm64.Load4xVector128AndUnzip(src); + var (s31, s32, s41, s42) = AdvSimd.Arm64.Load4xVector128AndUnzip(src + 32); + + if (Ascii.VectorContainsNonAsciiChar(s11 | s12 | s21 | s22 | s31 | s32 | s41 | s42)) + { + str1 = str2 = str3 = str4 = default; + return false; + } + + str1 = Ascii.ExtractAsciiVector(s11, s31); + str2 = Ascii.ExtractAsciiVector(s12, s32); + str3 = Ascii.ExtractAsciiVector(s21, s41); + str4 = Ascii.ExtractAsciiVector(s22, s42); + + return true; + } +#endif + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + public unsafe int DecodeFourElements(ushort* source, ref sbyte decodingMap) { // The 'source' span expected to have at least 4 elements, and the 'decodingMap' consists 256 sbytes uint t0 = source[0]; @@ -632,10 +735,10 @@ public static unsafe int DecodeFourElements(ushort* source, ref sbyte decodingMa return -1; // One or more chars falls outside the 00..ff range, invalid Base64Url character. } - int i0 = Unsafe.Add(ref decodingMap, t0); - int i1 = Unsafe.Add(ref decodingMap, t1); - int i2 = Unsafe.Add(ref decodingMap, t2); - int i3 = Unsafe.Add(ref decodingMap, t3); + int i0 = Unsafe.Add(ref decodingMap, (int)t0); + int i1 = Unsafe.Add(ref decodingMap, (int)t1); + int i2 = Unsafe.Add(ref decodingMap, (int)t2); + int i3 = Unsafe.Add(ref decodingMap, (int)t3); i0 <<= 18; i1 <<= 12; @@ -649,7 +752,7 @@ public static unsafe int DecodeFourElements(ushort* source, ref sbyte decodingMa } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe int DecodeRemaining(ushort* srcEnd, ref sbyte decodingMap, long remaining, out uint t2, out uint t3) + public unsafe int DecodeRemaining(ushort* srcEnd, ref sbyte decodingMap, long remaining, out uint t2, out uint t3) { uint t0; uint t1; @@ -692,7 +795,7 @@ public static unsafe int DecodeRemaining(ushort* srcEnd, ref sbyte decodingMap, } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static int IndexOfAnyExceptWhiteSpace(ReadOnlySpan span) + public int IndexOfAnyExceptWhiteSpace(ReadOnlySpan span) { for (int i = 0; i < span.Length; i++) { @@ -706,83 +809,9 @@ public static int IndexOfAnyExceptWhiteSpace(ReadOnlySpan span) } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static OperationStatus DecodeWithWhiteSpaceBlockwiseWrapper(ReadOnlySpan source, Span bytes, + public OperationStatus DecodeWithWhiteSpaceBlockwiseWrapper(TBase64Decoder decoder, ReadOnlySpan source, Span bytes, ref int bytesConsumed, ref int bytesWritten, bool isFinalBlock = true) where TBase64Decoder : IBase64Decoder => - DecodeWithWhiteSpaceBlockwise(source, bytes, ref bytesConsumed, ref bytesWritten, isFinalBlock); - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe bool TryLoadVector512(ushort* src, ushort* srcStart, int sourceLength, out Vector512 str) - { - AssertRead>(src, srcStart, sourceLength); - Vector512 utf16VectorLower = Vector512.Load(src); - Vector512 utf16VectorUpper = Vector512.Load(src + 32); - - if (Ascii.VectorContainsNonAsciiChar(utf16VectorLower | utf16VectorUpper)) - { - str = default; - return false; - } - - str = Vector512.Narrow(utf16VectorLower, utf16VectorUpper).AsSByte(); - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(Avx2))] - public static unsafe bool TryLoadAvxVector256(ushort* src, ushort* srcStart, int sourceLength, out Vector256 str) - { - AssertRead>(src, srcStart, sourceLength); - Vector256 utf16VectorLower = Avx.LoadVector256(src); - Vector256 utf16VectorUpper = Avx.LoadVector256(src + 16); - - if (Ascii.VectorContainsNonAsciiChar(utf16VectorLower | utf16VectorUpper)) - { - str = default; - return false; - } - - str = Vector256.Narrow(utf16VectorLower, utf16VectorUpper).AsSByte(); - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe bool TryLoadVector128(ushort* src, ushort* srcStart, int sourceLength, out Vector128 str) - { - AssertRead>(src, srcStart, sourceLength); - Vector128 utf16VectorLower = Vector128.LoadUnsafe(ref *src); - Vector128 utf16VectorUpper = Vector128.LoadUnsafe(ref *src, 8); - if (Ascii.VectorContainsNonAsciiChar(utf16VectorLower | utf16VectorUpper)) - { - str = default; - return false; - } - - str = Ascii.ExtractAsciiVector(utf16VectorLower, utf16VectorUpper); - return true; - } - - [MethodImpl(MethodImplOptions.AggressiveInlining)] - [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] - public static unsafe bool TryLoadArmVector128x4(ushort* src, ushort* srcStart, int sourceLength, - out Vector128 str1, out Vector128 str2, out Vector128 str3, out Vector128 str4) - { - AssertRead> (src, srcStart, sourceLength); - var (s11, s12, s21, s22) = AdvSimd.Arm64.Load4xVector128AndUnzip(src); - var (s31, s32, s41, s42) = AdvSimd.Arm64.Load4xVector128AndUnzip(src + 32); - - if (Ascii.VectorContainsNonAsciiChar(s11 | s12 | s21 | s22 | s31 | s32 | s41 | s42)) - { - str1 = str2 = str3 = str4 = default; - return false; - } - - str1 = Ascii.ExtractAsciiVector(s11, s31); - str2 = Ascii.ExtractAsciiVector(s12, s32); - str3 = Ascii.ExtractAsciiVector(s21, s41); - str4 = Ascii.ExtractAsciiVector(s22, s42); - - return true; - } + DecodeWithWhiteSpaceBlockwise(decoder, source, bytes, ref bytesConsumed, ref bytesWritten, isFinalBlock); } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Url/Base64UrlEncoder.cs b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Url/Base64UrlEncoder.cs index c23b2b3578d0ae..d3185b37ca799b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Url/Base64UrlEncoder.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Url/Base64UrlEncoder.cs @@ -4,10 +4,12 @@ using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; +#if NET using System.Runtime.Intrinsics; using System.Runtime.Intrinsics.Arm; using System.Runtime.Intrinsics.X86; -using static System.Buffers.Text.Base64; +#endif +using static System.Buffers.Text.Base64Helper; namespace System.Buffers.Text { @@ -24,9 +26,9 @@ public static partial class Base64Url /// such as when calling in a loop, subsequent calls with should end with call. The default is . /// One of the enumeration values that indicates the success or failure of the operation. /// This implementation of the base64url encoding omits the optional padding characters. - public static unsafe OperationStatus EncodeToUtf8(ReadOnlySpan source, + public static OperationStatus EncodeToUtf8(ReadOnlySpan source, Span destination, out int bytesConsumed, out int bytesWritten, bool isFinalBlock = true) => - EncodeTo(source, destination, out bytesConsumed, out bytesWritten, isFinalBlock); + EncodeTo(default(Base64UrlEncoderByte), source, destination, out bytesConsumed, out bytesWritten, isFinalBlock); /// /// Returns the length (in bytes) of the result if you were to encode binary data within a byte span of size . @@ -36,11 +38,22 @@ public static unsafe OperationStatus EncodeToUtf8(ReadOnlySpan source, /// public static int GetEncodedLength(int bytesLength) { - ArgumentOutOfRangeException.ThrowIfGreaterThan((uint)bytesLength, Base64.MaximumEncodeLength); +#if NET + ArgumentOutOfRangeException.ThrowIfGreaterThan((uint)bytesLength, MaximumEncodeLength); (uint whole, uint remainder) = uint.DivRem((uint)bytesLength, 3); return (int)(whole * 4 + (remainder > 0 ? remainder + 1 : 0)); // if remainder is 1 or 2, the encoded length will be 1 byte longer. +#else + if ((uint)bytesLength > MaximumEncodeLength) + { + throw new ArgumentOutOfRangeException(nameof(bytesLength)); + } + + int remainder = (int)((uint)bytesLength % 3); + + return (bytesLength / 3) * 4 + (remainder > 0 ? remainder + 1 : 0); +#endif } /// @@ -92,7 +105,7 @@ public static byte[] EncodeToUtf8(ReadOnlySpan source) /// This implementation of the base64url encoding omits the optional padding characters. public static OperationStatus EncodeToChars(ReadOnlySpan source, Span destination, out int bytesConsumed, out int charsWritten, bool isFinalBlock = true) => - EncodeTo(source, MemoryMarshal.Cast(destination), out bytesConsumed, out charsWritten, isFinalBlock); + EncodeTo(default(Base64UrlEncoderChar), source, MemoryMarshal.Cast(destination), out bytesConsumed, out charsWritten, isFinalBlock); /// /// Encodes the span of binary data into unicode ASCII chars represented as Base64Url. @@ -138,6 +151,7 @@ public static char[] EncodeToChars(ReadOnlySpan source) /// This implementation of the base64url encoding omits the optional padding characters. public static unsafe string EncodeToString(ReadOnlySpan source) { +#if NET int encodedLength = GetEncodedLength(source.Length); #pragma warning disable CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type @@ -148,6 +162,13 @@ public static unsafe string EncodeToString(ReadOnlySpan source) Debug.Assert(buffer.Length == charsWritten, $"The source length: {source.Length}, bytes written: {charsWritten}"); }); #pragma warning restore CS8500 // This takes the address of, gets the size of, or declares a pointer to a managed type +#else + char[] destination = new char[GetEncodedLength(source.Length)]; + EncodeToChars(source, destination, out _, out int charsWritten); + Debug.Assert(destination.Length == charsWritten); + + return new string(destination); +#endif } /// @@ -191,40 +212,40 @@ public static bool TryEncodeToUtf8(ReadOnlySpan source, Span destina /// When this method returns, contains the number of bytes written into the buffer. This parameter is treated as uninitialized. /// if bytes encoded successfully, otherwise . /// This implementation of the base64url encoding omits the optional padding characters. - public static unsafe bool TryEncodeToUtf8InPlace(Span buffer, int dataLength, out int bytesWritten) + public static bool TryEncodeToUtf8InPlace(Span buffer, int dataLength, out int bytesWritten) { - OperationStatus status = EncodeToUtf8InPlace(buffer, dataLength, out bytesWritten); + OperationStatus status = EncodeToUtf8InPlace(default(Base64UrlEncoderByte), buffer, dataLength, out bytesWritten); return status == OperationStatus.Done; } - private readonly struct Base64UrlEncoderByte : IBase64Encoder + private readonly struct Base64UrlEncoderByte : Base64Helper.IBase64Encoder { - public static ReadOnlySpan EncodingMap => "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"u8; + public ReadOnlySpan EncodingMap => "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"u8; - public static sbyte Avx2LutChar62 => -17; // char '-' diff + public sbyte Avx2LutChar62 => -17; // char '-' diff - public static sbyte Avx2LutChar63 => 32; // char '_' diff + public sbyte Avx2LutChar63 => 32; // char '_' diff - public static ReadOnlySpan AdvSimdLut4 => "wxyz0123456789-_"u8; + public ReadOnlySpan AdvSimdLut4 => "wxyz0123456789-_"u8; - public static uint Ssse3AdvSimdLutE3 => 0x000020EF; + public uint Ssse3AdvSimdLutE3 => 0x000020EF; - public static int IncrementPadTwo => 2; + public int IncrementPadTwo => 2; - public static int IncrementPadOne => 3; + public int IncrementPadOne => 3; - public static int GetMaxSrcLength(int srcLength, int destLength) => + public int GetMaxSrcLength(int srcLength, int destLength) => srcLength <= MaximumEncodeLength && destLength >= GetEncodedLength(srcLength) ? srcLength : GetMaxDecodedLength(destLength); - public static uint GetInPlaceDestinationLength(int encodedLength, int leftOver) => + public uint GetInPlaceDestinationLength(int encodedLength, int leftOver) => leftOver > 0 ? (uint)(encodedLength - leftOver - 1) : (uint)(encodedLength - 4); - public static int GetMaxEncodedLength(int srcLength) => GetEncodedLength(srcLength); + public int GetMaxEncodedLength(int srcLength) => GetEncodedLength(srcLength); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void EncodeOneOptionallyPadTwo(byte* oneByte, byte* dest, ref byte encodingMap) + public unsafe void EncodeOneOptionallyPadTwo(byte* oneByte, byte* dest, ref byte encodingMap) { uint t0 = oneByte[0]; @@ -246,7 +267,7 @@ public static unsafe void EncodeOneOptionallyPadTwo(byte* oneByte, byte* dest, r } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void EncodeTwoOptionallyPadOne(byte* twoBytes, byte* dest, ref byte encodingMap) + public unsafe void EncodeTwoOptionallyPadOne(byte* twoBytes, byte* dest, ref byte encodingMap) { uint t0 = twoBytes[0]; uint t1 = twoBytes[1]; @@ -271,55 +292,57 @@ public static unsafe void EncodeTwoOptionallyPadOne(byte* twoBytes, byte* dest, } } +#if NET [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void StoreVector512ToDestination(byte* dest, byte* destStart, int destLength, Vector512 str) => - Base64EncoderByte.StoreVector512ToDestination(dest, destStart, destLength, str); + public unsafe void StoreVector512ToDestination(byte* dest, byte* destStart, int destLength, Vector512 str) => + default(Base64EncoderByte).StoreVector512ToDestination(dest, destStart, destLength, str); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(Avx2))] - public static unsafe void StoreVector256ToDestination(byte* dest, byte* destStart, int destLength, Vector256 str) => - Base64EncoderByte.StoreVector256ToDestination(dest, destStart, destLength, str); + public unsafe void StoreVector256ToDestination(byte* dest, byte* destStart, int destLength, Vector256 str) => + default(Base64EncoderByte).StoreVector256ToDestination(dest, destStart, destLength, str); [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void StoreVector128ToDestination(byte* dest, byte* destStart, int destLength, Vector128 str) => - Base64EncoderByte.StoreVector128ToDestination(dest, destStart, destLength, str); + public unsafe void StoreVector128ToDestination(byte* dest, byte* destStart, int destLength, Vector128 str) => + default(Base64EncoderByte).StoreVector128ToDestination(dest, destStart, destLength, str); [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] - public static unsafe void StoreArmVector128x4ToDestination(byte* dest, byte* destStart, int destLength, + public unsafe void StoreArmVector128x4ToDestination(byte* dest, byte* destStart, int destLength, Vector128 res1, Vector128 res2, Vector128 res3, Vector128 res4) => - Base64EncoderByte.StoreArmVector128x4ToDestination(dest, destStart, destLength, res1, res2, res3, res4); + default(Base64EncoderByte).StoreArmVector128x4ToDestination(dest, destStart, destLength, res1, res2, res3, res4); +#endif [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void EncodeThreeAndWrite(byte* threeBytes, byte* destination, ref byte encodingMap) => - Base64EncoderByte.EncodeThreeAndWrite(threeBytes, destination, ref encodingMap); + public unsafe void EncodeThreeAndWrite(byte* threeBytes, byte* destination, ref byte encodingMap) => + default(Base64EncoderByte).EncodeThreeAndWrite(threeBytes, destination, ref encodingMap); } private readonly struct Base64UrlEncoderChar : IBase64Encoder { - public static ReadOnlySpan EncodingMap => Base64UrlEncoderByte.EncodingMap; + public ReadOnlySpan EncodingMap => default(Base64UrlEncoderByte).EncodingMap; - public static sbyte Avx2LutChar62 => Base64UrlEncoderByte.Avx2LutChar62; + public sbyte Avx2LutChar62 => default(Base64UrlEncoderByte).Avx2LutChar62; - public static sbyte Avx2LutChar63 => Base64UrlEncoderByte.Avx2LutChar63; + public sbyte Avx2LutChar63 => default(Base64UrlEncoderByte).Avx2LutChar63; - public static ReadOnlySpan AdvSimdLut4 => Base64UrlEncoderByte.AdvSimdLut4; + public ReadOnlySpan AdvSimdLut4 => default(Base64UrlEncoderByte).AdvSimdLut4; - public static uint Ssse3AdvSimdLutE3 => Base64UrlEncoderByte.Ssse3AdvSimdLutE3; + public uint Ssse3AdvSimdLutE3 => default(Base64UrlEncoderByte).Ssse3AdvSimdLutE3; - public static int IncrementPadTwo => Base64UrlEncoderByte.IncrementPadTwo; + public int IncrementPadTwo => default(Base64UrlEncoderByte).IncrementPadTwo; - public static int IncrementPadOne => Base64UrlEncoderByte.IncrementPadOne; + public int IncrementPadOne => default(Base64UrlEncoderByte).IncrementPadOne; - public static int GetMaxSrcLength(int srcLength, int destLength) => - Base64UrlEncoderByte.GetMaxSrcLength(srcLength, destLength); + public int GetMaxSrcLength(int srcLength, int destLength) => + default(Base64UrlEncoderByte).GetMaxSrcLength(srcLength, destLength); - public static uint GetInPlaceDestinationLength(int encodedLength, int _) => 0; // not used for char encoding + public uint GetInPlaceDestinationLength(int encodedLength, int _) => 0; // not used for char encoding - public static int GetMaxEncodedLength(int _) => 0; // not used for char encoding + public int GetMaxEncodedLength(int _) => 0; // not used for char encoding [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void EncodeOneOptionallyPadTwo(byte* oneByte, ushort* dest, ref byte encodingMap) + public unsafe void EncodeOneOptionallyPadTwo(byte* oneByte, ushort* dest, ref byte encodingMap) { uint t0 = oneByte[0]; @@ -341,7 +364,7 @@ public static unsafe void EncodeOneOptionallyPadTwo(byte* oneByte, ushort* dest, } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void EncodeTwoOptionallyPadOne(byte* twoBytes, ushort* dest, ref byte encodingMap) + public unsafe void EncodeTwoOptionallyPadOne(byte* twoBytes, ushort* dest, ref byte encodingMap) { uint t0 = twoBytes[0]; uint t1 = twoBytes[1]; @@ -366,8 +389,9 @@ public static unsafe void EncodeTwoOptionallyPadOne(byte* twoBytes, ushort* dest } } +#if NET [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void StoreVector512ToDestination(ushort* dest, ushort* destStart, int destLength, Vector512 str) + public unsafe void StoreVector512ToDestination(ushort* dest, ushort* destStart, int destLength, Vector512 str) { AssertWrite>(dest, destStart, destLength); (Vector512 utf16LowVector, Vector512 utf16HighVector) = Vector512.Widen(str); @@ -376,7 +400,7 @@ public static unsafe void StoreVector512ToDestination(ushort* dest, ushort* dest } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void StoreVector256ToDestination(ushort* dest, ushort* destStart, int destLength, Vector256 str) + public unsafe void StoreVector256ToDestination(ushort* dest, ushort* destStart, int destLength, Vector256 str) { AssertWrite>(dest, destStart, destLength); (Vector256 utf16LowVector, Vector256 utf16HighVector) = Vector256.Widen(str); @@ -385,7 +409,7 @@ public static unsafe void StoreVector256ToDestination(ushort* dest, ushort* dest } [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void StoreVector128ToDestination(ushort* dest, ushort* destStart, int destLength, Vector128 str) + public unsafe void StoreVector128ToDestination(ushort* dest, ushort* destStart, int destLength, Vector128 str) { AssertWrite>(dest, destStart, destLength); (Vector128 utf16LowVector, Vector128 utf16HighVector) = Vector128.Widen(str); @@ -395,7 +419,7 @@ public static unsafe void StoreVector128ToDestination(ushort* dest, ushort* dest [MethodImpl(MethodImplOptions.AggressiveInlining)] [CompExactlyDependsOn(typeof(AdvSimd.Arm64))] - public static unsafe void StoreArmVector128x4ToDestination(ushort* dest, ushort* destStart, int destLength, + public unsafe void StoreArmVector128x4ToDestination(ushort* dest, ushort* destStart, int destLength, Vector128 res1, Vector128 res2, Vector128 res3, Vector128 res4) { AssertWrite>(dest, destStart, destLength); @@ -406,9 +430,10 @@ public static unsafe void StoreArmVector128x4ToDestination(ushort* dest, ushort* AdvSimd.Arm64.StoreVectorAndZip(dest, (utf16LowVector1, utf16LowVector2, utf16LowVector3, utf16LowVector4)); AdvSimd.Arm64.StoreVectorAndZip(dest + 32, (utf16HighVector1, utf16HighVector2, utf16HighVector3, utf16HighVector4)); } +#endif [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static unsafe void EncodeThreeAndWrite(byte* threeBytes, ushort* destination, ref byte encodingMap) + public unsafe void EncodeThreeAndWrite(byte* threeBytes, ushort* destination, ref byte encodingMap) { uint t0 = threeBytes[0]; uint t1 = threeBytes[1]; diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Url/Base64UrlValidator.cs b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Url/Base64UrlValidator.cs index 20eb0e926d62aa..ea785fc2ca7420 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Url/Base64UrlValidator.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Url/Base64UrlValidator.cs @@ -17,7 +17,7 @@ public static partial class Base64Url /// Any amount of whitespace is allowed anywhere in the input, where whitespace is defined as the characters ' ', '\t', '\r', or '\n'. /// public static bool IsValid(ReadOnlySpan base64UrlText) => - Base64.IsValid(base64UrlText, out _); + Base64Helper.IsValid(default(Base64UrlCharValidatable), base64UrlText, out _); /// Validates that the specified span of text is comprised of valid base-64 encoded data. /// A span of text to validate. @@ -30,7 +30,7 @@ public static bool IsValid(ReadOnlySpan base64UrlText) => /// Any amount of whitespace is allowed anywhere in the input, where whitespace is defined as the characters ' ', '\t', '\r', or '\n'. /// public static bool IsValid(ReadOnlySpan base64UrlText, out int decodedLength) => - Base64.IsValid(base64UrlText, out decodedLength); + Base64Helper.IsValid(default(Base64UrlCharValidatable), base64UrlText, out decodedLength); /// Validates that the specified span of UTF-8 text is comprised of valid base-64 encoded data. /// A span of UTF-8 text to validate. @@ -39,7 +39,7 @@ public static bool IsValid(ReadOnlySpan base64UrlText, out int decodedLeng /// where whitespace is defined as the characters ' ', '\t', '\r', or '\n' (as bytes). /// public static bool IsValid(ReadOnlySpan utf8Base64UrlText) => - Base64.IsValid(utf8Base64UrlText, out _); + Base64Helper.IsValid(default(Base64UrlByteValidatable), utf8Base64UrlText, out _); /// Validates that the specified span of UTF-8 text is comprised of valid base-64 encoded data. /// A span of UTF-8 text to validate. @@ -49,33 +49,51 @@ public static bool IsValid(ReadOnlySpan utf8Base64UrlText) => /// where whitespace is defined as the characters ' ', '\t', '\r', or '\n' (as bytes). /// public static bool IsValid(ReadOnlySpan utf8Base64UrlText, out int decodedLength) => - Base64.IsValid(utf8Base64UrlText, out decodedLength); + Base64Helper.IsValid(default(Base64UrlByteValidatable), utf8Base64UrlText, out decodedLength); private const uint UrlEncodingPad = '%'; // allowed for url padding - private readonly struct Base64UrlCharValidatable : Base64.IBase64Validatable + private readonly struct Base64UrlCharValidatable : Base64Helper.IBase64Validatable { +#if NET private static readonly SearchValues s_validBase64UrlChars = SearchValues.Create("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"); - public static int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExcept(s_validBase64UrlChars); - public static bool IsWhiteSpace(char value) => Base64.IsWhiteSpace(value); - public static bool IsEncodingPad(char value) => value == Base64.EncodingPad || value == UrlEncodingPad; + public int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExcept(s_validBase64UrlChars); +#else + public int DecodeValue(char value) + { + if (value > byte.MaxValue) + { + // Invalid char was found. + return -2; + } + + return default(Base64UrlDecoderByte).DecodingMap[value]; + } +#endif + public bool IsWhiteSpace(char value) => Base64Helper.IsWhiteSpace(value); + public bool IsEncodingPad(char value) => value == Base64Helper.EncodingPad || value == UrlEncodingPad; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool ValidateAndDecodeLength(int length, int paddingCount, out int decodedLength) => - Base64UrlByteValidatable.ValidateAndDecodeLength(length, paddingCount, out decodedLength); + public bool ValidateAndDecodeLength(int length, int paddingCount, out int decodedLength) => + default(Base64UrlByteValidatable).ValidateAndDecodeLength(length, paddingCount, out decodedLength); } - private readonly struct Base64UrlByteValidatable : Base64.IBase64Validatable + private readonly struct Base64UrlByteValidatable : Base64Helper.IBase64Validatable { - private static readonly SearchValues s_validBase64UrlChars = SearchValues.Create(Base64UrlEncoderByte.EncodingMap); +#if NET + private static readonly SearchValues s_validBase64UrlChars = SearchValues.Create(default(Base64UrlEncoderByte).EncodingMap); - public static int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExcept(s_validBase64UrlChars); - public static bool IsWhiteSpace(byte value) => Base64.IsWhiteSpace(value); - public static bool IsEncodingPad(byte value) => value == Base64.EncodingPad || value == UrlEncodingPad; + public int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExcept(s_validBase64UrlChars); +#else + public int DecodeValue(byte value) => default(Base64UrlDecoderByte).DecodingMap[value]; +#endif + public bool IsWhiteSpace(byte value) => Base64Helper.IsWhiteSpace(value); + public bool IsEncodingPad(byte value) => value == Base64Helper.EncodingPad || value == UrlEncodingPad; [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool ValidateAndDecodeLength(int length, int paddingCount, out int decodedLength) + public bool ValidateAndDecodeLength(int length, int paddingCount, out int decodedLength) { // Padding is optional for Base64Url, so need to account remainder. If remainder is 1, then it's invalid. +#if NET (uint whole, uint remainder) = uint.DivRem((uint)(length), 4); if (remainder == 1 || (remainder > 1 && (remainder - paddingCount == 1 || paddingCount == remainder))) { @@ -84,6 +102,16 @@ public static bool ValidateAndDecodeLength(int length, int paddingCount, out int } decodedLength = (int)((whole * 3) + (remainder > 0 ? remainder - 1 : 0) - paddingCount); +#else + int remainder = (int)((uint)length % 4); + if (remainder == 1 || (remainder > 1 && (remainder - paddingCount == 1 || paddingCount == remainder))) + { + decodedLength = 0; + return false; + } + + decodedLength = (length >> 2) * 3 + (remainder > 0 ? remainder - 1 : 0) - paddingCount; +#endif return true; } } diff --git a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Validator.cs b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Validator.cs index 3012bf6f4a0a5e..c2e4ad82a4ba0d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Validator.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Buffers/Text/Base64Validator.cs @@ -1,7 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Runtime.CompilerServices; +using static System.Buffers.Text.Base64Helper; namespace System.Buffers.Text { @@ -17,7 +17,7 @@ public static partial class Base64 /// where whitespace is defined as the characters ' ', '\t', '\r', or '\n'. /// public static bool IsValid(ReadOnlySpan base64Text) => - IsValid(base64Text, out _); + Base64Helper.IsValid(default(Base64CharValidatable), base64Text, out _); /// Validates that the specified span of text is comprised of valid base-64 encoded data. /// A span of text to validate. @@ -30,7 +30,7 @@ public static bool IsValid(ReadOnlySpan base64Text) => /// where whitespace is defined as the characters ' ', '\t', '\r', or '\n'. /// public static bool IsValid(ReadOnlySpan base64Text, out int decodedLength) => - IsValid(base64Text, out decodedLength); + Base64Helper.IsValid(default(Base64CharValidatable), base64Text, out decodedLength); /// Validates that the specified span of UTF-8 text is comprised of valid base-64 encoded data. /// A span of UTF-8 text to validate. @@ -41,7 +41,7 @@ public static bool IsValid(ReadOnlySpan base64Text, out int decodedLength) /// where whitespace is defined as the characters ' ', '\t', '\r', or '\n' (as bytes). /// public static bool IsValid(ReadOnlySpan base64TextUtf8) => - IsValid(base64TextUtf8, out _); + Base64Helper.IsValid(default(Base64ByteValidatable), base64TextUtf8, out _); /// Validates that the specified span of UTF-8 text is comprised of valid base-64 encoded data. /// A span of UTF-8 text to validate. @@ -53,126 +53,7 @@ public static bool IsValid(ReadOnlySpan base64TextUtf8) => /// where whitespace is defined as the characters ' ', '\t', '\r', or '\n' (as bytes). /// public static bool IsValid(ReadOnlySpan base64TextUtf8, out int decodedLength) => - IsValid(base64TextUtf8, out decodedLength); + Base64Helper.IsValid(default(Base64ByteValidatable), base64TextUtf8, out decodedLength); - internal static bool IsValid(ReadOnlySpan base64Text, out int decodedLength) - where TBase64Validatable : IBase64Validatable - { - int length = 0, paddingCount = 0; - - if (!base64Text.IsEmpty) - { - while (true) - { - int index = TBase64Validatable.IndexOfAnyExcept(base64Text); - if ((uint)index >= (uint)base64Text.Length) - { - length += base64Text.Length; - break; - } - - length += index; - - T charToValidate = base64Text[index]; - base64Text = base64Text.Slice(index + 1); - - if (TBase64Validatable.IsWhiteSpace(charToValidate)) - { - // It's common if there's whitespace for there to be multiple whitespace characters in a row, - // e.g. \r\n. Optimize for that case by looping here. - while (!base64Text.IsEmpty && TBase64Validatable.IsWhiteSpace(base64Text[0])) - { - base64Text = base64Text.Slice(1); - } - continue; - } - - if (!TBase64Validatable.IsEncodingPad(charToValidate)) - { - // Invalid char was found. - goto Fail; - } - - // Encoding pad found. Determine if padding is valid, then stop processing. - paddingCount = 1; - foreach (T charToValidateInPadding in base64Text) - { - if (TBase64Validatable.IsEncodingPad(charToValidateInPadding)) - { - // There can be at most 2 padding chars. - if (paddingCount >= 2) - { - goto Fail; - } - - paddingCount++; - } - else if (!TBase64Validatable.IsWhiteSpace(charToValidateInPadding)) - { - // Invalid char was found. - goto Fail; - } - } - - length += paddingCount; - break; - } - - if (!TBase64Validatable.ValidateAndDecodeLength(length, paddingCount, out decodedLength)) - { - goto Fail; - } - - return true; - } - - decodedLength = 0; - return true; - - Fail: - decodedLength = 0; - return false; - } - - internal interface IBase64Validatable - { - static abstract int IndexOfAnyExcept(ReadOnlySpan span); - static abstract bool IsWhiteSpace(T value); - static abstract bool IsEncodingPad(T value); - static abstract bool ValidateAndDecodeLength(int length, int paddingCount, out int decodedLength); - } - - private readonly struct Base64CharValidatable : IBase64Validatable - { - private static readonly SearchValues s_validBase64Chars = SearchValues.Create("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"); - - public static int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExcept(s_validBase64Chars); - public static bool IsWhiteSpace(char value) => Base64.IsWhiteSpace(value); - public static bool IsEncodingPad(char value) => value == EncodingPad; - public static bool ValidateAndDecodeLength(int length, int paddingCount, out int decodedLength) => - Base64ByteValidatable.ValidateAndDecodeLength(length, paddingCount, out decodedLength); - } - - private readonly struct Base64ByteValidatable : IBase64Validatable - { - private static readonly SearchValues s_validBase64Chars = SearchValues.Create(Base64EncoderByte.EncodingMap); - - public static int IndexOfAnyExcept(ReadOnlySpan span) => span.IndexOfAnyExcept(s_validBase64Chars); - public static bool IsWhiteSpace(byte value) => Base64.IsWhiteSpace(value); - public static bool IsEncodingPad(byte value) => value == EncodingPad; - [MethodImpl(MethodImplOptions.AggressiveInlining)] - public static bool ValidateAndDecodeLength(int length, int paddingCount, out int decodedLength) - { - if (length % 4 == 0) - { - // Remove padding to get exact length. - decodedLength = (int)((uint)length / 4 * 3) - paddingCount; - return true; - } - - decodedLength = 0; - return false; - } - } } }