Optimized Encoder

Author: gfoidl

src/System.Memory/src/System/Buffers/Text/Base64Encoder.cs

@@ -59,31 +59,34 @@ public static unsafe OperationStatus EncodeToUtf8(ReadOnlySpan<byte> bytes, Span
 
                 byte* src = srcBytes;
                 byte* dest = destBytes;
-                byte* maxSrc = srcBytes + (uint)maxSrcLength;
+                byte* srcEnd = srcBytes + (uint)srcLength;
+                byte* srcMax = srcBytes + (uint)maxSrcLength;
 
                 if (maxSrcLength >= 16)
                 {
-                    if (Avx2.IsSupported && maxSrcLength >= 32)
+                    byte* end = srcMax - 32;
+                    if (Avx2.IsSupported && (end >= src))
                     {
-                        Avx2Encode(ref src, ref dest, maxSrcLength, destLength, srcBytes, destBytes);
+                        Avx2Encode(ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes);
 
-                        if (srcBytes + (uint)srcLength == src)
+                        if (src == srcEnd)
                             goto DoneExit;
                     }
 
-                    if (Ssse3.IsSupported && (maxSrc - src >= 16))
+                    end = srcMax - 16;
+                    if (Ssse3.IsSupported && (end >= src))
                     {
-                        Ssse3Encode(ref src, ref dest, maxSrcLength, destLength, srcBytes, destBytes);
+                        Ssse3Encode(ref src, ref dest, end, maxSrcLength, destLength, srcBytes, destBytes);
 
-                        if (srcBytes + (uint)srcLength == src)
+                        if (src == srcEnd)
                             goto DoneExit;
                     }
                 }
 
                 uint result = 0;
 
-                maxSrc -= 2;
-                while (src < maxSrc)
+                srcMax -= 2;
+                while (src < srcMax)
                 {
                     result = Encode(src, encodingMap);
                     Unsafe.WriteUnaligned(dest, result);
@@ -97,15 +100,15 @@ public static unsafe OperationStatus EncodeToUtf8(ReadOnlySpan<byte> bytes, Span
                 if (!isFinalBlock)
                     goto NeedMoreData;
 
-                maxSrc = srcBytes + (uint)srcLength;
-                if (src + 1 == maxSrc)
+                srcMax = srcBytes + (uint)srcLength;
+                if (src + 1 == srcMax)
                 {
                     result = EncodeAndPadTwo(src, encodingMap);
                     Unsafe.WriteUnaligned(dest, result);
                     src += 1;
                     dest += 4;
                 }
-                else if (src + 2 == maxSrc)
+                else if (src + 2 == srcMax)
                 {
                     result = EncodeAndPadOne(src, encodingMap);
                     Unsafe.WriteUnaligned(dest, result);
@@ -211,12 +214,8 @@ public static unsafe OperationStatus EncodeToUtf8InPlace(Span<byte> buffer, int
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static unsafe void Avx2Encode(ref byte* srcBytes, ref byte* destBytes, int sourceLength, int destLength, byte* srcStart, byte* destStart)
+        private static unsafe void Avx2Encode(ref byte* srcBytes, ref byte* destBytes, byte* srcEnd, int sourceLength, int destLength, byte* srcStart, byte* destStart)
         {
-            byte* src = srcBytes;
-            byte* dest = destBytes;
-            byte* srcEnd = srcStart + sourceLength - 28;     // 28 = 32 - 4
-
             // The JIT won't hoist these "constants", so help it
             Vector256<sbyte> shuffleVec = s_avxEncodeShuffleVec;
             Vector256<sbyte> shuffleConstant0 = Vector256.Create(0x0fc0fc00).AsSByte();
@@ -227,13 +226,19 @@ private static unsafe void Avx2Encode(ref byte* srcBytes, ref byte* destBytes, i
             Vector256<sbyte> translationContant1 = Vector256.Create((sbyte)25);
             Vector256<sbyte> lut = s_avxEncodeLut;
 
+            byte* src = srcBytes;
+            byte* dest = destBytes;
+
             // first load is done at c-0 not to get a segfault
             AssertRead<Vector256<sbyte>>(src, srcStart, sourceLength);
             Vector256<sbyte> str = Avx.LoadVector256(src).AsSByte();
 
             // shift by 4 bytes, as required by Reshuffle
             str = Avx2.PermuteVar8x32(str.AsInt32(), s_avxEncodePermuteVec).AsSByte();
 
+            // Next loads are done at src-4, as required by Reshuffle, so shift it once
+            src -= 4;
+
             while (true)
             {
                 // Reshuffle
@@ -259,22 +264,18 @@ private static unsafe void Avx2Encode(ref byte* srcBytes, ref byte* destBytes, i
                 if (src > srcEnd)
                     break;
 
-                // Load at c-4, as required by enc_reshuffle
-                AssertRead<Vector256<sbyte>>(src - 4, srcStart, sourceLength);
-                str = Avx.LoadVector256(src - 4).AsSByte();
+                // Load at src-4, as required by Reshuffle (already shifted by -4)
+                AssertRead<Vector256<sbyte>>(src, srcStart, sourceLength);
+                str = Avx.LoadVector256(src).AsSByte();
             }
 
-            srcBytes = src;
+            srcBytes = src + 4;
             destBytes = dest;
         }
 
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
-        private static unsafe void Ssse3Encode(ref byte* srcBytes, ref byte* destBytes, int sourceLength, int destLength, byte* srcStart, byte* destStart)
+        private static unsafe void Ssse3Encode(ref byte* srcBytes, ref byte* destBytes, byte* srcEnd, int sourceLength, int destLength, byte* srcStart, byte* destStart)
         {
-            byte* src = srcBytes;
-            byte* dest = destBytes;
-            byte* srcEnd = srcStart + sourceLength - 16;
-
             // The JIT won't hoist these "constants", so help it
             Vector128<sbyte> shuffleVec = s_sseEncodeShuffleVec;
             Vector128<sbyte> shuffleConstant0 = Vector128.Create(0x0fc0fc00).AsSByte();
@@ -285,6 +286,9 @@ private static unsafe void Ssse3Encode(ref byte* srcBytes, ref byte* destBytes,
             Vector128<sbyte> translationContant1 = Vector128.Create((sbyte)25);
             Vector128<sbyte> lut = s_sseEncodeLut;
 
+            byte* src = srcBytes;
+            byte* dest = destBytes;
+
             //while (remaining >= 16)
             while (src <= srcEnd)
             {