Skip to content

Speed up Convert.ToBase64String() #69884

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 4 commits into from
Closed

Speed up Convert.ToBase64String() #69884

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7421802
Speed up Convert.ToBase64String()
SergeiPavlov May 26, 2022
37a740e
Revert filemode of build.cmd
SergeiPavlov May 26, 2022
6195b95
Update src/libraries/System.Private.CoreLib/src/System/Convert.cs
SergeiPavlov May 29, 2022
1582156
Fix typo
SergeiPavlov May 29, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
118 changes: 72 additions & 46 deletions src/libraries/System.Private.CoreLib/src/System/Convert.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -127,7 +127,13 @@ public static partial class Convert
't', 'u', 'v', 'w', 'x', 'y', 'z', '0', '1', '2', '3', '4', '5', '6', '7',
'8', '9', '+', '/', '=' };

private const int base64LineBreakPosition = 76;
// Initialized on demand. Takes 16 KiB.
private static int[]? base64PairsTable;

private static readonly int crlf = BitConverter.IsLittleEndian ? 0x000a000d : 0x000d000a; // "\r\n"
private const int doublePad = 0x003d003d; // "=="

private const int base64LineBreakPosition = 76; // must be multiple of 4

#if DEBUG
static Convert()
Expand Down Expand Up @@ -2356,7 +2362,7 @@ public static string ToBase64String(ReadOnlySpan<byte> bytes, Base64FormattingOp
fixed (byte* bytesPtr = &MemoryMarshal.GetReference(bytes))
fixed (char* charsPtr = result)
{
int charsWritten = ConvertToBase64Array(charsPtr, bytesPtr, 0, bytes.Length, insertLineBreaks);
int charsWritten = ConvertToBase64Array(charsPtr, bytesPtr, bytes.Length, insertLineBreaks);
Debug.Assert(result.Length == charsWritten, $"Expected {result.Length} == {charsWritten}");
}
}
Expand Down Expand Up @@ -2414,7 +2420,7 @@ public static unsafe int ToBase64CharArray(byte[] inArray, int offsetIn, int len
{
fixed (byte* inData = &inArray[0])
{
retVal = ConvertToBase64Array(outChars, inData, offsetIn, length, insertLineBreaks);
retVal = ConvertToBase64Array(outChars, inData + offsetIn, length, insertLineBreaks);
}
}

Expand Down Expand Up @@ -2446,71 +2452,91 @@ public static unsafe bool TryToBase64Chars(ReadOnlySpan<byte> bytes, Span<char>
fixed (char* outChars = &MemoryMarshal.GetReference(chars))
fixed (byte* inData = &MemoryMarshal.GetReference(bytes))
{
charsWritten = ConvertToBase64Array(outChars, inData, 0, bytes.Length, insertLineBreaks);
charsWritten = ConvertToBase64Array(outChars, inData, bytes.Length, insertLineBreaks);
return true;
}
}

private static unsafe int ConvertToBase64Array(char* outChars, byte* inData, int offset, int length, bool insertLineBreaks)
private static int[] CreateBase64PairsTable()
{
var table = new int[64 * 64];
for (int i = table.Length; i-- > 0;)
{
var firstChar = base64Table[i >> 6];
var secondChar = base64Table[i & 0x3f];
table[i] = BitConverter.IsLittleEndian
? firstChar | (secondChar << 16)
: secondChar | (firstChar << 16);
}
return table;
}

private static unsafe int ConvertToBase64Array(char* outChars, byte* inData, int length, bool insertLineBreaks)
{
int* outPairs = (int*)outChars;
int lengthmod3 = length % 3;
int calcLength = offset + (length - lengthmod3);
int j = 0;
int charcount = 0;
// Convert three bytes at a time to base64 notation. This will consume 4 chars.
int i;

// get a pointer to the base64Table to avoid unnecessary range checking
fixed (char* base64 = &base64Table[0])
int calcLength = length - lengthmod3;
byte a, b, c;

base64PairsTable ??= CreateBase64PairsTable();

// get a pointer to the base64PairsTable to avoid unnecessary range checking
fixed (int* base64Pairs = &base64PairsTable[0])
{
for (i = offset; i < calcLength; i += 3)
while (calcLength > 0)
{
if (insertLineBreaks)
var rounds = insertLineBreaks
? Math.Min(calcLength, base64LineBreakPosition / 4 * 3)
: calcLength;

calcLength -= rounds;
// Convert three bytes at a time to base64 notation. This will consume 4 chars.
for (; rounds > 0; rounds -= 3)
{
if (charcount == base64LineBreakPosition)
{
outChars[j++] = '\r';
outChars[j++] = '\n';
charcount = 0;
}
charcount += 4;
a = *inData++;
b = *inData++;
c = *inData++;
*outPairs++ = base64Pairs[(a << 4) | (b >> 4)];
*outPairs++ = base64Pairs[((b << 8) | c) & 0xfff];
}
outChars[j] = base64[(inData[i] & 0xfc) >> 2];
outChars[j + 1] = base64[((inData[i] & 0x03) << 4) | ((inData[i + 1] & 0xf0) >> 4)];
outChars[j + 2] = base64[((inData[i + 1] & 0x0f) << 2) | ((inData[i + 2] & 0xc0) >> 6)];
outChars[j + 3] = base64[inData[i + 2] & 0x3f];
j += 4;
}

// Where we left off before
i = calcLength;

if (insertLineBreaks && (lengthmod3 != 0) && (charcount == base64LineBreakPosition))
if (insertLineBreaks
&& calcLength > 0
&& ((char*)outPairs - outChars) % (base64LineBreakPosition + 2) == base64LineBreakPosition)
{
*outPairs++ = crlf;
}
}
if (insertLineBreaks
&& lengthmod3 != 0
&& ((char*)outPairs - outChars) % (base64LineBreakPosition + 2) == base64LineBreakPosition)
{
outChars[j++] = '\r';
outChars[j++] = '\n';
*outPairs++ = crlf;
}

char* outPtr;
switch (lengthmod3)
{
case 2: // One character padding needed
outChars[j] = base64[(inData[i] & 0xfc) >> 2];
outChars[j + 1] = base64[((inData[i] & 0x03) << 4) | ((inData[i + 1] & 0xf0) >> 4)];
outChars[j + 2] = base64[(inData[i + 1] & 0x0f) << 2];
outChars[j + 3] = base64[64]; // Pad
j += 4;
a = *inData++;
b = *inData++;
*outPairs++ = base64Pairs[(a << 4) | (b >> 4)];
Copy link
Contributor

@deeprobin deeprobin May 30, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tannergooding
Why is the codegen soo different to LLVM? https://godbolt.org/z/T8eqE9chv

LLVM

        shr     sil, 4
        shl     dil, 4
        or      dil, sil
        movzx   eax, dil
        ret

.NET JIT

       movzx    rax, dil
       shl      eax, 4
       movzx    rdi, sil
       sar      edi, 4
       or       eax, edi
       ret

Copy link
Member

@EgorBo EgorBo May 30, 2022 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@deeprobin feel free to file an issue. Related one is #13816 (you can use PR that closed it as a foundation for your PR if you want to contribute 🙂)

outPtr = (char*)outPairs;
*outPtr++ = base64Table[(b & 0x0f) << 2];
*outPtr++ = '='; // Pad
break;
case 1: // Two character padding needed
outChars[j] = base64[(inData[i] & 0xfc) >> 2];
outChars[j + 1] = base64[(inData[i] & 0x03) << 4];
outChars[j + 2] = base64[64]; // Pad
outChars[j + 3] = base64[64]; // Pad
j += 4;
a = *inData++;
*outPairs++ = base64Pairs[a << 4];
*outPairs++ = doublePad;
outPtr = (char*)outPairs;
break;
default:
outPtr = (char*)outPairs;
break;
}
return (int)(outPtr - outChars);
}

return j;
}

private static int ToBase64_CalculateAndValidateOutputLength(int inputLength, bool insertLineBreaks)
Expand Down