We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 191ab03 commit b9b0f58Copy full SHA for b9b0f58
src/Microsoft.ML.Tokenizers/PreTokenizer/Roberta.cs
@@ -30,7 +30,7 @@ public override IEnumerable<Split> PreTokenize(string text, bool considerSpecial
30
return Array.Empty<Split>();
31
}
32
33
- return SplitText(sentence, Tokenizer.P50kBaseRegex());
+ return SplitText(text, Tokenizer.P50kBaseRegex());
34
35
36
0 commit comments