Introducing SentencePiece Unigram Tokenizer Model

THIRD-PARTY-NOTICES.TXT

@@ -133,6 +133,27 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.!
 
+License notice for DoubleArrayTrie (DART)
+--------------------------------------------
+The BSD 2-clause license
+
+https://github.com/s-yata/darts-clone/blob/master/COPYING.md
+
+Copyright (c) 2008-2014, Susumu Yata All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+Redistributions in binary form must reproduce the above copyright notice,
+this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY,
+OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
 License notice for CodeGen Tokenizer
 --------------------------------------------
 

eng/Versions.props

@@ -100,7 +100,7 @@
     <MicrosoftMLTensorFlowTestModelsVersion>0.0.13-test</MicrosoftMLTensorFlowTestModelsVersion>
     <MicrosoftMLTestDatabasesVersion>0.0.6-test</MicrosoftMLTestDatabasesVersion>
     <MicrosoftMLTestModelsVersion>0.0.7-test</MicrosoftMLTestModelsVersion>
-    <MicrosoftMLTestTokenizersVersion>2.0.0-beta.24455.2</MicrosoftMLTestTokenizersVersion>
+    <MicrosoftMLTestTokenizersVersion>2.0.0-beta.25110.1</MicrosoftMLTestTokenizersVersion>
     <SystemDataSqlClientVersion>4.9.0</SystemDataSqlClientVersion>
     <SystemDataSQLiteCoreVersion>1.0.118</SystemDataSQLiteCoreVersion>
     <XunitCombinatorialVersion>1.6.24</XunitCombinatorialVersion>

src/Microsoft.ML.Tokenizers/Model/CodeGenTokenizer.cs

@@ -22,6 +22,9 @@ namespace Microsoft.ML.Tokenizers
     /// </summary>
     public class CodeGenTokenizer : Tokenizer
     {
+        // The CodeGen tokenizer implementation is primarily adapted from
+        // https://github.com/huggingface/transformers/blob/main/src/transformers/models/codegen/tokenization_codegen.py,
+        // with modifications to align with C# code style, the API, and the tokenizer library design.
         private readonly Dictionary<StringSpanOrdinalKey, (int Id, string Token)> _vocab;
         private IReadOnlyDictionary<string, int>? _vocabOriginal;
         private readonly IReadOnlyDictionary<int, string> _vocabReverse;

src/Microsoft.ML.Tokenizers/Model/LlamaTokenizer.cs

@@ -31,7 +31,7 @@ internal LlamaTokenizer(ModelProto modelProto, bool addBos, bool addEos, IReadOn
         /// <remarks>
         /// When creating the tokenizer, ensure that the vocabulary stream is sourced from a trusted provider.
         /// </remarks>
-        public static LlamaTokenizer Create(
+        public static new LlamaTokenizer Create(
             Stream modelStream,
             bool addBeginOfSentence = true,
             bool addEndOfSentence = false,
@@ -54,13 +54,6 @@ public static LlamaTokenizer Create(
                 throw new ArgumentException($"Normalization '{modelProto.NormalizerSpec.Name}' is not supported.", nameof(modelProto));
             }
 
-            SentencePieceNormalizer normalizer = new(
-                                    modelProto.NormalizerSpec.RemoveExtraWhitespaces,
-                                    modelProto.NormalizerSpec.AddDummyPrefix,
-                                    modelProto.NormalizerSpec.EscapeWhitespaces,
-                                    modelProto.TrainerSpec.TreatWhitespaceAsSuffix,
-                                    specialTokens);
-
             return new LlamaTokenizer(modelProto, addBeginOfSentence, addEndOfSentence, specialTokens);
         }
     }