diff --git a/torchtext/csrc/bert_tokenizer.cpp b/torchtext/csrc/bert_tokenizer.cpp index 06aba8ae7c..4bade1153a 100644 --- a/torchtext/csrc/bert_tokenizer.cpp +++ b/torchtext/csrc/bert_tokenizer.cpp @@ -1,3 +1,25 @@ +/* Portions Copyright (c) Meta Platforms, Inc. and affiliates. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +Original code is taken from +https://github.com/LieluoboAi/radish/blob/master/radish/bert/bert_tokenizer.cc + +The code is modified and summary is provided in this PR +https://github.com/pytorch/text/pull/1707 +*/ + #include #include