We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 9a07eec commit 755f046Copy full SHA for 755f046
utils.cpp
@@ -323,7 +323,7 @@ struct llama_tokenizer {
323
int index = 0;
324
while (!text.empty()) {
325
llama_sp_symbol sym;
326
- size_t char_len = utf8_len(text.data()[0]);
+ size_t char_len = std::min(text.size(), utf8_len(text.data()[0]));
327
sym.text = std::string_view(text.data(), char_len);
328
sym.prev = index - 1;
329
text.remove_prefix(char_len);
0 commit comments