File tree Expand file tree Collapse file tree 1 file changed +5
-5
lines changed Expand file tree Collapse file tree 1 file changed +5
-5
lines changed Original file line number Diff line number Diff line change @@ -303,7 +303,7 @@ struct llama_sp_symbol {
303
303
304
304
struct llama_sp_bigram {
305
305
struct comparator {
306
- const bool operator ()(llama_sp_bigram& l, llama_sp_bigram& r) {
306
+ const bool operator ()(llama_sp_bigram & l, llama_sp_bigram & r) {
307
307
return (l.score < r.score ) || (l.score == r.score && l.left > r.left );
308
308
}
309
309
};
@@ -316,9 +316,9 @@ struct llama_sp_bigram {
316
316
};
317
317
318
318
struct llama_tokenizer {
319
- llama_tokenizer (gpt_vocab const & vocab): vocab_(vocab) {}
319
+ llama_tokenizer (const gpt_vocab & vocab): vocab_(vocab) {}
320
320
321
- void tokenize (std::string_view text, std::vector<gpt_vocab::id>& output) {
321
+ void tokenize (std::string_view text, std::vector<gpt_vocab::id> & output) {
322
322
// split string into utf8 chars
323
323
int index = 0 ;
324
324
while (!text.empty ()) {
@@ -342,8 +342,8 @@ struct llama_tokenizer {
342
342
auto bigram = work_queue_.top ();
343
343
work_queue_.pop ();
344
344
345
- auto & left_sym = symbols_[bigram.left ];
346
- auto & right_sym = symbols_[bigram.right ];
345
+ auto & left_sym = symbols_[bigram.left ];
346
+ auto & right_sym = symbols_[bigram.right ];
347
347
348
348
// if one of the symbols already got merged, skip it.
349
349
if (left_sym.text .empty () || right_sym.text .empty () ||
You can’t perform that action at this time.
0 commit comments