diff --git a/mysql-test/suite/innodb_fts/r/index_table.result b/mysql-test/suite/innodb_fts/r/index_table.result index 909a889db4230..78832669cdbca 100644 --- a/mysql-test/suite/innodb_fts/r/index_table.result +++ b/mysql-test/suite/innodb_fts/r/index_table.result @@ -5,7 +5,7 @@ id INT UNSIGNED AUTO_INCREMENT NOT NULL PRIMARY KEY, title VARCHAR(200), content TEXT ) ENGINE= InnoDB; -SET STATEMENT debug_dbug='+d,innodb_report_deadlock' FOR +SET STATEMENT debug_dbug='+d,fts_load_stopword_fail' FOR CREATE FULLTEXT INDEX idx ON articles (title, content); ERROR HY000: Got error 11 "Resource temporarily unavailable" from storage engine InnoDB CREATE FULLTEXT INDEX idx ON articles (title, content); diff --git a/mysql-test/suite/innodb_fts/t/index_table.test b/mysql-test/suite/innodb_fts/t/index_table.test index 89c0905323083..cfe27b4226848 100644 --- a/mysql-test/suite/innodb_fts/t/index_table.test +++ b/mysql-test/suite/innodb_fts/t/index_table.test @@ -18,7 +18,7 @@ CREATE TABLE articles ( ) ENGINE= InnoDB; --error ER_GET_ERRNO -SET STATEMENT debug_dbug='+d,innodb_report_deadlock' FOR +SET STATEMENT debug_dbug='+d,fts_load_stopword_fail' FOR CREATE FULLTEXT INDEX idx ON articles (title, content); CREATE FULLTEXT INDEX idx ON articles (title, content); diff --git a/storage/innobase/CMakeLists.txt b/storage/innobase/CMakeLists.txt index 3204b66721b17..63273ce293dbf 100644 --- a/storage/innobase/CMakeLists.txt +++ b/storage/innobase/CMakeLists.txt @@ -171,10 +171,10 @@ SET(INNOBASE_SOURCES fts/fts0ast.cc fts/fts0blex.cc fts/fts0config.cc + fts/fts0exec.cc fts/fts0opt.cc fts/fts0pars.cc fts/fts0que.cc - fts/fts0sql.cc fts/fts0tlex.cc gis/gis0geo.cc gis/gis0rtree.cc @@ -239,6 +239,7 @@ SET(INNOBASE_SOURCES include/fsp0types.h include/fts0ast.h include/fts0blex.h + include/fts0exec.h include/fts0fts.h include/fts0opt.h include/fts0pars.h @@ -311,6 +312,7 @@ SET(INNOBASE_SOURCES include/row0mysql.h include/row0purge.h include/row0quiesce.h + include/row0query.h include/row0row.h include/row0row.inl include/row0sel.h @@ -399,6 +401,7 @@ SET(INNOBASE_SOURCES row/row0undo.cc row/row0upd.cc row/row0quiesce.cc + row/row0query.cc row/row0vers.cc srv/srv0mon.cc srv/srv0srv.cc diff --git a/storage/innobase/fts/fts0config.cc b/storage/innobase/fts/fts0config.cc index 524f648676eb7..bdb9307a592c6 100644 --- a/storage/innobase/fts/fts0config.cc +++ b/storage/innobase/fts/fts0config.cc @@ -27,100 +27,37 @@ Created 2007/5/9 Sunny Bains #include "trx0roll.h" #include "row0sel.h" +#include "fts0exec.h" #include "fts0priv.h" -/******************************************************************//** -Callback function for fetching the config value. -@return always returns TRUE */ -static -ibool -fts_config_fetch_value( -/*===================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to - ib_vector_t */ -{ - sel_node_t* node = static_cast(row); - fts_string_t* value = static_cast(user_arg); - - dfield_t* dfield = que_node_get_val(node->select_list); - dtype_t* type = dfield_get_type(dfield); - ulint len = dfield_get_len(dfield); - void* data = dfield_get_data(dfield); - - ut_a(dtype_get_mtype(type) == DATA_VARCHAR); - - if (len != UNIV_SQL_NULL) { - ulint max_len = ut_min(value->f_len - 1, len); - - memcpy(value->f_str, data, max_len); - value->f_len = max_len; - value->f_str[value->f_len] = '\0'; - } - - return(TRUE); -} - -/******************************************************************//** -Get value from the config table. The caller must ensure that enough +/** Get value from the config table. The caller must ensure that enough space is allocated for value to hold the column contents. +@param trx transaction +@param table Indexed fts table +@param name name of the key +@param value value of the key @return DB_SUCCESS or error code */ -dberr_t -fts_config_get_value( -/*=================*/ - trx_t* trx, /*!< transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: get config value for - this parameter name */ - fts_string_t* value) /*!< out: value read from - config table */ +dberr_t fts_config_get_value(trx_t *trx, const dict_table_t *table, + const char *name, fts_string_t *value) { - pars_info_t* info; - que_t* graph; - dberr_t error; - ulint name_len = strlen(name); - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - *value->f_str = '\0'; - ut_a(value->f_len > 0); - - pars_info_bind_function(info, "my_func", fts_config_fetch_value, - value); - - /* The len field of value must be set to the max bytes that - it can hold. On a successful read, the len field will be set - to the actual number of bytes copied to value. */ - pars_info_bind_varchar_literal(info, "name", (byte*) name, name_len); - - fts_table->suffix = "CONFIG"; - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS SELECT value FROM $table_name" - " WHERE key = :name;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - trx->op_info = "getting FTS config value"; - - error = fts_eval_sql(trx, graph); - que_graph_free(graph); - return(error); + trx->op_info = "getting FTS config value"; + FTSQueryExecutor executor(trx, nullptr, table); + ConfigReader reader; + dberr_t err= executor.read_config(name, reader); + if (err == DB_SUCCESS) + { + if (reader.config_pairs.size() != 1) err= DB_ERROR; + else + { + const std::string& config_value= reader.config_pairs[0].second; + ulint max_len= ut_min(value->f_len - 1, config_value.length()); + memcpy(value->f_str, config_value.c_str(), max_len); + value->f_len= max_len; + value->f_str[value->f_len]= '\0'; + } + } + else value->f_str[0]= '\0'; + return err; } /*********************************************************************//** @@ -164,98 +101,32 @@ fts_config_get_index_value( fts_string_t* value) /*!< out: value read from config table */ { - char* name; - dberr_t error; - fts_table_t fts_table; - - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, - index->table); - /* We are responsible for free'ing name. */ - name = fts_config_create_index_param_name(param, index); + char *name = fts_config_create_index_param_name(param, index); - error = fts_config_get_value(trx, &fts_table, name, value); + dberr_t error = fts_config_get_value(trx, index->table, name, value); ut_free(name); return(error); } -/******************************************************************//** -Set the value in the config table for name. +/** Set the value in the config table for name. +@param trx transaction +@param fts_table indexed fulltext table +@param name key for the config +@param value value of the key @return DB_SUCCESS or error code */ dberr_t -fts_config_set_value( -/*=================*/ - trx_t* trx, /*!< transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: get config value for - this parameter name */ - const fts_string_t* - value) /*!< in: value to update */ +fts_config_set_value(trx_t *trx, const dict_table_t *table, + const char *name, const fts_string_t *value) { - pars_info_t* info; - que_t* graph; - dberr_t error; - undo_no_t undo_no; - undo_no_t n_rows_updated; - ulint name_len = strlen(name); - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - pars_info_bind_varchar_literal(info, "name", (byte*) name, name_len); - pars_info_bind_varchar_literal(info, "value", - value->f_str, value->f_len); - - const bool dict_locked = fts_table->table->fts->dict_locked; - - fts_table->suffix = "CONFIG"; - fts_get_table_name(fts_table, table_name, dict_locked); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, info, - "BEGIN UPDATE $table_name SET value = :value" - " WHERE key = :name;"); - - trx->op_info = "setting FTS config value"; - - undo_no = trx->undo_no; - - error = fts_eval_sql(trx, graph); - - que_graph_free(graph); - - n_rows_updated = trx->undo_no - undo_no; - - /* Check if we need to do an insert. */ - if (error == DB_SUCCESS && n_rows_updated == 0) { - info = pars_info_create(); - - pars_info_bind_varchar_literal( - info, "name", (byte*) name, name_len); - - pars_info_bind_varchar_literal( - info, "value", value->f_str, value->f_len); - - fts_get_table_name(fts_table, table_name, dict_locked); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, info, - "BEGIN\n" - "INSERT INTO $table_name VALUES(:name, :value);"); - - trx->op_info = "inserting FTS config value"; - - error = fts_eval_sql(trx, graph); - - que_graph_free(graph); - } - - return(error); + trx->op_info= "setting FTS config value"; + FTSQueryExecutor executor(trx, nullptr, table, table->fts->dict_locked); + char value_str[FTS_MAX_CONFIG_VALUE_LEN + 1]; + memcpy(value_str, value->f_str, value->f_len); + value_str[value->f_len]= '\0'; + return executor.update_config_record(name, value_str); } /******************************************************************//** @@ -271,17 +142,10 @@ fts_config_set_index_value( fts_string_t* value) /*!< out: value read from config table */ { - char* name; - dberr_t error; - fts_table_t fts_table; - - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, - index->table); - /* We are responsible for free'ing name. */ - name = fts_config_create_index_param_name(param, index); + char *name = fts_config_create_index_param_name(param, index); - error = fts_config_set_value(trx, &fts_table, name, value); + dberr_t error = fts_config_set_value(trx, index->table, name, value); ut_free(name); @@ -358,71 +222,50 @@ fts_config_set_index_ulint( } #endif /* FTS_OPTIMIZE_DEBUG */ -/******************************************************************//** -Get an ulint value from the config table. +/** Get an ulint value from the config table. +@param trx transaction +@param table user table +@param name key value +@param int_value value of the key @return DB_SUCCESS if all OK else error code */ dberr_t -fts_config_get_ulint( -/*=================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: param name */ - ulint* int_value) /*!< out: value */ +fts_config_get_ulint(trx_t *trx, const dict_table_t *table, + const char *name, ulint *int_value) { - dberr_t error; - fts_string_t value; - - /* We set the length of value to the max bytes it can hold. This - information is used by the callback that reads the value.*/ - value.f_len = FTS_MAX_CONFIG_VALUE_LEN; - value.f_str = static_cast(ut_malloc_nokey(value.f_len + 1)); - - error = fts_config_get_value(trx, fts_table, name, &value); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") reading `" << name << "'"; - } else { - *int_value = strtoul((char*) value.f_str, NULL, 10); - } - - ut_free(value.f_str); - - return(error); + fts_string_t value; + /* We set the length of value to the max bytes it can hold. This + information is used by the callback that reads the value.*/ + value.f_len= FTS_MAX_CONFIG_VALUE_LEN; + value.f_str= static_cast(ut_malloc_nokey(value.f_len + 1)); + dberr_t error= fts_config_get_value(trx, table, name, &value); + if (UNIV_UNLIKELY(error != DB_SUCCESS)) + ib::error() << "(" << error << ") reading `" << name << "'"; + else *int_value = strtoul((char*) value.f_str, NULL, 10); + ut_free(value.f_str); + return error; } -/******************************************************************//** -Set an ulint value in the config table. +/** Set an ulint value in the config table. +@param trx transaction +@param table user table +@param name name of the key +@param int_value value of the key to be set @return DB_SUCCESS if all OK else error code */ dberr_t -fts_config_set_ulint( -/*=================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed - FTS table */ - const char* name, /*!< in: param name */ - ulint int_value) /*!< in: value */ +fts_config_set_ulint(trx_t *trx, const dict_table_t *table, + const char *name, ulint int_value) { - dberr_t error; - fts_string_t value; - - /* We set the length of value to the max bytes it can hold. This - information is used by the callback that reads the value.*/ - value.f_len = FTS_MAX_CONFIG_VALUE_LEN; - value.f_str = static_cast(ut_malloc_nokey(value.f_len + 1)); - - ut_a(FTS_MAX_INT_LEN < FTS_MAX_CONFIG_VALUE_LEN); - - value.f_len = (ulint) snprintf( - (char*) value.f_str, FTS_MAX_INT_LEN, ULINTPF, int_value); - - error = fts_config_set_value(trx, fts_table, name, &value); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") writing `" << name << "'"; - } - - ut_free(value.f_str); - - return(error); + fts_string_t value; + /* We set the length of value to the max bytes it can hold. This + information is used by the callback that reads the value.*/ + value.f_len= FTS_MAX_CONFIG_VALUE_LEN; + value.f_str= static_cast(ut_malloc_nokey(value.f_len + 1)); + ut_a(FTS_MAX_INT_LEN < FTS_MAX_CONFIG_VALUE_LEN); + value.f_len= (ulint) snprintf((char*) value.f_str, FTS_MAX_INT_LEN, + ULINTPF, int_value); + dberr_t error= fts_config_set_value(trx, table, name, &value); + if (UNIV_UNLIKELY(error != DB_SUCCESS)) + ib::error() << "(" << error << ") writing `" << name << "'"; + ut_free(value.f_str); + return error; } diff --git a/storage/innobase/fts/fts0exec.cc b/storage/innobase/fts/fts0exec.cc new file mode 100644 index 0000000000000..8f2e1a57fceff --- /dev/null +++ b/storage/innobase/fts/fts0exec.cc @@ -0,0 +1,801 @@ +/***************************************************************************** +Copyright (c) 2025, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA +*****************************************************************************/ + +/**************************************************//** +@file fts/fts0exec.cc + +Created 2025/11/05 +*******************************************************/ + +#include "fts0exec.h" +#include "row0query.h" +#include "fts0fts.h" +#include "fts0types.h" +#include "fts0vlc.h" +#include "fts0priv.h" +#include "btr0btr.h" +#include "btr0cur.h" +#include "dict0dict.h" +#include "row0ins.h" +#include "row0upd.h" +#include "row0sel.h" +#include "eval0eval.h" +#include "que0que.h" +#include "trx0trx.h" +#include "lock0lock.h" +#include "rem0cmp.h" +#include "ha_prototypes.h" + +/** Defined in fts0fts.cc */ +extern const char* fts_common_tables[]; + +/** Find common table index by name */ +uint8_t find_common_table(const char* tbl_name) +{ + for (uint8_t i= 0; fts_common_tables[i]; i++) + if (!strcmp(tbl_name, fts_common_tables[i])) return i; + return UINT8_MAX; +} + +FTSQueryExecutor::FTSQueryExecutor( + trx_t *trx, const dict_index_t *fts_index, const dict_table_t *fts_table, + bool dict_locked) : m_executor(new QueryExecutor(trx)), + m_dict_locked(dict_locked), m_fts_index(fts_index), + m_fts_table(fts_table) +{ + for (uint8_t i = 0; i < FTS_NUM_AUX_INDEX; i++) + m_aux_tables[i] = nullptr; + + for (uint8_t i = 0; i < FTS_NUM_AUX_INDEX - 1; i++) + m_common_tables[i] = nullptr; +} + +FTSQueryExecutor::~FTSQueryExecutor() +{ + for (uint8_t i = 0; i < FTS_NUM_AUX_INDEX; i++) + if (m_aux_tables[i]) m_aux_tables[i]->release(); + + for (uint8_t i = 0; i < FTS_NUM_AUX_INDEX - 1; i++) + if (m_common_tables[i]) m_common_tables[i]->release(); + delete m_executor; +} + +dberr_t FTSQueryExecutor::open_aux_table(uint8_t aux_index) noexcept +{ + if (m_aux_tables[aux_index]) return DB_SUCCESS; + fts_table_t fts_table; + FTS_INIT_INDEX_TABLE(&fts_table, nullptr, FTS_INDEX_TABLE, m_fts_index); + fts_table.suffix= fts_get_suffix(aux_index); + + char table_name[MAX_FULL_NAME_LEN]; + fts_get_table_name(&fts_table, table_name, m_dict_locked); + + m_aux_tables[aux_index]= dict_table_open_on_name( + table_name, m_dict_locked, DICT_ERR_IGNORE_TABLESPACE); + return m_aux_tables[aux_index] ? DB_SUCCESS : DB_TABLE_NOT_FOUND; +} + +dberr_t FTSQueryExecutor::open_common_table(const char *tbl_name) noexcept +{ + uint8_t index= find_common_table(tbl_name); + if (index == UINT8_MAX) return DB_ERROR; + if (m_common_tables[index]) return DB_SUCCESS; + fts_table_t fts_table; + FTS_INIT_FTS_TABLE(&fts_table, nullptr, FTS_COMMON_TABLE, m_fts_table); + fts_table.suffix= tbl_name; + char table_name[MAX_FULL_NAME_LEN]; + fts_get_table_name(&fts_table, table_name, m_dict_locked); + + m_common_tables[index]= dict_table_open_on_name( + table_name, m_dict_locked, DICT_ERR_IGNORE_TABLESPACE); + return m_common_tables[index] ? DB_SUCCESS : DB_TABLE_NOT_FOUND; +} + +dberr_t FTSQueryExecutor::lock_aux_tables(uint8_t aux_index, + lock_mode mode) noexcept +{ + dict_table_t *table= m_aux_tables[aux_index]; + if (table == nullptr) return DB_TABLE_NOT_FOUND; + dberr_t err= m_executor->lock_table(table, mode); + if (err == DB_LOCK_WAIT) err= m_executor->handle_wait(err, true); + return err; +} + +dberr_t FTSQueryExecutor::lock_common_tables(uint8_t index, + lock_mode mode) noexcept +{ + dict_table_t *table= m_common_tables[index]; + if (table == nullptr) return DB_TABLE_NOT_FOUND; + dberr_t err = m_executor->lock_table(table, mode); + if (err == DB_LOCK_WAIT) err= m_executor->handle_wait(err, true); + return err; +} + +dberr_t FTSQueryExecutor::insert_aux_record( + uint8_t aux_index, const fts_aux_data_t* aux_data) noexcept +{ + if (aux_index >= FTS_NUM_AUX_INDEX) return DB_ERROR; + + dberr_t err= open_aux_table(aux_index); + if (err != DB_SUCCESS) return err; + err= lock_aux_tables(aux_index, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_aux_tables[aux_index]; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 7 || index->n_uniq != 2) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t fields[7]; + doc_id_t first_doc_id, last_doc_id; + + dtuple_t tuple{0, 7, 2, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 7); + /* Field 0: word (VARCHAR) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, aux_data->word, aux_data->word_len); + + /* Field 1: first_doc_id (INT) */ + field= dtuple_get_nth_field(&tuple, 1); + fts_write_doc_id(&first_doc_id, aux_data->first_doc_id); + dfield_set_data(field, &first_doc_id, sizeof(doc_id_t)); + + /* Field 2: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&tuple, 2); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 3: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&tuple, 3); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + /* Field 4: last_doc_id (UNSIGNED INT) */ + field= dtuple_get_nth_field(&tuple, 4); + fts_write_doc_id(&last_doc_id, aux_data->last_doc_id); + dfield_set_data(field, &last_doc_id, sizeof(doc_id_t)); + + /* Field 5: doc_count (UINT32_T) */ + byte doc_count[4]; + mach_write_to_4(doc_count, aux_data->doc_count); + field= dtuple_get_nth_field(&tuple, 5); + dfield_set_data(field, doc_count, sizeof(doc_count)); + + /* Field 6: ilist (VARBINARY) */ + field= dtuple_get_nth_field(&tuple, 6); + dfield_set_data(field, aux_data->ilist, aux_data->ilist_len); + + return m_executor->insert_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::insert_common_record( + const char *tbl_name, doc_id_t doc_id) noexcept +{ + dberr_t err= open_common_table(tbl_name); + if (err != DB_SUCCESS) return err; + uint8_t index_no= find_common_table(tbl_name); + err= lock_common_tables(index_no, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 3 || index->n_uniq != 1) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t fields[3]; + + dtuple_t tuple{0, 3, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 3); + /* Field 0: doc_id (INT) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id(&write_doc_id, doc_id); + dfield_set_data(field, &write_doc_id, sizeof(doc_id_t)); + + /* Field 1: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&tuple, 1); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 2: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&tuple, 2); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + return m_executor->insert_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::insert_config_record( + const char *key, const char *value) noexcept +{ + dberr_t err= open_common_table("CONFIG"); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= find_common_table("CONFIG"); + err= lock_common_tables(index_no, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 4 || index->n_uniq != 1) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t fields[4]; + + dtuple_t tuple{0, 4, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 4); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + /* Field 1: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&tuple, 1); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 2: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&tuple, 2); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + /* Field 3: value (CHAR(200)) */ + field= dtuple_get_nth_field(&tuple, 3); + dfield_set_data(field, value, strlen(value)); + + return m_executor->insert_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::update_config_record( + const char *key, const char *value) noexcept +{ + dberr_t err= open_common_table("CONFIG"); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= find_common_table("CONFIG"); + err= lock_common_tables(index_no, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + dict_index_t* index= dict_table_get_first_index(table); + + if (index->n_fields != 4 || index->n_uniq != 1) + return DB_ERROR; + + byte sys_buf[DATA_TRX_ID_LEN + DATA_ROLL_PTR_LEN]= {0}; + dfield_t search_fields[1]; + dfield_t insert_fields[4]; + + dtuple_t search_tuple{0, 1, 1, 0, search_fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&search_tuple, index, 1); + dfield_t *field= dtuple_get_nth_field(&search_tuple, 0); + dfield_set_data(field, key, strlen(key)); + + dtuple_t insert_tuple{0, 4, 1, 0, insert_fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&insert_tuple, index, 4); + + /* Field 0: key (CHAR(50)) */ + field= dtuple_get_nth_field(&insert_tuple, 0); + dfield_set_data(field, key, strlen(key)); + + /* Field 1: trx_id (DB_TRX_ID) */ + field= dtuple_get_nth_field(&insert_tuple, 1); + dfield_set_data(field, sys_buf, DATA_TRX_ID_LEN); + + /* Field 2: roll_ptr (DB_ROLL_PTR) */ + field= dtuple_get_nth_field(&insert_tuple, 2); + dfield_set_data(field, sys_buf + DATA_TRX_ID_LEN, DATA_ROLL_PTR_LEN); + + /* Field 3: value (CHAR(200)) */ + field= dtuple_get_nth_field(&insert_tuple, 3); + dfield_set_data(field, value, strlen(value)); + + upd_field_t upd_field; + upd_field.field_no = 3; + upd_field.orig_len = 0; + upd_field.exp = nullptr; + dfield_set_data(&upd_field.new_val, value, strlen(value)); + dict_col_copy_type(dict_index_get_nth_col(index, 3), + dfield_get_type(&upd_field.new_val)); + + upd_t update; + update.heap = nullptr; + update.info_bits = 0; + update.old_vrow = nullptr; + update.n_fields = 1; + update.fields = &upd_field; + + return m_executor->replace_record(table, &search_tuple, &update, + &insert_tuple); +} + +dberr_t FTSQueryExecutor::delete_aux_record( + uint8_t aux_index, const fts_aux_data_t* aux_data) noexcept +{ + if (aux_index >= FTS_NUM_AUX_INDEX) return DB_ERROR; + + dberr_t err= open_aux_table(aux_index); + if (err != DB_SUCCESS) return err; + err= lock_aux_tables(aux_index, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_aux_tables[aux_index]; + dict_index_t* index= dict_table_get_first_index(table); + + if (dict_table_get_next_index(index) != nullptr) + return DB_ERROR; + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: word (VARCHAR) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, aux_data->word, aux_data->word_len); + + return m_executor->delete_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::delete_common_record( + const char *table_name, doc_id_t doc_id) noexcept +{ + dberr_t err= open_common_table(table_name); + if (err != DB_SUCCESS) return err; + + uint8_t cached_index= find_common_table(table_name); + err= lock_common_tables(cached_index, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[cached_index]; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: doc_id */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id(&write_doc_id, doc_id); + dfield_set_data(field, &write_doc_id, sizeof(doc_id_t)); + + return m_executor->delete_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::delete_all_common_records( + const char *table_name) noexcept +{ + dberr_t err= open_common_table(table_name); + if (err != DB_SUCCESS) return err; + + uint8_t cached_index= find_common_table(table_name); + err= lock_common_tables(cached_index, LOCK_X); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[cached_index]; + return m_executor->delete_all(table); +} + +dberr_t FTSQueryExecutor::delete_config_record( + const char *key) noexcept +{ + dberr_t err= open_common_table("CONFIG"); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= find_common_table("CONFIG"); + err= lock_common_tables(index_no, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + return m_executor->delete_record(table, &tuple); +} + +dberr_t FTSQueryExecutor::read_all_config(RecordCallback& callback) noexcept +{ + dberr_t err= open_common_table("CONFIG"); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= find_common_table("CONFIG"); + err= lock_common_tables(index_no, LOCK_IS); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + return m_executor->read(table, nullptr, PAGE_CUR_GE, callback); +} + +dberr_t FTSQueryExecutor::read_config(const char *key, + RecordCallback& callback) noexcept +{ + dberr_t err= open_common_table("CONFIG"); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= find_common_table("CONFIG"); + err= lock_common_tables(index_no, LOCK_IS); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + return m_executor->read(table, &tuple, PAGE_CUR_GE, callback); +} + +dberr_t FTSQueryExecutor::read_config_with_lock(const char *key, + RecordCallback& callback) noexcept +{ + dberr_t err= open_common_table("CONFIG"); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= find_common_table("CONFIG"); + + err= lock_common_tables(index_no, LOCK_IX); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: key (CHAR(50)) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, key, strlen(key)); + + return m_executor->select_for_update(table, &tuple, &callback); +} + +dberr_t FTSQueryExecutor::read_aux(uint8_t aux_index, + const char *word, + page_cur_mode_t mode, + RecordCallback& callback) noexcept +{ + if (aux_index >= FTS_NUM_AUX_INDEX) return DB_ERROR; + dberr_t err= open_aux_table(aux_index); + if (err != DB_SUCCESS) return err; + + err= lock_aux_tables(aux_index, LOCK_IS); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_aux_tables[aux_index]; + dict_index_t* index= dict_table_get_first_index(table); + + dfield_t fields[1]; + dtuple_t tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&tuple, index, 1); + /* Field 0: word (VARCHAR) */ + dfield_t *field= dtuple_get_nth_field(&tuple, 0); + dfield_set_data(field, word, strlen(word)); + + return m_executor->read(table, &tuple, mode, callback); +} + +dberr_t FTSQueryExecutor::read_aux_all(uint8_t aux_index, RecordCallback& callback) noexcept +{ + if (aux_index >= FTS_NUM_AUX_INDEX) return DB_ERROR; + dberr_t err= open_aux_table(aux_index); + if (err != DB_SUCCESS) return err; + + err= lock_aux_tables(aux_index, LOCK_IS); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_aux_tables[aux_index]; + return m_executor->read(table, nullptr, PAGE_CUR_GE, callback); +} + +dberr_t FTSQueryExecutor::read_all_common(const char *tbl_name, + RecordCallback& callback) noexcept +{ + dberr_t err= open_common_table(tbl_name); + if (err != DB_SUCCESS) return err; + + uint8_t index_no= find_common_table(tbl_name); + err= lock_common_tables(index_no, LOCK_IS); + if (err != DB_SUCCESS) return err; + + dict_table_t* table= m_common_tables[index_no]; + return m_executor->read(table, nullptr, PAGE_CUR_GE, callback); +} + +CommonTableReader::CommonTableReader() : RecordCallback( + [this](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> bool + { + ulint len; + const byte* id_data= rec_get_nth_field(rec, offsets, 0, &len); + if (id_data && len != UNIV_SQL_NULL && len == 8) + { + doc_id_t doc_id= mach_read_from_8(id_data); + doc_ids.push_back(doc_id); + } + return true; + }, + [](const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index, const rec_offs* offsets) -> RecordCompareAction + { return RecordCompareAction::PROCESS; }) {} + + +ConfigReader::ConfigReader() : RecordCallback( + [this](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> bool + { + ulint key_len, value_len; + const byte* key_data= rec_get_nth_field(rec, offsets, 0, &key_len); + const byte* value_data= rec_get_nth_field(rec, offsets, 3, &value_len); + + if (key_data && value_data && key_len != UNIV_SQL_NULL && + value_len != UNIV_SQL_NULL) + { + std::string key(reinterpret_cast(key_data), key_len); + std::string value(reinterpret_cast(value_data), value_len); + config_pairs.emplace_back(key, value); + } + return true; + }, + [](const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index, const rec_offs* offsets) -> RecordCompareAction + { + if (!search_tuple) return RecordCompareAction::PROCESS; + uint16_t matched_fields= 0; + int cmp_result= cmp_dtuple_rec_with_match(search_tuple, rec, index, + offsets, &matched_fields); + return (cmp_result == 0) ? RecordCompareAction::PROCESS + : RecordCompareAction::STOP; + }) {} + +/** Initial size of nodes in fts_word_t. */ +static const ulint FTS_WORD_NODES_INIT_SIZE = 64; + +/** Initialize fts_word_t structure */ +static void init_fts_word(fts_word_t* word, const byte* utf8, ulint len) +{ + mem_heap_t* heap = mem_heap_create(sizeof(fts_node_t)); + memset(word, 0, sizeof(*word)); + word->text.f_len = len; + word->text.f_str = static_cast(mem_heap_alloc(heap, len + 1)); + memcpy(word->text.f_str, utf8, len); + word->text.f_str[len] = 0; + word->heap_alloc = ib_heap_allocator_create(heap); + word->nodes = ib_vector_create(word->heap_alloc, sizeof(fts_node_t), + FTS_WORD_NODES_INIT_SIZE); +} + +/** AuxRecordReader default word processor implementation */ +bool AuxRecordReader::default_word_processor( + const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets, void* user_arg) +{ + ib_vector_t *words= static_cast(user_arg); + ulint word_len; + const byte *word_data= rec_get_nth_field(rec, offsets, 0, &word_len); + fts_word_t *word; + bool is_word_init = false; + if (!word_data || word_len == UNIV_SQL_NULL || word_len > FTS_MAX_WORD_LEN) + return true; + + ut_ad(word_len <= FTS_MAX_WORD_LEN); + + if (ib_vector_size(words) == 0) + { + /* First word - push and initialize */ + word = static_cast(ib_vector_push(words, nullptr)); + init_fts_word(word, word_data, word_len); + is_word_init = true; + } + else + { + /* Check if this word is different from the last word */ + word = static_cast(ib_vector_last(words)); + if (word_len != word->text.f_len || + memcmp(word->text.f_str, word_data, word_len)) + { + /* Different word - push new word and initialize */ + word = static_cast(ib_vector_push(words, nullptr)); + init_fts_word(word, word_data, word_len); + is_word_init = true; + } + } + fts_node_t *node= static_cast( + ib_vector_push(word->nodes, nullptr)); + + ulint doc_id_len; + const byte *doc_id_data= rec_get_nth_field(rec, offsets, 1, &doc_id_len); + if (doc_id_data && doc_id_len == 8) + node->first_doc_id= fts_read_doc_id(doc_id_data); + else node->first_doc_id= 0; + + /* Read last_doc_id (field 4) */ + doc_id_data= rec_get_nth_field(rec, offsets, 4, &doc_id_len); + if (doc_id_data && doc_id_len == 8) + node->last_doc_id= fts_read_doc_id(doc_id_data); + else node->last_doc_id= 0; + + /* Read doc_count (field 5) */ + ulint doc_count_len; + const byte *doc_count_data= rec_get_nth_field(rec, offsets, 5, + &doc_count_len); + if (doc_count_data && doc_count_len == 4) + node->doc_count= mach_read_from_4(doc_count_data); + else node->doc_count= 0; + + /* Read ilist (field 6) with external BLOB support */ + ulint ilist_len= 0; + const byte *ilist_data= rec_get_nth_field(rec, offsets, 6, &ilist_len); + byte *external_data= nullptr; + mem_heap_t *temp_heap= nullptr; + + node->ilist_size_alloc= node->ilist_size= 0; + node->ilist= nullptr; + + if (ilist_data && ilist_len != UNIV_SQL_NULL && ilist_len > 0) + { + if (rec_offs_nth_extern(offsets, 6)) + { + temp_heap= mem_heap_create(ilist_len); + ulint external_len; + external_data= btr_copy_externally_stored_field( + &external_len, ilist_data, index->table->space->zip_size(), + ilist_len, temp_heap); + if (external_data) + { + ilist_data= external_data; + ilist_len= external_len; + } + } + node->ilist_size_alloc= node->ilist_size= ilist_len; + if (ilist_len) + { + node->ilist= static_cast(ut_malloc_nokey(ilist_len)); + memcpy(node->ilist, ilist_data, ilist_len); + } + if (temp_heap) mem_heap_free(temp_heap); + if (ilist_len == 0) return false; + } + + if (this->total_memory) + { + if (is_word_init) + { + *this->total_memory+= + sizeof(fts_word_t) + sizeof(ib_alloc_t) + + sizeof(ib_vector_t) + word_len + + sizeof(fts_node_t) * FTS_WORD_NODES_INIT_SIZE; + } + *this->total_memory += node->ilist_size; + if (*this->total_memory >= fts_result_cache_limit) + return false; + } + return true; +} + +/** AuxRecordReader comparison logic implementation */ +RecordCompareAction AuxRecordReader::compare_record( + const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index, const rec_offs* offsets) noexcept +{ + if (!search_tuple) return RecordCompareAction::PROCESS; + const dfield_t* search_field= dtuple_get_nth_field(search_tuple, 0); + const void* search_data= dfield_get_data(search_field); + ulint search_len= dfield_get_len(search_field); + + ulint rec_len; + const byte* rec_data= rec_get_nth_field(rec, offsets, 0, &rec_len); + + if (!rec_data || rec_len == UNIV_SQL_NULL) + return RecordCompareAction::SKIP; + if (!search_data || search_len == UNIV_SQL_NULL) + return RecordCompareAction::PROCESS; + int cmp_result; + switch (compare_mode) + { + case AuxCompareMode::GREATER_EQUAL: + case AuxCompareMode::GREATER: + { + uint16_t matched_fields= 0; + cmp_result= cmp_dtuple_rec_with_match(search_tuple, rec, index, + offsets, &matched_fields); + if (compare_mode == AuxCompareMode::GREATER_EQUAL) + return (cmp_result <= 0) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; + else + return (cmp_result < 0) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; + } + case AuxCompareMode::LIKE: + case AuxCompareMode::EQUAL: + { + /* Use charset-aware comparison for LIKE and EQUAL modes */ + const dtype_t* type= dfield_get_type(search_field); + cmp_result= cmp_data(type->mtype, type->prtype, false, + static_cast(search_data), + search_len, rec_data, rec_len); + + if (compare_mode == AuxCompareMode::EQUAL) + return cmp_result == 0 + ? RecordCompareAction::PROCESS + : RecordCompareAction::STOP; + else /* AuxCompareMode::LIKE */ + { + /* For LIKE mode, compare only the prefix (search_len bytes) */ + int prefix_cmp = cmp_data(type->mtype, type->prtype, false, + static_cast(search_data), + search_len, rec_data, + search_len <= rec_len ? search_len : rec_len); + + if (prefix_cmp != 0) return RecordCompareAction::STOP; + return (search_len <= rec_len) ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; + } + } + } + return RecordCompareAction::PROCESS; +} diff --git a/storage/innobase/fts/fts0fts.cc b/storage/innobase/fts/fts0fts.cc index ad7df21504fdf..2a84b5d6dab98 100644 --- a/storage/innobase/fts/fts0fts.cc +++ b/storage/innobase/fts/fts0fts.cc @@ -30,6 +30,7 @@ Full Text Search interface #include "dict0stats_bg.h" #include "row0sel.h" #include "fts0fts.h" +#include "fts0exec.h" #include "fts0priv.h" #include "fts0types.h" #include "fts0types.inl" @@ -44,6 +45,37 @@ static const ulint FTS_MAX_ID_LEN = 32; /** Column name from the FTS config table */ #define FTS_MAX_CACHE_SIZE_IN_MB "cache_size_in_mb" +/** Compare function to check if record's doc_id > search tuple's doc_id +@param[in] search_tuple Search tuple containing target doc_id +@param[in] rec Record to check +@param[in] index Index containing the record +@param[in] offsets Record offsets +@return true if record's doc_id > search tuple's doc_id */ +static +RecordCompareAction doc_id_comparator( + const dtuple_t* search_tuple, + const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) +{ + /* Get target doc_id from search tuple */ + const dfield_t* search_field= dtuple_get_nth_field(search_tuple, 0); + const byte* search_data= static_cast(dfield_get_data(search_field)); + doc_id_t target_doc_id= fts_read_doc_id(search_data); + + /* Get doc_id from record */ + ulint len; + const byte* rec_data= rec_get_nth_field(rec, offsets, 0, &len); + + if (len != sizeof(doc_id_t)) + return RecordCompareAction::SKIP; + + doc_id_t rec_doc_id= fts_read_doc_id(rec_data); + return (rec_doc_id > target_doc_id) + ? RecordCompareAction::PROCESS + : RecordCompareAction::SKIP; +} + /** Verify if a aux table name is a obsolete table by looking up the key word in the obsolete table names */ #define FTS_IS_OBSOLETE_AUX_TABLE(table_name) \ @@ -156,27 +188,6 @@ const fts_index_selector_t fts_index_selector[] = { { 0 , NULL } }; -/** Default config values for FTS indexes on a table. */ -static const char* fts_config_table_insert_values_sql = - "PROCEDURE P() IS\n" - "BEGIN\n" - "\n" - "INSERT INTO $config_table VALUES('" - FTS_MAX_CACHE_SIZE_IN_MB "', '256');\n" - "" - "INSERT INTO $config_table VALUES('" - FTS_OPTIMIZE_LIMIT_IN_SECS "', '180');\n" - "" - "INSERT INTO $config_table VALUES ('" - FTS_SYNCED_DOC_ID "', '0');\n" - "" - "INSERT INTO $config_table VALUES ('" - FTS_TOTAL_DELETED_COUNT "', '0');\n" - "" /* Note: 0 == FTS_TABLE_STATE_RUNNING */ - "INSERT INTO $config_table VALUES ('" - FTS_TABLE_STATE "', '0');\n" - "END;\n"; - /** FTS tokenize parameter for plugin parser */ struct fts_tokenize_param_t { fts_doc_t* result_doc; /*!< Result doc for tokens */ @@ -225,30 +236,6 @@ fts_add_doc_by_id( fts_trx_table_t*ftt, /*!< in: FTS trx table */ doc_id_t doc_id); /*!< in: doc id */ -/** Tokenize a document. -@param[in,out] doc document to tokenize -@param[out] result tokenization result -@param[in] parser pluggable parser */ -static -void -fts_tokenize_document( - fts_doc_t* doc, - fts_doc_t* result, - st_mysql_ftparser* parser); - -/** Continue to tokenize a document. -@param[in,out] doc document to tokenize -@param[in] add_pos add this position to all tokens from this tokenization -@param[out] result tokenization result -@param[in] parser pluggable parser */ -static -void -fts_tokenize_document_next( - fts_doc_t* doc, - ulint add_pos, - fts_doc_t* result, - st_mysql_ftparser* parser); - /** Create the vector of fts_get_doc_t instances. @param[in,out] cache fts cache @return vector of fts_get_doc_t instances */ @@ -283,7 +270,6 @@ fts_cache_destroy(fts_cache_t* cache) /** Get a character set based on precise type. @param prtype precise type @return the corresponding character set */ -UNIV_INLINE CHARSET_INFO* fts_get_charset(ulint prtype) { @@ -360,187 +346,154 @@ fts_load_default_stopword( stopword_info->status = STOPWORD_FROM_DEFAULT; } -/****************************************************************//** -Callback function to read a single stopword value. -@return Always return TRUE */ -static -ibool -fts_read_stopword( -/*==============*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - ib_alloc_t* allocator; - fts_stopword_t* stopword_info; - sel_node_t* sel_node; - que_node_t* exp; - ib_rbt_t* stop_words; - dfield_t* dfield; - fts_string_t str; - mem_heap_t* heap; - ib_rbt_bound_t parent; - dict_table_t* table; - - sel_node = static_cast(row); - table = sel_node->table_list->table; - stopword_info = static_cast(user_arg); - - stop_words = stopword_info->cached_stopword; - allocator = static_cast(stopword_info->heap); - heap = static_cast(allocator->arg); - - exp = sel_node->select_list; - - /* We only need to read the first column */ - dfield = que_node_get_val(exp); - - str.f_n_char = 0; - str.f_str = static_cast(dfield_get_data(dfield)); - str.f_len = dfield_get_len(dfield); - exp = que_node_get_next(exp); - ut_ad(exp); - - if (table->versioned()) { - dfield = que_node_get_val(exp); - ut_ad(dfield_get_type(dfield)->vers_sys_end()); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - if (table->versioned_by_id()) { - ut_ad(len == sizeof trx_id_max_bytes); - if (0 != memcmp(data, trx_id_max_bytes, len)) { - return true; - } - } else { - ut_ad(len == sizeof timestamp_max_bytes); - if (!IS_MAX_TIMESTAMP(data)) { - return true; - } - } - } - ut_ad(!que_node_get_next(exp)); - - /* Only create new node if it is a value not already existed */ - if (str.f_len != UNIV_SQL_NULL - && rbt_search(stop_words, &parent, &str) != 0) { - - fts_tokenizer_word_t new_word; - - new_word.nodes = ib_vector_create( - allocator, sizeof(fts_node_t), 4); - - new_word.text.f_str = static_cast( - mem_heap_alloc(heap, str.f_len + 1)); - - memcpy(new_word.text.f_str, str.f_str, str.f_len); - - new_word.text.f_n_char = 0; - new_word.text.f_len = str.f_len; - new_word.text.f_str[str.f_len] = 0; - - rbt_insert(stop_words, &new_word, &new_word); - } - - return(TRUE); -} - -/******************************************************************//** -Load user defined stopword from designated user table +/** Load user defined stopword from designated user table +@param fts fulltext structure +@param stopword_table stopword table +@param stopword_info stopword information @return whether the operation is successful */ static -bool -fts_load_user_stopword( -/*===================*/ - fts_t* fts, /*!< in: FTS struct */ - const char* stopword_table_name, /*!< in: Stopword table - name */ - fts_stopword_t* stopword_info) /*!< in: Stopword info */ -{ - if (!fts->dict_locked) { - dict_sys.lock(SRW_LOCK_CALL); - } - - /* Validate the user table existence in the right format */ - bool ret= false; - const char* row_end; - stopword_info->charset = fts_valid_stopword_table(stopword_table_name, - &row_end); - if (!stopword_info->charset) { +bool fts_load_user_stopword(fts_t *fts, const char *stopword_table, + fts_stopword_t *stopword_info) +{ + if (!fts->dict_locked) dict_sys.lock(SRW_LOCK_CALL); + /* Validate the user table existence in the right format */ + bool ret= false; + const char* row_end; + stopword_info->charset= fts_valid_stopword_table( + stopword_table, &row_end); + if (!stopword_info->charset) + { cleanup: - if (!fts->dict_locked) { - dict_sys.unlock(); - } - - return ret; - } + if (!fts->dict_locked) dict_sys.unlock(); + return ret; + } - trx_t* trx = trx_create(); - trx->op_info = "Load user stopword table into FTS cache"; + if (!stopword_info->cached_stopword) + { + /* Create the stopword RB tree with the stopword column + charset. All comparison will use this charset */ + stopword_info->cached_stopword= rbt_create_arg_cmp( + sizeof(fts_tokenizer_word_t), innobase_fts_text_cmp, + (void*)stopword_info->charset); + } - if (!stopword_info->cached_stopword) { - /* Create the stopword RB tree with the stopword column - charset. All comparison will use this charset */ - stopword_info->cached_stopword = rbt_create_arg_cmp( - sizeof(fts_tokenizer_word_t), innobase_fts_text_cmp, - (void*)stopword_info->charset); + /* Load the stopword table */ + dict_table_t* table= + dict_sys.load_table({stopword_table, strlen(stopword_table)}); + if (!table) + goto cleanup; - } + trx_t *trx= trx_create(); + trx->op_info= "Load user stopword table into FTS cache"; - pars_info_t* info = pars_info_create(); + QueryExecutor executor(trx); + ib_rbt_t* stop_words= stopword_info->cached_stopword; + ib_alloc_t* allocator= static_cast(stopword_info->heap); + mem_heap_t* heap= static_cast(allocator->arg); - pars_info_bind_id(info, "table_stopword", stopword_table_name); - pars_info_bind_id(info, "row_end", row_end); + /* Find the field number for 'value' column */ + dict_index_t* clust_index= dict_table_get_first_index(table); + ulint value_field_no= ULINT_UNDEFINED; + for (ulint i= 0; i < dict_index_get_n_fields(clust_index); i++) + { + const dict_field_t* field= dict_index_get_nth_field(clust_index, i); + if (strcmp(field->name, "value") == 0) + { + value_field_no= i; + break; + } + } + if (value_field_no == ULINT_UNDEFINED) + { + ib::error() << "Could not find 'value' column in stopword table " + << stopword_table; + goto cleanup; + } - pars_info_bind_function(info, "my_func", fts_read_stopword, - stopword_info); + auto process_stopword= [&](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> bool + { + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, value_field_no, + &field_len); - que_t* graph = pars_sql( - info, - "PROCEDURE P() IS\n" - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT value, $row_end" - " FROM $table_stopword;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;" - "END;\n"); + if (field_len == UNIV_SQL_NULL) return true; - for (;;) { - dberr_t error = fts_eval_sql(trx, graph); + fts_string_t str; + str.f_n_char= 0; + str.f_str= const_cast(field_data); + str.f_len= field_len; - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - stopword_info->status = STOPWORD_USER_TABLE; - break; - } else { - fts_sql_rollback(trx); + /* Handle system versioning - check row_end column if versioned */ + if (table->versioned()) + { + ulint end_len; + const byte* end_data= rec_get_nth_field( + rec, offsets, table->vers_end + index->n_uniq + 2, &end_len); - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout reading user" - " stopword table. Retrying!"; + if (table->versioned_by_id()) + { + ut_ad(end_len == sizeof trx_id_max_bytes); + if (0 != memcmp(end_data, trx_id_max_bytes, end_len)) + return true; /* Skip this version, continue processing */ + } + else + { + ut_ad(end_len == sizeof timestamp_max_bytes); + if (!IS_MAX_TIMESTAMP(end_data)) + return true; /* Skip this version, continue processing */ + } + } - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "Error '" << error - << "' while reading user stopword" - " table."; - ret = FALSE; - break; - } - } - } + ib_rbt_bound_t parent; + if (str.f_len != UNIV_SQL_NULL && + rbt_search(stop_words, &parent, &str) != 0) + { + fts_tokenizer_word_t new_word; + new_word.nodes= ib_vector_create(allocator, sizeof(fts_node_t), 4); + new_word.text.f_str= static_cast( + mem_heap_alloc(heap, str.f_len + 1)); + memcpy(new_word.text.f_str, str.f_str, str.f_len); + new_word.text.f_n_char= 0; + new_word.text.f_len= str.f_len; + new_word.text.f_str[str.f_len]= 0; + rbt_insert(stop_words, &new_word, &new_word); + } + return true; /* Continue processing */ + }; - que_graph_free(graph); - trx->free(); - ret = true; - goto cleanup; + RecordCallback callback(process_stopword); + /* Read all records from the stopword table */ + for (;;) + { + dberr_t error= executor.read(table, nullptr, PAGE_CUR_G, callback); + if (UNIV_LIKELY(error == DB_SUCCESS)) + { + fts_sql_commit(trx); + stopword_info->status= STOPWORD_USER_TABLE; + ret= true; + break; + } + else + { + fts_sql_rollback(trx); + if (error == DB_LOCK_WAIT_TIMEOUT) + { + ib::warn() << "Lock wait timeout reading user" + " stopword table. Retrying!"; + trx->error_state= DB_SUCCESS; + } + else + { + ib::error() << "Error '" << error + << "' while reading user stopword table."; + ret= false; + break; + } + } + } + trx->free(); + goto cleanup; } /******************************************************************//** @@ -604,6 +557,46 @@ fts_cache_init( } } +/** Construct the name of an internal FTS table for the given table. +@param[in] fts_table metadata on fulltext-indexed table +@param[out] table_name a name up to MAX_FULL_NAME_LEN +@param[in] dict_locked whether dict_sys.latch is being held */ +void fts_get_table_name(const fts_table_t* fts_table, char* table_name, + bool dict_locked) +{ + if (!dict_locked) dict_sys.freeze(SRW_LOCK_CALL); + ut_ad(dict_sys.frozen()); + /* Include the separator as well. */ + const size_t dbname_len= fts_table->table->name.dblen() + 1; + ut_ad(dbname_len > 1); + memcpy(table_name, fts_table->table->name.m_name, dbname_len); + if (!dict_locked) dict_sys.unfreeze(); + + memcpy(table_name += dbname_len, "FTS_", 4); + table_name += 4; + int len; + switch (fts_table->type) + { + case FTS_COMMON_TABLE: + len= fts_write_object_id(fts_table->table_id, table_name); + break; + + case FTS_INDEX_TABLE: + len= fts_write_object_id(fts_table->table_id, table_name); + table_name[len]= '_'; + ++len; + len+= fts_write_object_id(fts_table->index_id, table_name + len); + break; + + default: ut_error; + } + ut_a(len >= 16); + ut_a(len < FTS_AUX_MIN_TABLE_ID_LENGTH); + table_name+= len; + *table_name++= '_'; + strcpy(table_name, fts_table->suffix); +} + /****************************************************************//** Create a FTS cache. */ fts_cache_t* @@ -1892,88 +1885,74 @@ CREATE TABLE $FTS_PREFIX_CONFIG @param[in] skip_doc_id_index Skip index on doc id @return DB_SUCCESS if succeed */ dberr_t -fts_create_common_tables( - trx_t* trx, - dict_table_t* table, - bool skip_doc_id_index) +fts_create_common_tables(trx_t *trx, dict_table_t *table, + bool skip_doc_id_index) { - dberr_t error; - que_t* graph; - fts_table_t fts_table; - mem_heap_t* heap = mem_heap_create(1024); - pars_info_t* info; - char fts_name[MAX_FULL_NAME_LEN]; - char full_name[sizeof(fts_common_tables) / sizeof(char*)] - [MAX_FULL_NAME_LEN]; - - dict_index_t* index = NULL; - - FTS_INIT_FTS_TABLE(&fts_table, NULL, FTS_COMMON_TABLE, table); - - error = fts_drop_common_tables(trx, &fts_table, true); - - if (error != DB_SUCCESS) { - - goto func_exit; - } - - /* Create the FTS tables that are common to an FTS index. */ - for (ulint i = 0; fts_common_tables[i] != NULL; ++i) { - - fts_table.suffix = fts_common_tables[i]; - fts_get_table_name(&fts_table, full_name[i], true); - dict_table_t* common_table = fts_create_one_common_table( - trx, table, full_name[i], fts_table.suffix, heap); - - if (!common_table) { - trx->error_state = DB_SUCCESS; - error = DB_ERROR; - goto func_exit; - } - - mem_heap_empty(heap); - } - - /* Write the default settings to the config table. */ - info = pars_info_create(); - - fts_table.suffix = "CONFIG"; - fts_get_table_name(&fts_table, fts_name, true); - pars_info_bind_id(info, "config_table", fts_name); + dberr_t error= DB_SUCCESS; + char full_name[sizeof(fts_common_tables) / sizeof(char*)][MAX_FULL_NAME_LEN]; + dict_index_t *index= nullptr; + fts_table_t fts_table; + FTS_INIT_FTS_TABLE(&fts_table, NULL, FTS_COMMON_TABLE, table); + error = fts_drop_common_tables(trx, &fts_table, true); + if (error != DB_SUCCESS) return error; + mem_heap_t *heap= mem_heap_create(1024); + /* Create the FTS tables that are common to an FTS index. */ + for (ulint i = 0; fts_common_tables[i] != NULL; ++i) + { + fts_table.suffix = fts_common_tables[i]; + fts_get_table_name(&fts_table, full_name[i], true); + dict_table_t *common_table= fts_create_one_common_table( + trx, table, full_name[i], fts_table.suffix, heap); + if (!common_table) + { + trx->error_state = DB_SUCCESS; + error = DB_ERROR; + mem_heap_free(heap); + return error; + } + mem_heap_empty(heap); + } - graph = pars_sql( - info, fts_config_table_insert_values_sql); + FTSQueryExecutor executor(trx, nullptr, table, true); - error = fts_eval_sql(trx, graph); + /** Does the following insert operation: + INSERT INTO $config_table VALUES('"FTS_MAX_CACHE_SIZE_IN_MB"', '256');" + INSERT INTO $config_table VALUES('"FTS_OPTIMIZE_LIMIT_IN_SECS"', '180');" + INSERT INTO $config_table VALUES ('"FTS_SYNCED_DOC_ID "', '0');" + INSERT INTO $config_table VALUES ('"FTS_TOTAL_DELETED_COUNT "', '0');" + INSERT INTO $config_table VALUES ('"FTS_TABLE_STATE "', '0');" */ + error= executor.insert_config_record(FTS_MAX_CACHE_SIZE_IN_MB, "256"); + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_OPTIMIZE_LIMIT_IN_SECS, "180"); - que_graph_free(graph); + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_SYNCED_DOC_ID, "0"); - if (error != DB_SUCCESS || skip_doc_id_index) { + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_TOTAL_DELETED_COUNT, "0"); - goto func_exit; - } + if (error == DB_SUCCESS) + error= executor.insert_config_record(FTS_TABLE_STATE, "0"); - if (table->versioned()) { - index = dict_mem_index_create(table, - FTS_DOC_ID_INDEX.str, - DICT_UNIQUE, 2); - dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); - dict_mem_index_add_field(index, table->cols[table->vers_end].name(*table).str, 0); - } else { - index = dict_mem_index_create(table, - FTS_DOC_ID_INDEX.str, - DICT_UNIQUE, 1); - dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); - } + if (error != DB_SUCCESS || skip_doc_id_index) goto func_exit; - error = row_create_index_for_mysql(index, trx, NULL, - FIL_ENCRYPTION_DEFAULT, - FIL_DEFAULT_ENCRYPTION_KEY); + if (table->versioned()) + { + index= dict_mem_index_create(table, FTS_DOC_ID_INDEX.str, DICT_UNIQUE, 2); + dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); + dict_mem_index_add_field(index, table->cols[table->vers_end].name(*table).str, 0); + } + else + { + index= dict_mem_index_create(table, FTS_DOC_ID_INDEX.str, DICT_UNIQUE, 1); + dict_mem_index_add_field(index, FTS_DOC_ID.str, 0); + } + error= row_create_index_for_mysql(index, trx, NULL, FIL_ENCRYPTION_DEFAULT, + FIL_DEFAULT_ENCRYPTION_KEY); func_exit: - mem_heap_free(heap); - - return(error); + mem_heap_free(heap); + return error; } /** Create one FTS auxiliary index table for an FTS index. @@ -2438,39 +2417,6 @@ fts_trx_add_op( fts_trx_table_add_op(stmt_ftt, doc_id, state, fts_indexes); } -/******************************************************************//** -Fetch callback that converts a textual document id to a binary value and -stores it in the given place. -@return always returns NULL */ -static -ibool -fts_fetch_store_doc_id( -/*===================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: doc_id_t* to store - doc_id in */ -{ - int n_parsed; - sel_node_t* node = static_cast(row); - doc_id_t* doc_id = static_cast(user_arg); - dfield_t* dfield = que_node_get_val(node->select_list); - dtype_t* type = dfield_get_type(dfield); - ulint len = dfield_get_len(dfield); - - char buf[32]; - - ut_a(dtype_get_mtype(type) == DATA_VARCHAR); - ut_a(len > 0 && len < sizeof(buf)); - - memcpy(buf, dfield_get_data(dfield), len); - buf[len] = '\0'; - - n_parsed = sscanf(buf, FTS_DOC_ID_FORMAT, doc_id); - ut_a(n_parsed == 1); - - return(FALSE); -} - #ifdef FTS_CACHE_SIZE_DEBUG /******************************************************************//** Get the max cache size in bytes. If there is an error reading the @@ -2482,7 +2428,7 @@ ulint fts_get_max_cache_size( /*===================*/ trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table) /*!< in: table instance */ + const dict_table_t *table) /*!< in: table */ { dberr_t error; fts_string_t value; @@ -2498,7 +2444,7 @@ fts_get_max_cache_size( value.f_str = ut_malloc_nokey(value.f_len + 1); error = fts_config_get_value( - trx, fts_table, FTS_MAX_CACHE_SIZE_IN_MB, &value); + trx, table, FTS_MAX_CACHE_SIZE_IN_MB, &value); if (UNIV_LIKELY(error == DB_SUCCESS)) { value.f_str[value.f_len] = 0; @@ -2582,43 +2528,22 @@ dberr_t fts_read_synced_doc_id(const dict_table_t *table, doc_id_t *doc_id, trx_t *trx) { - dberr_t error; - char table_name[MAX_FULL_NAME_LEN]; - - fts_table_t fts_table; - fts_table.suffix= "CONFIG"; - fts_table.table_id= table->id; - fts_table.type= FTS_COMMON_TABLE; - fts_table.table= table; ut_a(table->fts->doc_col != ULINT_UNDEFINED); - - trx->op_info = "update the next FTS document id"; - pars_info_t *info= pars_info_create(); - pars_info_bind_function(info, "my_func", fts_fetch_store_doc_id, - doc_id); - - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "config_table", table_name); - - que_t *graph= fts_parse_sql( - &fts_table, info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS SELECT value FROM $config_table" - " WHERE key = 'synced_doc_id' FOR UPDATE;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - + trx->op_info= "reading synced FTS document id"; + FTSQueryExecutor executor(trx, nullptr, table); + ConfigReader reader; *doc_id= 0; - error = fts_eval_sql(trx, graph); - que_graph_free(graph); + dberr_t error= executor.read_config_with_lock("synced_doc_id", reader); + if (error == DB_SUCCESS) + { + const auto& config_pairs= reader.config_pairs; + if (config_pairs.size() == 1) + { + const std::string& value_str= config_pairs[0].second; + int n_parsed= sscanf(value_str.c_str(), FTS_DOC_ID_FORMAT, doc_id); + if (n_parsed != 1) error= DB_ERROR; + } + } return error; } @@ -2711,74 +2636,42 @@ transaction to update the last document id. @param trx update trx or null @retval DB_SUCCESS if OK */ dberr_t -fts_update_sync_doc_id( - const dict_table_t* table, - doc_id_t doc_id, - trx_t* trx) +fts_update_sync_doc_id(const dict_table_t *table, doc_id_t doc_id, trx_t *trx) { - byte id[FTS_MAX_ID_LEN]; - pars_info_t* info; - fts_table_t fts_table; - ulint id_len; - que_t* graph = NULL; - dberr_t error; - ibool local_trx = FALSE; - fts_cache_t* cache = table->fts->cache; - char fts_name[MAX_FULL_NAME_LEN]; - - if (srv_read_only_mode) { - return DB_READ_ONLY; - } - - fts_table.suffix = "CONFIG"; - fts_table.table_id = table->id; - fts_table.type = FTS_COMMON_TABLE; - fts_table.table = table; - - if (!trx) { - trx = trx_create(); - trx_start_internal(trx); - - trx->op_info = "setting last FTS document id"; - local_trx = TRUE; - } - - info = pars_info_create(); - - id_len = (ulint) snprintf( - (char*) id, sizeof(id), FTS_DOC_ID_FORMAT, doc_id + 1); - - pars_info_bind_varchar_literal(info, "doc_id", id, id_len); - - fts_get_table_name(&fts_table, fts_name, - table->fts->dict_locked); - pars_info_bind_id(info, "table_name", fts_name); - - graph = fts_parse_sql( - &fts_table, info, - "BEGIN" - " UPDATE $table_name SET value = :doc_id" - " WHERE key = 'synced_doc_id';"); - - error = fts_eval_sql(trx, graph); - - que_graph_free(graph); - - if (local_trx) { - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - cache->synced_doc_id = doc_id; - } else { - ib::error() << "(" << error << ") while" - " updating last doc id for table" - << table->name; + if (srv_read_only_mode) return DB_READ_ONLY; + bool local_trx= false; + if (!trx) + { + trx= trx_create(); + trx_start_internal(trx); + trx->op_info= "setting last FTS document id"; + local_trx= true; + } - fts_sql_rollback(trx); - } - trx->free(); - } + char id[FTS_MAX_ID_LEN]; + snprintf(id, sizeof(id), FTS_DOC_ID_FORMAT, doc_id + 1); - return(error); + FTSQueryExecutor executor(trx, nullptr, + const_cast(table), + table->fts->dict_locked); + fts_cache_t *cache= table->fts->cache; + dberr_t error= executor.update_config_record("synced_doc_id", id); + if (local_trx) + { + if (UNIV_LIKELY(error == DB_SUCCESS)) + { + fts_sql_commit(trx); + cache->synced_doc_id = doc_id; + } + else + { + sql_print_error("InnoDB: ( %s ) while updating last doc id for table %s", + ut_strerr(error), table->name.m_name); + fts_sql_rollback(trx); + } + trx->free(); + } + return error; } /*********************************************************************//** @@ -2837,13 +2730,9 @@ fts_delete( fts_trx_table_t*ftt, /*!< in: FTS trx table */ fts_trx_row_t* row) /*!< in: row */ { - que_t* graph; - fts_table_t fts_table; - doc_id_t write_doc_id; dict_table_t* table = ftt->table; doc_id_t doc_id = row->doc_id; trx_t* trx = ftt->fts_trx->trx; - pars_info_t* info = pars_info_create(); fts_cache_t* cache = table->fts->cache; /* we do not index Documents whose Doc ID value is 0 */ @@ -2854,12 +2743,6 @@ fts_delete( ut_a(row->state == FTS_DELETE || row->state == FTS_MODIFY); - FTS_INIT_FTS_TABLE(&fts_table, "DELETED", FTS_COMMON_TABLE, table); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, doc_id); - fts_bind_doc_id(info, "doc_id", &write_doc_id); - /* It is possible we update a record that has not yet been sync-ed into cache from last crash (delete Doc will not initialize the sync). Avoid any added counter accounting until the FTS cache @@ -2884,20 +2767,9 @@ fts_delete( } /* Note the deleted document for OPTIMIZE to purge. */ - char table_name[MAX_FULL_NAME_LEN]; - trx->op_info = "adding doc id to FTS DELETED"; - - fts_table.suffix = "DELETED"; - - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "deleted", table_name); - - graph = fts_parse_sql(&fts_table, info, - "BEGIN INSERT INTO $deleted VALUES (:doc_id);"); - - dberr_t error = fts_eval_sql(trx, graph); - que_graph_free(graph); + FTSQueryExecutor executor(trx, nullptr, table); + dberr_t error= executor.insert_common_record("DELETED", doc_id); /* Increment the total deleted count, this is used to calculate the number of documents indexed. */ @@ -3065,105 +2937,18 @@ fts_doc_free( } /*********************************************************************//** -Callback function for fetch that stores the text of an FTS document, -converting each column to UTF-16. -@return always FALSE */ -ibool -fts_query_expansion_fetch_doc( -/*==========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ -{ - que_node_t* exp; - sel_node_t* node = static_cast(row); - fts_doc_t* result_doc = static_cast(user_arg); - dfield_t* dfield; - ulint len; - ulint doc_len; - fts_doc_t doc; - CHARSET_INFO* doc_charset = NULL; - ulint field_no = 0; - - len = 0; - - fts_doc_init(&doc); - doc.found = TRUE; - - exp = node->select_list; - doc_len = 0; - - doc_charset = result_doc->charset; - - /* Copy each indexed column content into doc->text.f_str */ - while (exp) { - dfield = que_node_get_val(exp); - len = dfield_get_len(dfield); - - /* NULL column */ - if (len == UNIV_SQL_NULL) { - exp = que_node_get_next(exp); - continue; - } - - if (!doc_charset) { - doc_charset = fts_get_charset(dfield->type.prtype); - } - - doc.charset = doc_charset; - - if (dfield_is_ext(dfield)) { - /* We ignore columns that are stored externally, this - could result in too many words to search */ - exp = que_node_get_next(exp); - continue; - } else { - doc.text.f_n_char = 0; - - doc.text.f_str = static_cast( - dfield_get_data(dfield)); - - doc.text.f_len = len; - } - - if (field_no == 0) { - fts_tokenize_document(&doc, result_doc, - result_doc->parser); - } else { - fts_tokenize_document_next(&doc, doc_len, result_doc, - result_doc->parser); - } - - exp = que_node_get_next(exp); - - doc_len += (exp) ? len + 1 : len; - - field_no++; - } - - ut_ad(doc_charset); - - if (!result_doc->charset) { - result_doc->charset = doc_charset; - } - - fts_doc_free(&doc); - - return(FALSE); -} - -/*********************************************************************//** -fetch and tokenize the document. */ -static -void -fts_fetch_doc_from_rec( -/*===================*/ - fts_get_doc_t* get_doc, /*!< in: FTS index's get_doc struct */ - dict_index_t* clust_index, /*!< in: cluster index */ - btr_pcur_t* pcur, /*!< in: cursor whose position - has been stored */ - rec_offs* offsets, /*!< in: offsets */ - fts_doc_t* doc) /*!< out: fts doc to hold parsed - documents */ +fetch and tokenize the document. */ +static +void +fts_fetch_doc_from_rec( +/*===================*/ + fts_get_doc_t* get_doc, /*!< in: FTS index's get_doc struct */ + dict_index_t* clust_index, /*!< in: cluster index */ + btr_pcur_t* pcur, /*!< in: cursor whose position + has been stored */ + rec_offs* offsets, /*!< in: offsets */ + fts_doc_t* doc) /*!< out: fts doc to hold parsed + documents */ { dict_index_t* index; const rec_t* clust_rec; @@ -3587,28 +3372,6 @@ fts_add_doc_by_id( mem_heap_free(heap); } - -/*********************************************************************//** -Callback function to read a single ulint column. -return always returns TRUE */ -static -ibool -fts_read_ulint( -/*===========*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ulint */ -{ - sel_node_t* sel_node = static_cast(row); - ulint* value = static_cast(user_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - - *value = mach_read_from_4(static_cast(data)); - - return(TRUE); -} - /*********************************************************************//** Get maximum Doc ID in a table if index "FTS_DOC_ID_INDEX" exists @return max Doc ID or 0 if index "FTS_DOC_ID_INDEX" does not exist */ @@ -3685,200 +3448,19 @@ fts_get_max_doc_id( return(doc_id); } -/*********************************************************************//** -Fetch document with the given document id. -@return DB_SUCCESS if OK else error */ -dberr_t -fts_doc_fetch_by_doc_id( -/*====================*/ - fts_get_doc_t* get_doc, /*!< in: state */ - doc_id_t doc_id, /*!< in: id of document to - fetch */ - dict_index_t* index_to_use, /*!< in: caller supplied FTS index, - or NULL */ - ulint option, /*!< in: search option, if it is - greater than doc_id or equal */ - fts_sql_callback - callback, /*!< in: callback to read */ - void* arg) /*!< in: callback arg */ +/** Write out a single word's data as new entry/entries in the INDEX table. +@param executor FTS Query Executor +@param selected auxiliary index number +@param aux_data auxiliary table data +@return DB_SUCCESS if all OK or error code */ +dberr_t fts_write_node(FTSQueryExecutor *executor, uint8_t selected, + const fts_aux_data_t *aux_data) { - pars_info_t* info; - dberr_t error; - const char* select_str; - doc_id_t write_doc_id; - dict_index_t* index; - trx_t* trx = trx_create(); - que_t* graph; - - trx->op_info = "fetching indexed FTS document"; - - /* The FTS index can be supplied by caller directly with - "index_to_use", otherwise, get it from "get_doc" */ - index = (index_to_use) ? index_to_use : get_doc->index_cache->index; - - if (get_doc && get_doc->get_document_graph) { - info = get_doc->get_document_graph->info; - } else { - info = pars_info_create(); - } - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, doc_id); - fts_bind_doc_id(info, "doc_id", &write_doc_id); - pars_info_bind_function(info, "my_func", callback, arg); - - select_str = fts_get_select_columns_str(index, info, info->heap); - pars_info_bind_id(info, "table_name", index->table->name.m_name); - - if (!get_doc || !get_doc->get_document_graph) { - if (option == FTS_FETCH_DOC_BY_ID_EQUAL) { - graph = fts_parse_sql( - NULL, - info, - mem_heap_printf(info->heap, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT %s FROM $table_name" - " WHERE %s = :doc_id;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c %% NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;", - select_str, - FTS_DOC_ID.str)); - } else { - ut_ad(option == FTS_FETCH_DOC_BY_ID_LARGE); - - /* This is used for crash recovery of table with - hidden DOC ID or FTS indexes. We will scan the table - to re-processing user table rows whose DOC ID or - FTS indexed documents have not been sync-ed to disc - during recent crash. - In the case that all fulltext indexes are dropped - for a table, we will keep the "hidden" FTS_DOC_ID - column, and this scan is to retreive the largest - DOC ID being used in the table to determine the - appropriate next DOC ID. - In the case of there exists fulltext index(es), this - operation will re-tokenize any docs that have not - been sync-ed to the disk, and re-prime the FTS - cached */ - graph = fts_parse_sql( - NULL, - info, - mem_heap_printf(info->heap, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT %s, %s FROM $table_name" - " WHERE %s > :doc_id;\n" - "BEGIN\n" - "" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c %% NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;", - FTS_DOC_ID.str, - select_str, - FTS_DOC_ID.str)); - } - if (get_doc) { - get_doc->get_document_graph = graph; - } - } else { - graph = get_doc->get_document_graph; - } - - error = fts_eval_sql(trx, graph); - fts_sql_commit(trx); - trx->free(); - - if (!get_doc) { - que_graph_free(graph); - } - - return(error); -} - -/*********************************************************************//** -Write out a single word's data as new entry/entries in the INDEX table. -@return DB_SUCCESS if all OK. */ -dberr_t -fts_write_node( -/*===========*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: query graph */ - fts_table_t* fts_table, /*!< in: aux table */ - fts_string_t* word, /*!< in: word in UTF-8 */ - fts_node_t* node) /*!< in: node columns */ -{ - pars_info_t* info; - dberr_t error; - ib_uint32_t doc_count; - time_t start_time; - doc_id_t last_doc_id; - doc_id_t first_doc_id; - char table_name[MAX_FULL_NAME_LEN]; - - ut_a(node->ilist != NULL); - - if (*graph) { - info = (*graph)->info; - } else { - info = pars_info_create(); - - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "index_table_name", table_name); - } - - pars_info_bind_varchar_literal(info, "token", word->f_str, word->f_len); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &first_doc_id, node->first_doc_id); - fts_bind_doc_id(info, "first_doc_id", &first_doc_id); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &last_doc_id, node->last_doc_id); - fts_bind_doc_id(info, "last_doc_id", &last_doc_id); - - ut_a(node->last_doc_id >= node->first_doc_id); - - /* Convert to "storage" byte order. */ - mach_write_to_4((byte*) &doc_count, node->doc_count); - pars_info_bind_int4_literal( - info, "doc_count", (const ib_uint32_t*) &doc_count); - - /* Set copy_name to FALSE since it's a static. */ - pars_info_bind_literal( - info, "ilist", node->ilist, node->ilist_size, - DATA_BLOB, DATA_BINARY_TYPE); - - if (!*graph) { - - *graph = fts_parse_sql( - fts_table, - info, - "BEGIN\n" - "INSERT INTO $index_table_name VALUES" - " (:token, :first_doc_id," - " :last_doc_id, :doc_count, :ilist);"); - } - - start_time = time(NULL); - error = fts_eval_sql(trx, *graph); - elapsed_time += time(NULL) - start_time; - ++n_nodes; - - return(error); + time_t start_time = time(NULL); + dberr_t error= executor->insert_aux_record(selected, aux_data); + elapsed_time += time(NULL) - start_time; + ++n_nodes; + return error; } /** Sort an array of doc_id */ @@ -3888,60 +3470,24 @@ void fts_doc_ids_sort(ib_vector_t *doc_ids) std::sort(data, data + doc_ids->used); } -/*********************************************************************//** -Add rows to the DELETED_CACHE table. +/** Add rows to the DELETED_CACHE table. @return DB_SUCCESS if all went well else error code*/ static MY_ATTRIBUTE((nonnull, warn_unused_result)) dberr_t -fts_sync_add_deleted_cache( -/*=======================*/ - fts_sync_t* sync, /*!< in: sync state */ - ib_vector_t* doc_ids) /*!< in: doc ids to add */ +fts_sync_add_deleted_cache(fts_sync_t *sync, ib_vector_t *doc_ids) { - ulint i; - pars_info_t* info; - que_t* graph; - fts_table_t fts_table; - char table_name[MAX_FULL_NAME_LEN]; - doc_id_t dummy = 0; - dberr_t error = DB_SUCCESS; - ulint n_elems = ib_vector_size(doc_ids); - - ut_a(ib_vector_size(doc_ids) > 0); - - fts_doc_ids_sort(doc_ids); - - info = pars_info_create(); - - fts_bind_doc_id(info, "doc_id", &dummy); - - FTS_INIT_FTS_TABLE( - &fts_table, "DELETED_CACHE", FTS_COMMON_TABLE, sync->table); - - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - &fts_table, - info, - "BEGIN INSERT INTO $table_name VALUES (:doc_id);"); - - for (i = 0; i < n_elems && error == DB_SUCCESS; ++i) { - doc_id_t* update; - doc_id_t write_doc_id; - - update = static_cast(ib_vector_get(doc_ids, i)); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, *update); - fts_bind_doc_id(info, "doc_id", &write_doc_id); - - error = fts_eval_sql(sync->trx, graph); - } - - que_graph_free(graph); - - return(error); + ulint n_elems= ib_vector_size(doc_ids); + ut_a(n_elems > 0); + fts_doc_ids_sort(doc_ids); + FTSQueryExecutor executor(sync->trx, nullptr, sync->table); + dberr_t error= DB_SUCCESS; + for (uint32_t i= 0; i < n_elems && error == DB_SUCCESS; ++i) + { + doc_id_t *update= + static_cast(ib_vector_get(doc_ids, i)); + error= executor.insert_common_record("DELETED_CACHE", *update); + } + return error; } /** Write the words and ilist to disk. @@ -3950,104 +3496,63 @@ fts_sync_add_deleted_cache( @param[in] unlock_cache whether unlock cache when write node @return DB_SUCCESS if all went well else error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_sync_write_words( - trx_t* trx, - fts_index_cache_t* index_cache, - bool unlock_cache) +dberr_t fts_sync_write_words(trx_t *trx, fts_index_cache_t *index_cache, + bool unlock_cache) { - fts_table_t fts_table; - ulint n_nodes = 0; - ulint n_words = 0; - const ib_rbt_node_t* rbt_node; - dberr_t error = DB_SUCCESS; - ibool print_error = FALSE; - dict_table_t* table = index_cache->index->table; - - FTS_INIT_INDEX_TABLE( - &fts_table, NULL, FTS_INDEX_TABLE, index_cache->index); - - n_words = rbt_size(index_cache->words); - - /* We iterate over the entire tree, even if there is an error, - since we want to free the memory used during caching. */ - for (rbt_node = rbt_first(index_cache->words); - rbt_node; - rbt_node = rbt_next(index_cache->words, rbt_node)) { - - ulint i; - ulint selected; - fts_tokenizer_word_t* word; - - word = rbt_value(fts_tokenizer_word_t, rbt_node); - - DBUG_EXECUTE_IF( - "fts_instrument_write_words_before_select_index", - std::this_thread::sleep_for( - std::chrono::milliseconds(300));); - - selected = fts_select_index( - index_cache->charset, word->text.f_str, - word->text.f_len); - - fts_table.suffix = fts_get_suffix(selected); - - /* We iterate over all the nodes even if there was an error */ - for (i = 0; i < ib_vector_size(word->nodes); ++i) { - - fts_node_t* fts_node = static_cast( - ib_vector_get(word->nodes, i)); - - if (fts_node->synced) { - continue; - } else { - fts_node->synced = true; - } - - /*FIXME: we need to handle the error properly. */ - if (error == DB_SUCCESS) { - if (unlock_cache) { - mysql_mutex_unlock( - &table->fts->cache->lock); - } - - error = fts_write_node( - trx, - &index_cache->ins_graph[selected], - &fts_table, &word->text, fts_node); - - DEBUG_SYNC_C("fts_write_node"); - DBUG_EXECUTE_IF("fts_write_node_crash", - DBUG_SUICIDE();); - - DBUG_EXECUTE_IF( - "fts_instrument_sync_sleep", - std::this_thread::sleep_for( - std::chrono::seconds(1));); - - if (unlock_cache) { - mysql_mutex_lock( - &table->fts->cache->lock); - } - } - } - - n_nodes += ib_vector_size(word->nodes); + dict_table_t *table= index_cache->index->table; + FTSQueryExecutor executor(trx, index_cache->index, table); + ulint n_words= rbt_size(index_cache->words); + bool print_error= false; + dberr_t error= DB_SUCCESS; + for (const ib_rbt_node_t *rbt_node= rbt_first(index_cache->words); + rbt_node; rbt_node = rbt_next(index_cache->words, rbt_node)) + { + fts_tokenizer_word_t *word= rbt_value(fts_tokenizer_word_t, rbt_node); + DBUG_EXECUTE_IF("fts_instrument_write_words_before_select_index", + std::this_thread::sleep_for( + std::chrono::milliseconds(300));); + uint8_t selected= fts_select_index( + index_cache->charset, word->text.f_str, word->text.f_len); + + for (ulint i = 0; i < ib_vector_size(word->nodes); ++i) + { + fts_node_t* fts_node= + static_cast(ib_vector_get(word->nodes, i)); + if (fts_node->synced) continue; + else fts_node->synced= true; + /* FIXME: we need to handle the error properly. */ + if (error == DB_SUCCESS) + { + if (unlock_cache) mysql_mutex_unlock(&table->fts->cache->lock); + fts_aux_data_t aux_data((const char*)word->text.f_str, word->text.f_len, + fts_node->first_doc_id, fts_node->last_doc_id, + static_cast(fts_node->doc_count), fts_node->ilist, + fts_node->ilist_size); + error= fts_write_node(&executor, selected, &aux_data); + DEBUG_SYNC_C("fts_write_node"); + DBUG_EXECUTE_IF("fts_write_node_crash", DBUG_SUICIDE();); + DBUG_EXECUTE_IF("fts_instrument_sync_sleep", + std::this_thread::sleep_for(std::chrono::seconds(1));); + + if (unlock_cache) mysql_mutex_lock(&table->fts->cache->lock); + } - if (UNIV_UNLIKELY(error != DB_SUCCESS) && !print_error) { - ib::error() << "(" << error << ") writing" - " word node to FTS auxiliary index table " - << table->name; - print_error = TRUE; - } - } + n_nodes += ib_vector_size(word->nodes); - if (UNIV_UNLIKELY(fts_enable_diag_print)) { - printf("Avg number of nodes: %lf\n", - (double) n_nodes / (double) (n_words > 1 ? n_words : 1)); - } + if (UNIV_UNLIKELY(error != DB_SUCCESS) && !print_error) + { + sql_print_error("InnoDB: ( %s ) writing word node to FTS auxiliary " + "index table %s", ut_strerr(error), + table->name.m_name); + print_error= true; + } + } + } - return(error); + if (UNIV_UNLIKELY(fts_enable_diag_print)) + printf("Avg number of nodes: %lf\n", + (double) n_nodes / (double) (n_words > 1 ? n_words : 1)); + return error; } /*********************************************************************//** @@ -4695,7 +4200,6 @@ fts_tokenize_by_parser( @param[in,out] doc document to tokenize @param[out] result tokenization result @param[in] parser pluggable parser */ -static void fts_tokenize_document( fts_doc_t* doc, @@ -4725,12 +4229,6 @@ fts_tokenize_document( } } -/** Continue to tokenize a document. -@param[in,out] doc document to tokenize -@param[in] add_pos add this position to all tokens from this tokenization -@param[out] result tokenization result -@param[in] parser pluggable parser */ -static void fts_tokenize_document_next( fts_doc_t* doc, @@ -4897,81 +4395,6 @@ fts_is_index_updated( } #endif -/*********************************************************************//** -Fetch COUNT(*) from specified table. -@return the number of rows in the table */ -ulint -fts_get_rows_count( -/*===============*/ - fts_table_t* fts_table) /*!< in: fts table to read */ -{ - trx_t* trx; - pars_info_t* info; - que_t* graph; - dberr_t error; - ulint count = 0; - char table_name[MAX_FULL_NAME_LEN]; - - trx = trx_create(); - trx->op_info = "fetching FT table rows count"; - - info = pars_info_create(); - - pars_info_bind_function(info, "my_func", fts_read_ulint, &count); - - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT COUNT(*)" - " FROM $table_name;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - - break; /* Exit the loop. */ - } else { - fts_sql_rollback(trx); - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading" - " FTS table. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "(" << error - << ") while reading FTS table " - << table_name; - - break; /* Exit the loop. */ - } - } - } - - que_graph_free(graph); - - trx->free(); - - return(count); -} - #ifdef FTS_CACHE_SIZE_DEBUG /*********************************************************************//** Read the max cache size parameter from the config table. */ @@ -4982,14 +4405,11 @@ fts_update_max_cache_size( fts_sync_t* sync) /*!< in: sync state */ { trx_t* trx; - fts_table_t fts_table; trx = trx_create(); - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, sync->table); - /* The size returned is in bytes. */ - sync->max_cache_size = fts_get_max_cache_size(trx, &fts_table); + sync->max_cache_size = fts_get_max_cache_size(trx, sync->table); fts_sql_commit(trx); @@ -5832,7 +5252,6 @@ fts_load_stopword( bool reload) /*!< in: Whether it is for reloading FTS table */ { - fts_table_t fts_table; fts_string_t str; dberr_t error = DB_SUCCESS; ulint use_stopword; @@ -5841,8 +5260,6 @@ fts_load_stopword( ibool new_trx = FALSE; byte str_buffer[MAX_FULL_NAME_LEN + 1]; - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, table); - cache = table->fts->cache; if (!reload && !(cache->stopword_info.status & STOPWORD_NOT_INIT)) { @@ -5863,12 +5280,12 @@ fts_load_stopword( /* First check whether stopword filtering is turned off */ if (reload) { error = fts_config_get_ulint( - trx, &fts_table, FTS_USE_STOPWORD, &use_stopword); + trx, table, FTS_USE_STOPWORD, &use_stopword); } else { use_stopword = (ulint) stopword_is_on; error = fts_config_set_ulint( - trx, &fts_table, FTS_USE_STOPWORD, use_stopword); + trx, table, FTS_USE_STOPWORD, use_stopword); } if (error != DB_SUCCESS) { @@ -5890,7 +5307,7 @@ fts_load_stopword( str.f_len = sizeof(str_buffer) - 1; error = fts_config_get_value( - trx, &fts_table, FTS_STOPWORD_TABLE_NAME, &str); + trx, table, FTS_STOPWORD_TABLE_NAME, &str); if (error != DB_SUCCESS) { goto cleanup; @@ -5914,7 +5331,7 @@ fts_load_stopword( str.f_len = strlen(stopword_to_use); error = fts_config_set_value( - trx, &fts_table, FTS_STOPWORD_TABLE_NAME, &str); + trx, table, FTS_STOPWORD_TABLE_NAME, &str); } } else { /* Load system default stopword list */ @@ -5941,168 +5358,205 @@ fts_load_stopword( return error == DB_SUCCESS; } -/**********************************************************************//** -Callback function when we initialize the FTS at the start up -time. It recovers the maximum Doc IDs presented in the current table. -Tested by innodb_fts.crash_recovery -@return: always returns TRUE */ -static -ibool -fts_init_get_doc_id( -/*================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: table with fts */ -{ - doc_id_t doc_id = FTS_NULL_DOC_ID; - sel_node_t* node = static_cast(row); - que_node_t* exp = node->select_list; - dict_table_t* table = static_cast(user_arg); - fts_cache_t* cache = table->fts->cache; - - ut_ad(ib_vector_is_empty(cache->get_docs)); - - /* Copy each indexed column content into doc->text.f_str */ - if (exp) { - dfield_t* dfield = que_node_get_val(exp); - dtype_t* type = dfield_get_type(dfield); - void* data = dfield_get_data(dfield); - - ut_a(dtype_get_mtype(type) == DATA_INT); - - doc_id = static_cast(mach_read_from_8( - static_cast(data))); - - exp = que_node_get_next(que_node_get_next(exp)); - if (exp) { - ut_ad(table->versioned()); - dfield = que_node_get_val(exp); - type = dfield_get_type(dfield); - ut_ad(type->vers_sys_end()); - data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - if (table->versioned_by_id()) { - ut_ad(len == sizeof trx_id_max_bytes); - if (0 != memcmp(data, trx_id_max_bytes, len)) { - return true; - } - } else { - ut_ad(len == sizeof timestamp_max_bytes); - if (!IS_MAX_TIMESTAMP(data)) { - return true; - } - } - ut_ad(!(exp = que_node_get_next(exp))); - } - ut_ad(!exp); - - if (doc_id >= cache->next_doc_id) { - cache->next_doc_id = doc_id + 1; - } - } - - return(TRUE); -} - -/**********************************************************************//** -Callback function when we initialize the FTS at the start up +/** Callback function when we initialize the FTS at the start up time. It recovers Doc IDs that have not sync-ed to the auxiliary table, and require to bring them back into FTS index. +@param get_doc Document +@param doc_id document id to be fetched @return: always returns TRUE */ -static -ibool -fts_init_recover_doc( -/*=================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts cache */ -{ - - fts_doc_t doc; - ulint doc_len = 0; - ulint field_no = 0; - fts_get_doc_t* get_doc = static_cast(user_arg); - doc_id_t doc_id = FTS_NULL_DOC_ID; - sel_node_t* node = static_cast(row); - que_node_t* exp = node->select_list; - fts_cache_t* cache = get_doc->cache; - st_mysql_ftparser* parser = get_doc->index_cache->index->parser; - - fts_doc_init(&doc); - doc.found = TRUE; - - ut_ad(cache); - - /* Copy each indexed column content into doc->text.f_str */ - while (exp) { - dfield_t* dfield = que_node_get_val(exp); - ulint len = dfield_get_len(dfield); - - if (field_no == 0) { - dtype_t* type = dfield_get_type(dfield); - void* data = dfield_get_data(dfield); - - ut_a(dtype_get_mtype(type) == DATA_INT); - - doc_id = static_cast(mach_read_from_8( - static_cast(data))); - - field_no++; - exp = que_node_get_next(exp); - continue; - } +static void fts_init_recover_all_docs(fts_get_doc_t *get_doc, + doc_id_t doc_id) +{ + trx_t *trx= trx_create(); + trx->op_info= "fetching indexed FTS document"; + dict_index_t *fts_index= get_doc->index_cache->index; + dict_table_t *user_table= fts_index->table; + dict_index_t *fts_doc_id_index= user_table->fts_doc_id_index; + dict_index_t *clust_index= dict_table_get_first_index(user_table); + fts_cache_t *cache= get_doc->cache; + ut_a(user_table->fts->doc_col != ULINT_UNDEFINED); + ut_a(fts_doc_id_index); + QueryExecutor executor(trx); + /* Map FTS index columns to clustered index field positions */ + ulint *clust_field_nos= static_cast( + mem_heap_alloc(executor.get_heap(), + fts_index->n_user_defined_cols * sizeof(ulint))); + + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); + clust_field_nos[i]= dict_col_get_index_pos(fts_field->col, clust_index); + } - if (len == UNIV_SQL_NULL) { - exp = que_node_get_next(exp); - continue; - } + dfield_t fields[1]; + dtuple_t search_tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&search_tuple, fts_doc_id_index, 1); + dfield_t* dfield= dtuple_get_nth_field(&search_tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id((byte*) &write_doc_id, doc_id); + dfield_set_data(dfield, &write_doc_id, sizeof(write_doc_id)); + + auto process_doc_recovery= [get_doc, cache, user_table, fts_index, + clust_field_nos](const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) -> bool + { + fts_doc_t doc; + ulint doc_len= 0; + doc_id_t doc_id= FTS_NULL_DOC_ID; + st_mysql_ftparser* parser= fts_index->parser; - ut_ad(get_doc); + fts_doc_init(&doc); + doc.found= TRUE; - if (!get_doc->index_cache->charset) { - get_doc->index_cache->charset = fts_get_charset( - dfield->type.prtype); - } + /* Extract doc_id from the clustered index record */ + ulint doc_col_pos= dict_col_get_index_pos( + &user_table->cols[user_table->fts->doc_col], index); - doc.charset = get_doc->index_cache->charset; + ulint processed_field= 0; + ulint len; + const byte* doc_id_data= rec_get_nth_field(rec, offsets, doc_col_pos, &len); - if (dfield_is_ext(dfield)) { - dict_table_t* table = cache->sync->table; + if (len == sizeof(doc_id_t)) + { + doc_id= fts_read_doc_id(doc_id_data); - doc.text.f_str = btr_copy_externally_stored_field( - &doc.text.f_len, - static_cast(dfield_get_data(dfield)), - table->space->zip_size(), len, - static_cast(doc.self_heap->arg)); - } else { - doc.text.f_str = static_cast( - dfield_get_data(dfield)); + /* Process each indexed column content */ + for (unsigned i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, col_pos, + &field_len); + if (field_len == UNIV_SQL_NULL) + continue; + if (!get_doc->index_cache->charset) + { + dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); + get_doc->index_cache->charset= fts_get_charset(fts_field->col->prtype); + } + doc.charset= get_doc->index_cache->charset; + + /* Handle externally stored fields */ + if (rec_offs_nth_extern(offsets, col_pos)) + doc.text.f_str= btr_copy_externally_stored_field( + &doc.text.f_len, const_cast(field_data), + user_table->space->zip_size(), field_len, + static_cast(doc.self_heap->arg)); + else + { + doc.text.f_str= const_cast(field_data); + doc.text.f_len= field_len; + } - doc.text.f_len = len; - } + if (processed_field == 0) fts_tokenize_document(&doc, NULL, parser); + else fts_tokenize_document_next(&doc, doc_len, NULL, parser); - if (field_no == 1) { - fts_tokenize_document(&doc, NULL, parser); - } else { - fts_tokenize_document_next(&doc, doc_len, NULL, parser); - } + processed_field++; + doc_len+= + (i < (unsigned) get_doc->index_cache->index->n_user_defined_cols - 1) + ? field_len + 1 + : field_len; + } - exp = que_node_get_next(exp); + fts_cache_add_doc(cache, get_doc->index_cache, doc_id, doc.tokens); + fts_doc_free(&doc); + cache->added++; - doc_len += (exp) ? len + 1 : len; + if (doc_id >= cache->next_doc_id) + cache->next_doc_id= doc_id + 1; + } - field_no++; - } + return true; + }; + RecordCallback reader(process_doc_recovery, doc_id_comparator); + if (fts_doc_id_index == clust_index) + executor.read(user_table, &search_tuple, PAGE_CUR_G, reader); + else + executor.read_by_index(user_table, fts_doc_id_index, + &search_tuple, PAGE_CUR_G, reader); + trx_commit_for_mysql(trx); + trx->free(); +} + +/** Get the next large document id and update it in fulltext cache +@param doc_id document id to be updated +@param index fulltext index */ +static void fts_init_get_doc_id(doc_id_t doc_id, dict_index_t *index) +{ + trx_t* trx= trx_create(); + trx->op_info= "fetching indexed FTS document"; + dict_table_t* user_table= index->table; + fts_cache_t* cache= user_table->fts->cache; + ut_a(user_table->fts->doc_col != ULINT_UNDEFINED); + + dfield_t fields[1]; + dtuple_t search_tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N +#endif + }; + dict_index_copy_types(&search_tuple, index, 1); + dfield_t* dfield= dtuple_get_nth_field(&search_tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id((byte*) &write_doc_id, doc_id); + dfield_set_data(dfield, &write_doc_id, sizeof(write_doc_id)); + + auto process_doc_id= [cache, user_table](const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) -> bool + { + ulint doc_col_pos= dict_col_get_index_pos( + &user_table->cols[user_table->fts->doc_col], index); - fts_cache_add_doc(cache, get_doc->index_cache, doc_id, doc.tokens); + ulint len; + const byte* doc_id_data= rec_get_nth_field(rec, offsets, doc_col_pos, &len); - fts_doc_free(&doc); + if (len == sizeof(doc_id_t)) + { + doc_id_t found_doc_id= mach_read_from_8(doc_id_data); + if (user_table->versioned()) + { + ulint vers_end_pos= dict_col_get_index_pos( + &user_table->cols[user_table->vers_end], index); + ulint vers_len; + const byte* vers_data= rec_get_nth_field(rec, offsets, + vers_end_pos, &vers_len); - cache->added++; + if (user_table->versioned_by_id()) + { + if (vers_len == sizeof(trx_id_max_bytes) && + memcmp(vers_data, trx_id_max_bytes, vers_len) != 0) + return true; + } + else + { + if (vers_len == sizeof(timestamp_max_bytes) && + !IS_MAX_TIMESTAMP(vers_data)) + return true; + } + } - if (doc_id >= cache->next_doc_id) { - cache->next_doc_id = doc_id + 1; - } + /* Update cache->next_doc_id if this doc_id is larger */ + if (found_doc_id >= cache->next_doc_id) + cache->next_doc_id= found_doc_id + 1; + } + return true; + }; - return(TRUE); + RecordCallback reader(process_doc_id, doc_id_comparator); + QueryExecutor executor(trx); + if (dict_index_is_clust(index)) + executor.read(user_table, &search_tuple, PAGE_CUR_G, reader); + else + executor.read_by_index(user_table, index, &search_tuple, + PAGE_CUR_G, reader); + trx_commit_for_mysql(trx); + trx->free(); } /**********************************************************************//** @@ -6165,9 +5619,7 @@ fts_init_index( ut_a(index); - fts_doc_fetch_by_doc_id(NULL, start_doc, index, - FTS_FETCH_DOC_BY_ID_LARGE, - fts_init_get_doc_id, table); + fts_init_get_doc_id(start_doc, index); } else { if (table->fts->cache->stopword_info.status & STOPWORD_NOT_INIT) { @@ -6180,9 +5632,7 @@ fts_init_index( index = get_doc->index_cache->index; - fts_doc_fetch_by_doc_id(NULL, start_doc, index, - FTS_FETCH_DOC_BY_ID_LARGE, - fts_init_recover_doc, get_doc); + fts_init_recover_all_docs(get_doc, start_doc); } } diff --git a/storage/innobase/fts/fts0opt.cc b/storage/innobase/fts/fts0opt.cc index af4b9eed2a491..05844cdd6f361 100644 --- a/storage/innobase/fts/fts0opt.cc +++ b/storage/innobase/fts/fts0opt.cc @@ -27,6 +27,7 @@ Completed 2011/7/10 Sunny and Jimmy Yang ***********************************************************************/ #include "fts0fts.h" +#include "fts0exec.h" #include "row0sel.h" #include "que0types.h" #include "fts0priv.h" @@ -70,9 +71,6 @@ static bool fts_opt_start_shutdown = false; Protected by fts_optimize_wq->mutex. */ static pthread_cond_t fts_opt_shutdown_cond; -/** Initial size of nodes in fts_word_t. */ -static const ulint FTS_WORD_NODES_INIT_SIZE = 64; - /** Last time we did check whether system need a sync */ static time_t last_check_sync_time; @@ -140,11 +138,6 @@ struct fts_optimize_t { char* name_prefix; /*!< FTS table name prefix */ - fts_table_t fts_index_table;/*!< Common table definition */ - - /*!< Common table definition */ - fts_table_t fts_common_table; - dict_table_t* table; /*!< Table that has to be queried */ dict_index_t* index; /*!< The FTS index to be optimized */ @@ -238,28 +231,6 @@ static ulint fts_optimize_time_limit; /** It's defined in fts0fts.cc */ extern const char* fts_common_tables[]; -/** SQL Statement for changing state of rows to be deleted from FTS Index. */ -static const char* fts_init_delete_sql = - "BEGIN\n" - "\n" - "INSERT INTO $BEING_DELETED\n" - "SELECT doc_id FROM $DELETED;\n" - "\n" - "INSERT INTO $BEING_DELETED_CACHE\n" - "SELECT doc_id FROM $DELETED_CACHE;\n"; - -static const char* fts_delete_doc_ids_sql = - "BEGIN\n" - "\n" - "DELETE FROM $DELETED WHERE doc_id = :doc_id1;\n" - "DELETE FROM $DELETED_CACHE WHERE doc_id = :doc_id2;\n"; - -static const char* fts_end_delete_sql = - "BEGIN\n" - "\n" - "DELETE FROM $BEING_DELETED;\n" - "DELETE FROM $BEING_DELETED_CACHE;\n"; - /**********************************************************************//** Initialize fts_zip_t. */ static @@ -330,238 +301,50 @@ fts_zip_init( *zip->word.f_str = '\0'; } -/**********************************************************************//** -Create a fts_optimizer_word_t instance. -@return new instance */ -static -fts_word_t* -fts_word_init( -/*==========*/ - fts_word_t* word, /*!< in: word to initialize */ - byte* utf8, /*!< in: UTF-8 string */ - ulint len) /*!< in: length of string in bytes */ -{ - mem_heap_t* heap = mem_heap_create(sizeof(fts_node_t)); - - memset(word, 0, sizeof(*word)); - - word->text.f_len = len; - word->text.f_str = static_cast(mem_heap_alloc(heap, len + 1)); - - /* Need to copy the NUL character too. */ - memcpy(word->text.f_str, utf8, word->text.f_len); - word->text.f_str[word->text.f_len] = 0; - - word->heap_alloc = ib_heap_allocator_create(heap); - - word->nodes = ib_vector_create( - word->heap_alloc, sizeof(fts_node_t), FTS_WORD_NODES_INIT_SIZE); - - return(word); -} - -/**********************************************************************//** -Read the FTS INDEX row. -@return fts_node_t instance */ -static -fts_node_t* -fts_optimize_read_node( -/*===================*/ - fts_word_t* word, /*!< in: */ - que_node_t* exp) /*!< in: */ -{ - int i; - fts_node_t* node = static_cast( - ib_vector_push(word->nodes, NULL)); - - /* Start from 1 since the first node has been read by the caller */ - for (i = 1; exp; exp = que_node_get_next(exp), ++i) { - - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - ulint len = dfield_get_len(dfield); - - ut_a(len != UNIV_SQL_NULL); - - /* Note: The column numbers below must match the SELECT */ - switch (i) { - case 1: /* DOC_COUNT */ - node->doc_count = mach_read_from_4(data); - break; - - case 2: /* FIRST_DOC_ID */ - node->first_doc_id = fts_read_doc_id(data); - break; - - case 3: /* LAST_DOC_ID */ - node->last_doc_id = fts_read_doc_id(data); - break; - - case 4: /* ILIST */ - node->ilist_size_alloc = node->ilist_size = len; - node->ilist = static_cast(ut_malloc_nokey(len)); - memcpy(node->ilist, data, len); - break; - - default: - ut_error; - } - } - - /* Make sure all columns were read. */ - ut_a(i == 5); - - return(node); -} - -/**********************************************************************//** -Callback function to fetch the rows in an FTS INDEX record. -@return always returns non-NULL */ -ibool -fts_optimize_index_fetch_node( -/*==========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - fts_word_t* word; - sel_node_t* sel_node = static_cast(row); - fts_fetch_t* fetch = static_cast(user_arg); - ib_vector_t* words = static_cast(fetch->read_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint dfield_len = dfield_get_len(dfield); - fts_node_t* node; - bool is_word_init = false; - - ut_a(dfield_len <= FTS_MAX_WORD_LEN); - - if (ib_vector_size(words) == 0) { - - word = static_cast(ib_vector_push(words, NULL)); - fts_word_init(word, (byte*) data, dfield_len); - is_word_init = true; - } - - word = static_cast(ib_vector_last(words)); - - if (dfield_len != word->text.f_len - || memcmp(word->text.f_str, data, dfield_len)) { - - word = static_cast(ib_vector_push(words, NULL)); - fts_word_init(word, (byte*) data, dfield_len); - is_word_init = true; - } - - node = fts_optimize_read_node(word, que_node_get_next(exp)); - - fetch->total_memory += node->ilist_size; - if (is_word_init) { - fetch->total_memory += sizeof(fts_word_t) - + sizeof(ib_alloc_t) + sizeof(ib_vector_t) + dfield_len - + sizeof(fts_node_t) * FTS_WORD_NODES_INIT_SIZE; - } else if (ib_vector_size(words) > FTS_WORD_NODES_INIT_SIZE) { - fetch->total_memory += sizeof(fts_node_t); - } - - if (fetch->total_memory >= fts_result_cache_limit) { - return(FALSE); - } - - return(TRUE); -} - -/**********************************************************************//** -Read the rows from the FTS inde. -@return DB_SUCCESS or error code */ -dberr_t -fts_index_fetch_nodes( -/*==================*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: prepared statement */ - fts_table_t* fts_table, /*!< in: table of the FTS INDEX */ - const fts_string_t* - word, /*!< in: the word to fetch */ - fts_fetch_t* fetch) /*!< in: fetch callback.*/ +dberr_t fts_index_fetch_nodes(trx_t *trx, dict_index_t *index, + const fts_string_t *word, void *user_arg, + FTSRecordProcessor processor, + AuxCompareMode compare_mode) { - pars_info_t* info; - dberr_t error; - char table_name[MAX_FULL_NAME_LEN]; - - trx->op_info = "fetching FTS index nodes"; - - if (*graph) { - info = (*graph)->info; - } else { - ulint selected; - - info = pars_info_create(); - - ut_a(fts_table->type == FTS_INDEX_TABLE); - - selected = fts_select_index(fts_table->charset, - word->f_str, word->f_len); - - fts_table->suffix = fts_get_suffix(selected); - - fts_get_table_name(fts_table, table_name); - - pars_info_bind_id(info, "table_name", table_name); - } - - pars_info_bind_function(info, "my_func", fetch->read_record, fetch); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - if (!*graph) { - - *graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT word, doc_count, first_doc_id, last_doc_id," - " ilist\n" - " FROM $table_name\n" - " WHERE word LIKE :word\n" - " ORDER BY first_doc_id;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - } - - for (;;) { - error = fts_eval_sql(trx, *graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - - break; /* Exit the loop. */ - } else { - fts_sql_rollback(trx); - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading" - " FTS index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "(" << error - << ") while reading FTS index."; - - break; /* Exit the loop. */ - } - } - } - - return(error); + dberr_t error= DB_SUCCESS; + trx->op_info= "fetching FTS index nodes"; + CHARSET_INFO *cs= fts_index_get_charset(index); + uint8_t selected= fts_select_index(cs, word->f_str, word->f_len); + ulint total_memory= 0; + for (;;) + { + FTSQueryExecutor executor(trx, index, index->table); + AuxRecordReader reader= processor + ? AuxRecordReader(user_arg, processor, compare_mode) + : AuxRecordReader(user_arg, &total_memory, compare_mode); + if (word->f_len == 0) + error= executor.read_aux_all(selected, reader); + else + error= executor.read_aux( + selected, reinterpret_cast(word->f_str), + PAGE_CUR_GE, reader); + if (UNIV_LIKELY(error == DB_SUCCESS || error == DB_RECORD_NOT_FOUND)) + { + if (error == DB_RECORD_NOT_FOUND) error = DB_SUCCESS; + fts_sql_commit(trx); + break; + } + else + { + fts_sql_rollback(trx); + if (error == DB_LOCK_WAIT_TIMEOUT) + { + ib::warn() << "Lock wait timeout reading FTS index. Retrying!"; + trx->error_state= DB_SUCCESS; + } + else + { + ib::error() << "(" << error << ") while reading FTS index."; + break; + } + } + } + return error; } /**********************************************************************//** @@ -665,88 +448,6 @@ fts_zip_read_word( return(zip->status == Z_OK || zip->status == Z_STREAM_END ? ptr : NULL); } -/**********************************************************************//** -Callback function to fetch and compress the word in an FTS -INDEX record. -@return FALSE on EOF */ -static -ibool -fts_fetch_index_words( -/*==================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - sel_node_t* sel_node = static_cast(row); - fts_zip_t* zip = static_cast(user_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - - ut_a(dfield_get_len(dfield) <= FTS_MAX_WORD_LEN); - - uint16 len = uint16(dfield_get_len(dfield)); - void* data = dfield_get_data(dfield); - - /* Skip the duplicate words. */ - if (zip->word.f_len == len && !memcmp(zip->word.f_str, data, len)) { - return(TRUE); - } - - memcpy(zip->word.f_str, data, len); - zip->word.f_len = len; - - ut_a(zip->zp->avail_in == 0); - ut_a(zip->zp->next_in == NULL); - - /* The string is prefixed by len. */ - /* FIXME: This is not byte order agnostic (InnoDB data files - with FULLTEXT INDEX are not portable between little-endian and - big-endian systems!) */ - zip->zp->next_in = reinterpret_cast(&len); - zip->zp->avail_in = sizeof(len); - - /* Compress the word, create output blocks as necessary. */ - while (zip->zp->avail_in > 0) { - - /* No space left in output buffer, create a new one. */ - if (zip->zp->avail_out == 0) { - byte* block; - - block = static_cast( - ut_malloc_nokey(zip->block_sz)); - - ib_vector_push(zip->blocks, &block); - - zip->zp->next_out = block; - zip->zp->avail_out = static_cast(zip->block_sz); - } - - switch (zip->status = deflate(zip->zp, Z_NO_FLUSH)) { - case Z_OK: - if (zip->zp->avail_in == 0) { - zip->zp->next_in = static_cast(data); - zip->zp->avail_in = uInt(len); - ut_a(len <= FTS_MAX_WORD_LEN); - len = 0; - } - continue; - - case Z_STREAM_END: - case Z_BUF_ERROR: - case Z_STREAM_ERROR: - default: - ut_error; - } - } - - /* All data should have been compressed. */ - ut_a(zip->zp->avail_in == 0); - zip->zp->next_in = NULL; - - ++zip->n_words; - - return(zip->n_words >= zip->max_words ? FALSE : TRUE); -} - /**********************************************************************//** Finish Zip deflate. */ static @@ -789,241 +490,194 @@ fts_zip_deflate_end( memset(zip->zp, 0, sizeof(*zip->zp)); } -/**********************************************************************//** -Read the words from the FTS INDEX. +/** Read the words from the FTS INDEX. +@param optim optimize scratch pad +@param word get words gerater than this +@param n_words max words to read @return DB_SUCCESS if all OK, DB_TABLE_NOT_FOUND if no more indexes to search else error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_index_fetch_words( -/*==================*/ - fts_optimize_t* optim, /*!< in: optimize scratch pad */ - const fts_string_t* word, /*!< in: get words greater than this - word */ - ulint n_words)/*!< in: max words to read */ +dberr_t fts_index_fetch_words(fts_optimize_t *optim, + const fts_string_t *word, + ulint n_words) { - pars_info_t* info; - que_t* graph; - ulint selected; - fts_zip_t* zip = NULL; - dberr_t error = DB_SUCCESS; - mem_heap_t* heap = static_cast(optim->self_heap->arg); - ibool inited = FALSE; + dberr_t error= DB_SUCCESS; + mem_heap_t *heap= static_cast(optim->self_heap->arg); + optim->trx->op_info= "fetching FTS index words"; - optim->trx->op_info = "fetching FTS index words"; + if (optim->zip == NULL) + optim->zip = fts_zip_create(heap, FTS_ZIP_BLOCK_SIZE, n_words); + else fts_zip_initialize(optim->zip); - if (optim->zip == NULL) { - optim->zip = fts_zip_create(heap, FTS_ZIP_BLOCK_SIZE, n_words); - } else { - fts_zip_initialize(optim->zip); - } + CHARSET_INFO *cs= fts_index_get_charset(optim->index); - for (selected = fts_select_index( - optim->fts_index_table.charset, word->f_str, word->f_len); - selected < FTS_NUM_AUX_INDEX; - selected++) { - - char table_name[MAX_FULL_NAME_LEN]; - - optim->fts_index_table.suffix = fts_get_suffix(selected); - - info = pars_info_create(); - - pars_info_bind_function( - info, "my_func", fts_fetch_index_words, optim->zip); - - pars_info_bind_varchar_literal( - info, "word", word->f_str, word->f_len); - - fts_get_table_name(&optim->fts_index_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - &optim->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT word\n" - " FROM $table_name\n" - " WHERE word > :word\n" - " ORDER BY word;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - zip = optim->zip; - - for (;;) { - int err; - - if (!inited && ((err = deflateInit(zip->zp, 9)) - != Z_OK)) { - ib::error() << "ZLib deflateInit() failed: " - << err; - - error = DB_ERROR; - break; - } else { - inited = TRUE; - error = fts_eval_sql(optim->trx, graph); - } - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - //FIXME fts_sql_commit(optim->trx); - break; - } else { - //FIXME fts_sql_rollback(optim->trx); + /* Create compression processor with state */ + bool compress_inited = false; + auto compress_processor= [&compress_inited]( + const rec_t *rec, const dict_index_t *index, + const rec_offs *offsets, void *user_arg) + { + fts_zip_t* zip= static_cast(user_arg); - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout" - " reading document. Retrying!"; + ulint word_len; + const byte* word_data= rec_get_nth_field(rec, offsets, 0, &word_len); + if (!word_data || word_len == UNIV_SQL_NULL || + word_len > FTS_MAX_WORD_LEN) + return false; - /* We need to reset the ZLib state. */ - inited = FALSE; - deflateEnd(zip->zp); - fts_zip_init(zip); + /* Skip duplicate words */ + if (zip->word.f_len == word_len && + !memcmp(zip->word.f_str, word_data, word_len)) + return true; - optim->trx->error_state = DB_SUCCESS; - } else { - ib::error() << "(" << error - << ") while reading document."; + /* Initialize deflate if not done yet */ + if (!compress_inited) + { + int err = deflateInit(zip->zp, 9); + if (err != Z_OK) + { + ib::error() << "ZLib deflateInit() failed: " << err; + return false; + } + compress_inited = true; + } - break; /* Exit the loop. */ - } - } - } + /* Update current word */ + memcpy(zip->word.f_str, word_data, word_len); + zip->word.f_len = word_len; + ut_a(zip->zp->avail_in == 0); + ut_a(zip->zp->next_in == NULL); - que_graph_free(graph); + /* Compress the word with length prefix */ + uint16_t len = static_cast(word_len); + zip->zp->next_in = reinterpret_cast(&len); + zip->zp->avail_in = sizeof(len); - /* Check if max word to fetch is exceeded */ - if (optim->zip->n_words >= n_words) { - break; - } - } + /* Compress the word, create output blocks as necessary */ + while (zip->zp->avail_in > 0) + { + /* No space left in output buffer, create a new one */ + if (zip->zp->avail_out == 0) + { + byte* block= static_cast(ut_malloc_nokey(zip->block_sz)); + ib_vector_push(zip->blocks, &block); + zip->zp->next_out= block; + zip->zp->avail_out= static_cast(zip->block_sz); + } + + switch (zip->status = deflate(zip->zp, Z_NO_FLUSH)) + { + case Z_OK: + if (zip->zp->avail_in == 0) + { + zip->zp->next_in= static_cast(const_cast(word_data)); + zip->zp->avail_in = static_cast(len); + ut_a(len <= FTS_MAX_WORD_LEN); + len = 0; + } + continue; + case Z_STREAM_END: + case Z_BUF_ERROR: + case Z_STREAM_ERROR: + default: + ut_error; + } + } - if (error == DB_SUCCESS && zip->status == Z_OK && zip->n_words > 0) { + /* All data should have been compressed */ + ut_a(zip->zp->avail_in == 0); + zip->zp->next_in = NULL; - /* All data should have been read. */ - ut_a(zip->zp->avail_in == 0); + ++zip->n_words; - fts_zip_deflate_end(zip); - } else { - deflateEnd(zip->zp); - } + /* Continue until we reach max words */ + return zip->n_words < zip->max_words; + }; - return(error); -} + for (uint8_t selected= fts_select_index(cs, word->f_str, word->f_len); + selected < FTS_NUM_AUX_INDEX; selected++) + { + for (;;) + { + FTSQueryExecutor executor(optim->trx, optim->index, optim->table); + AuxRecordReader aux_reader(optim->zip, compress_processor, + AuxCompareMode::GREATER); + + if (word->f_len == 0) + error= executor.read_aux_all(selected, aux_reader); + else error= executor.read_aux( + selected, + reinterpret_cast(word->f_str), + PAGE_CUR_G, aux_reader); + + if (UNIV_LIKELY(error == DB_SUCCESS || error == DB_RECORD_NOT_FOUND)) { + if (error == DB_RECORD_NOT_FOUND) error = DB_SUCCESS; + break; + } + else + { + if (error == DB_LOCK_WAIT_TIMEOUT) + { + ib::warn() << "Lock wait timeout reading words. Retrying!"; + if (compress_inited) + { + deflateEnd(optim->zip->zp); + fts_zip_init(optim->zip); + compress_inited= false; + } + optim->trx->error_state = DB_SUCCESS; + } + else + { + ib::error() << "(" << ut_strerr(error) << ") while reading words."; + break; + } + } + } -/**********************************************************************//** -Callback function to fetch the doc id from the record. -@return always returns TRUE */ -static -ibool -fts_fetch_doc_ids( -/*==============*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to ib_vector_t */ -{ - que_node_t* exp; - int i = 0; - sel_node_t* sel_node = static_cast(row); - fts_doc_ids_t* fts_doc_ids = static_cast(user_arg); - doc_id_t* update = static_cast( - ib_vector_push(fts_doc_ids->doc_ids, NULL)); - - for (exp = sel_node->select_list; - exp; - exp = que_node_get_next(exp), ++i) { - - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - - ut_a(len != UNIV_SQL_NULL); - - /* Note: The column numbers below must match the SELECT. */ - switch (i) { - case 0: /* DOC_ID */ - *update = fts_read_doc_id( - static_cast(data)); - break; + if (optim->zip->n_words >= n_words) break; + } - default: - ut_error; - } - } + fts_zip_t *zip = optim->zip; + if (error == DB_SUCCESS && zip->status == Z_OK && zip->n_words > 0) { + /* All data should have been read */ + ut_a(zip->zp->avail_in == 0); + fts_zip_deflate_end(zip); + } + else deflateEnd(zip->zp); - return(TRUE); + return error; } -/**********************************************************************//** -Read the rows from a FTS common auxiliary table. -@return DB_SUCCESS or error code */ -dberr_t -fts_table_fetch_doc_ids( -/*====================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: table */ - fts_doc_ids_t* doc_ids) /*!< in: For collecting doc ids */ +dberr_t fts_table_fetch_doc_ids(trx_t *trx, dict_table_t *table, + const char *tbl_name, + fts_doc_ids_t *doc_ids) noexcept { - dberr_t error; - que_t* graph; - pars_info_t* info = pars_info_create(); - ibool alloc_bk_trx = FALSE; - char table_name[MAX_FULL_NAME_LEN]; - - ut_a(fts_table->suffix != NULL); - ut_a(fts_table->type == FTS_COMMON_TABLE); - - if (!trx) { - trx = trx_create(); - alloc_bk_trx = TRUE; - } - - trx->op_info = "fetching FTS doc ids"; - - pars_info_bind_function(info, "my_func", fts_fetch_doc_ids, doc_ids); - - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT doc_id FROM $table_name;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - error = fts_eval_sql(trx, graph); - fts_sql_commit(trx); - que_graph_free(graph); - - if (error == DB_SUCCESS) { - fts_doc_ids_sort(doc_ids->doc_ids); - } - - if (alloc_bk_trx) { - trx->free(); - } + bool bk_trx= false; + if (!trx) + { + trx= trx_create(); + bk_trx= true; + } + trx->op_info = "fetching FTS doc ids"; + FTSQueryExecutor executor(trx, nullptr, table); + CommonTableReader reader; + dberr_t err= executor.read_all_common(tbl_name, reader); - return(error); + if (err == DB_SUCCESS) + { + const auto& doc_id_vector= reader.get_doc_ids(); + for (doc_id_t doc_id : doc_id_vector) + ib_vector_push(doc_ids->doc_ids, &doc_id); + fts_doc_ids_sort(doc_ids->doc_ids); + } + if (bk_trx) + { + if (err == DB_SUCCESS) fts_sql_commit(trx); + else fts_sql_rollback(trx); + trx->free(); + } + return err; } /**********************************************************************//** @@ -1423,87 +1077,55 @@ fts_optimize_word( return(nodes); } -/**********************************************************************//** -Update the FTS index table. This is a delete followed by an insert. +/** Update the FTS index table. This is a delete followed +by an insert operation +@param trx transaction +@param index index to update +@param word word to update +@param nodes nodes to update @return DB_SUCCESS or error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_write_word( -/*====================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: table of FTS index */ - fts_string_t* word, /*!< in: word data to write */ - ib_vector_t* nodes) /*!< in: the nodes to write */ +dberr_t fts_optimize_write_word(trx_t *trx, dict_index_t *index, + fts_string_t *word, ib_vector_t *nodes) { - ulint i; - pars_info_t* info; - que_t* graph; - ulint selected; - dberr_t error = DB_SUCCESS; - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - ut_ad(fts_table->charset); - - pars_info_bind_varchar_literal( - info, "word", word->f_str, word->f_len); - - selected = fts_select_index(fts_table->charset, - word->f_str, word->f_len); - - fts_table->suffix = fts_get_suffix(selected); - fts_get_table_name(fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - fts_table, - info, - "BEGIN DELETE FROM $table_name WHERE word = :word;"); - - error = fts_eval_sql(trx, graph); - - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ") during optimize," - " when deleting a word from the FTS index."; - } - - que_graph_free(graph); - graph = NULL; - - /* Even if the operation needs to be rolled back and redone, - we iterate over the nodes in order to free the ilist. */ - for (i = 0; i < ib_vector_size(nodes); ++i) { - - fts_node_t* node = (fts_node_t*) ib_vector_get(nodes, i); - - if (error == DB_SUCCESS) { - /* Skip empty node. */ - if (node->ilist == NULL) { - ut_ad(node->ilist_size == 0); - continue; - } - - error = fts_write_node( - trx, &graph, fts_table, word, node); + CHARSET_INFO *cs= fts_index_get_charset(index); + uint8_t selected= fts_select_index(cs, word->f_str, word->f_len); + FTSQueryExecutor executor(trx, index, index->table); + fts_aux_data_t aux_data((const char*)word->f_str, word->f_len); + dberr_t err= executor.delete_aux_record(selected, &aux_data); + if (err != DB_SUCCESS) + { + sql_print_error("InnoDB: (%s) during optimize, when " + "deleting a word from the FTS index.", + ut_strerr(err)); + return err; + } - if (UNIV_UNLIKELY(error != DB_SUCCESS)) { - ib::error() << "(" << error << ")" - " during optimize, while adding a" - " word to the FTS index."; - } - } + for (ulint i = 0; i < ib_vector_size(nodes); ++i) + { + fts_node_t* node = (fts_node_t*) ib_vector_get(nodes, i); + if (!node->ilist || node->ilist_size == 0) continue; - ut_free(node->ilist); - node->ilist = NULL; - node->ilist_size = node->ilist_size_alloc = 0; - } + fts_aux_data_t insert_data( + (const char*)word->f_str, word->f_len, + node->first_doc_id, node->last_doc_id, + static_cast(node->doc_count), node->ilist, + node->ilist_size); - if (graph != NULL) { - que_graph_free(graph); - } + err = executor.insert_aux_record(selected, &insert_data); + if (err != DB_SUCCESS) + { + sql_print_error("InnoDB: (%s) during optimize, when " + "inserting a word to the FTS index.", + ut_strerr(err)); + return err; + } + ut_free(node->ilist); + node->ilist= nullptr; + node->ilist_size= node->ilist_size_alloc= 0; + } - return(error); + return DB_SUCCESS; } /**********************************************************************//** @@ -1551,7 +1173,7 @@ fts_optimize_compact( /* Update the data on disk. */ error = fts_optimize_write_word( - trx, &optim->fts_index_table, &word->text, nodes); + trx, index, &word->text, nodes); if (error == DB_SUCCESS) { /* Write the last word optimized to the config table, @@ -1603,18 +1225,10 @@ fts_optimize_create( optim->trx = trx_create(); trx_start_internal(optim->trx); - optim->fts_common_table.table_id = table->id; - optim->fts_common_table.type = FTS_COMMON_TABLE; - optim->fts_common_table.table = table; - - optim->fts_index_table.table_id = table->id; - optim->fts_index_table.type = FTS_INDEX_TABLE; - optim->fts_index_table.table = table; - /* The common prefix for all this parent table's aux tables. */ char table_id[FTS_AUX_MIN_TABLE_ID_LENGTH]; const size_t table_id_len = 1 - + size_t(fts_get_table_id(&optim->fts_common_table, table_id)); + + size_t(fts_write_object_id(table->id, table_id)); dict_sys.freeze(SRW_LOCK_CALL); /* Include the separator as well. */ const size_t dbname_len = table->name.dblen() + 1; @@ -1748,117 +1362,70 @@ fts_optimize_free( mem_heap_free(heap); } -/**********************************************************************//** -Get the max time optimize should run in millisecs. +/** Get the max time optimize should run in millisecs. +@param trx transaction +@param table user table to be optimized @return max optimize time limit in millisecs. */ static -ulint -fts_optimize_get_time_limit( -/*========================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table) /*!< in: aux table */ +ulint fts_optimize_get_time_limit(trx_t *trx, const dict_table_t *table) { - ulint time_limit = 0; - - fts_config_get_ulint( - trx, fts_table, - FTS_OPTIMIZE_LIMIT_IN_SECS, &time_limit); - - /* FIXME: This is returning milliseconds, while the variable - is being stored and interpreted as seconds! */ - return(time_limit * 1000); + ulint time_limit= 0; + fts_config_get_ulint(trx, table, FTS_OPTIMIZE_LIMIT_IN_SECS, &time_limit); + /* FIXME: This is returning milliseconds, while the variable + is being stored and interpreted as seconds! */ + return(time_limit * 1000); } -/**********************************************************************//** -Run OPTIMIZE on the given table. Note: this can take a very long time -(hours). */ +/** Run OPTIMIZE on the given table. Note: this can take a very +long time (hours). +@param optim optimize instance +@param index current fts being optimized +@param word starting word to optimize */ static -void -fts_optimize_words( -/*===============*/ - fts_optimize_t* optim, /*!< in: optimize instance */ - dict_index_t* index, /*!< in: current FTS being optimized */ - fts_string_t* word) /*!< in: the starting word to optimize */ +void fts_optimize_words(fts_optimize_t *optim, dict_index_t *index, + fts_string_t *word) { - fts_fetch_t fetch; - que_t* graph = NULL; - CHARSET_INFO* charset = optim->fts_index_table.charset; - - ut_a(!optim->done); + ut_a(!optim->done); + /* Get the time limit from the config table. */ + fts_optimize_time_limit= + fts_optimize_get_time_limit(optim->trx, index->table); + const time_t start_time= time(NULL); - /* Get the time limit from the config table. */ - fts_optimize_time_limit = fts_optimize_get_time_limit( - optim->trx, &optim->fts_common_table); - - const time_t start_time = time(NULL); - - /* Setup the callback to use for fetching the word ilist etc. */ - fetch.read_arg = optim->words; - fetch.read_record = fts_optimize_index_fetch_node; - - while (!optim->done) { - dberr_t error; - trx_t* trx = optim->trx; - ulint selected; - - ut_a(ib_vector_size(optim->words) == 0); - - selected = fts_select_index(charset, word->f_str, word->f_len); - - /* Read the index records to optimize. */ - fetch.total_memory = 0; - error = fts_index_fetch_nodes( - trx, &graph, &optim->fts_index_table, word, - &fetch); - ut_ad(fetch.total_memory < fts_result_cache_limit); - - if (error == DB_SUCCESS) { - /* There must be some nodes to read. */ - ut_a(ib_vector_size(optim->words) > 0); - - /* Optimize the nodes that were read and write - back to DB. */ - error = fts_optimize_compact(optim, index, start_time); - - if (error == DB_SUCCESS) { - fts_sql_commit(optim->trx); - } else { - fts_sql_rollback(optim->trx); - } - } - - ib_vector_reset(optim->words); - - if (error == DB_SUCCESS) { - if (!optim->done) { - if (!fts_zip_read_word(optim->zip, word)) { - optim->done = TRUE; - } else if (selected - != fts_select_index( - charset, word->f_str, - word->f_len) - && graph) { - que_graph_free(graph); - graph = NULL; - } - } - } else if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout during optimize." - " Retrying!"; - - trx->error_state = DB_SUCCESS; - } else if (error == DB_DEADLOCK) { - ib::warn() << "Deadlock during optimize. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - optim->done = TRUE; /* Exit the loop. */ - } - } + while (!optim->done) + { + trx_t *trx= optim->trx; + ut_a(ib_vector_size(optim->words) == 0); + /* Read the index records to optimize. */ + dberr_t error= fts_index_fetch_nodes( + trx, index, word, optim->words, nullptr, AuxCompareMode::LIKE); + if (error == DB_SUCCESS) + { + /* There must be some nodes to read. */ + ut_a(ib_vector_size(optim->words) > 0); + /* Optimize the nodes that were read and write back to DB. */ + error = fts_optimize_compact(optim, index, start_time); + if (error == DB_SUCCESS) fts_sql_commit(optim->trx); + else fts_sql_rollback(optim->trx); + } + ib_vector_reset(optim->words); - if (graph != NULL) { - que_graph_free(graph); - } + if (error == DB_SUCCESS) + { + if (!optim->done && !fts_zip_read_word(optim->zip, word)) + optim->done= TRUE; + } + else if (error == DB_LOCK_WAIT_TIMEOUT) + { + ib::warn() << "Lock wait timeout during optimize. Retrying!"; + trx->error_state= DB_SUCCESS; + } + else if (error == DB_DEADLOCK) + { + ib::warn() << "Deadlock during optimize. Retrying!"; + trx->error_state = DB_SUCCESS; + } + else optim->done = TRUE; + } } /**********************************************************************//** @@ -1930,6 +1497,7 @@ fts_optimize_index_read_words( error = DB_SUCCESS; } + optim->index = index; while (error == DB_SUCCESS) { error = fts_index_fetch_words( @@ -1965,10 +1533,6 @@ fts_optimize_index( dberr_t error; byte str[FTS_MAX_WORD_LEN + 1]; - /* Set the current index that we have to optimize. */ - optim->fts_index_table.index_id = index->id; - optim->fts_index_table.charset = fts_index_get_charset(index); - optim->done = FALSE; /* Optimize until !done */ /* We need to read the last word optimized so that we start from @@ -2022,190 +1586,97 @@ fts_optimize_index( return(error); } -/**********************************************************************//** -Delete the document ids in the delete, and delete cache tables. +/** Delete the document ids in the delete, and delete cache tables. +@param optim optimize instance @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_purge_deleted_doc_ids( -/*===============================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_purge_deleted_doc_ids(fts_optimize_t *optim) { - ulint i; - pars_info_t* info; - que_t* graph; - doc_id_t* update; - doc_id_t write_doc_id; - dberr_t error = DB_SUCCESS; - char deleted[MAX_FULL_NAME_LEN]; - char deleted_cache[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - ut_a(ib_vector_size(optim->to_delete->doc_ids) > 0); - - update = static_cast( - ib_vector_get(optim->to_delete->doc_ids, 0)); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, *update); - - /* This is required for the SQL parser to work. It must be able - to find the following variables. So we do it twice. */ - fts_bind_doc_id(info, "doc_id1", &write_doc_id); - fts_bind_doc_id(info, "doc_id2", &write_doc_id); - - /* Make sure the following two names are consistent with the name - used in the fts_delete_doc_ids_sql */ - optim->fts_common_table.suffix = fts_common_tables[3]; - fts_get_table_name(&optim->fts_common_table, deleted); - pars_info_bind_id(info, fts_common_tables[3], deleted); - - optim->fts_common_table.suffix = fts_common_tables[4]; - fts_get_table_name(&optim->fts_common_table, deleted_cache); - pars_info_bind_id(info, fts_common_tables[4], deleted_cache); - - graph = fts_parse_sql(NULL, info, fts_delete_doc_ids_sql); - - /* Delete the doc ids that were copied at the start. */ - for (i = 0; i < ib_vector_size(optim->to_delete->doc_ids); ++i) { - - update = static_cast(ib_vector_get( - optim->to_delete->doc_ids, i)); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &write_doc_id, *update); - - fts_bind_doc_id(info, "doc_id1", &write_doc_id); - - fts_bind_doc_id(info, "doc_id2", &write_doc_id); - - error = fts_eval_sql(optim->trx, graph); - - // FIXME: Check whether delete actually succeeded! - if (error != DB_SUCCESS) { - - fts_sql_rollback(optim->trx); - break; - } - } - - que_graph_free(graph); + dberr_t error= DB_SUCCESS; + ut_a(ib_vector_size(optim->to_delete->doc_ids) > 0); + FTSQueryExecutor executor(optim->trx, nullptr, optim->table); + for (ulint i= 0; + i < ib_vector_size(optim->to_delete->doc_ids) && error != DB_SUCCESS; + ++i) + { + doc_id_t *update= + static_cast(ib_vector_get(optim->to_delete->doc_ids, i)); + error= executor.delete_common_record("DELETED", *update); + if (error == DB_SUCCESS) + error= executor.delete_common_record("DELTED_CACHED", *update); + } - return(error); + if (error != DB_SUCCESS) + fts_sql_rollback(optim->trx); + return error; } -/**********************************************************************//** -Delete the document ids in the pending delete, and delete tables. +/** Delete the document ids in the pending delete, and delete tables. +@param optim optimize instance @return DB_SUCCESS if all OK */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_purge_deleted_doc_id_snapshot( -/*=======================================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_purge_deleted_doc_id_snapshot(fts_optimize_t *optim) { - dberr_t error; - que_t* graph; - pars_info_t* info; - char being_deleted[MAX_FULL_NAME_LEN]; - char being_deleted_cache[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - /* Make sure the following two names are consistent with the name - used in the fts_end_delete_sql */ - optim->fts_common_table.suffix = fts_common_tables[0]; - fts_get_table_name(&optim->fts_common_table, being_deleted); - pars_info_bind_id(info, fts_common_tables[0], being_deleted); - - optim->fts_common_table.suffix = fts_common_tables[1]; - fts_get_table_name(&optim->fts_common_table, being_deleted_cache); - pars_info_bind_id(info, fts_common_tables[1], being_deleted_cache); - - /* Delete the doc ids that were copied to delete pending state at - the start of optimize. */ - graph = fts_parse_sql(NULL, info, fts_end_delete_sql); - - error = fts_eval_sql(optim->trx, graph); - que_graph_free(graph); - - return(error); + FTSQueryExecutor executor(optim->trx, nullptr, optim->table); + dberr_t error= executor.delete_all_common_records("BEING_DELETED"); + if (error == DB_SUCCESS) + error= executor.delete_all_common_records("BEING_DELETED_CACHE"); + return error; } -/**********************************************************************//** -Copy the deleted doc ids that will be purged during this optimize run -to the being deleted FTS auxiliary tables. The transaction is committed -upon successfull copy and rolled back on DB_DUPLICATE_KEY error. +/** Copy the deleted doc ids that will be purged during this +optimize run to the being deleted FTS auxiliary tables. +The transaction is committed upon successfull copy and rolled +back on DB_DUPLICATE_KEY error. +@param optim optimize instance +@param n_rows number of rows exist in being_deleted table @return DB_SUCCESS if all OK */ static -ulint -fts_optimize_being_deleted_count( -/*=============================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_being_deleted_count(fts_optimize_t *optim, + ulint *n_rows) { - fts_table_t fts_table; - - FTS_INIT_FTS_TABLE(&fts_table, "BEING_DELETED", FTS_COMMON_TABLE, - optim->table); - - return(fts_get_rows_count(&fts_table)); + FTSQueryExecutor executor(optim->trx, nullptr, optim->table); + CommonTableReader reader; + dberr_t err= executor.read_all_common("BEING_DELETED", reader); + if (err == DB_SUCCESS) *n_rows= reader.get_doc_ids().size(); + return err; } -/*********************************************************************//** -Copy the deleted doc ids that will be purged during this optimize run -to the being deleted FTS auxiliary tables. The transaction is committed -upon successfull copy and rolled back on DB_DUPLICATE_KEY error. -@return DB_SUCCESS if all OK */ +/** Create a snapshot of deleted document IDs by moving them from +DELETED to BEING_DELETED and from DELETED_CACHE to +BEING_DELETED_CACHE. +@param optim optimize fts instance +@return DB_SUCCESS or error code */ static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_optimize_create_deleted_doc_id_snapshot( -/*========================================*/ - fts_optimize_t* optim) /*!< in: optimize instance */ +dberr_t fts_optimize_create_deleted_doc_id_snapshot(fts_optimize_t *optim) { - dberr_t error; - que_t* graph; - pars_info_t* info; - char being_deleted[MAX_FULL_NAME_LEN]; - char deleted[MAX_FULL_NAME_LEN]; - char being_deleted_cache[MAX_FULL_NAME_LEN]; - char deleted_cache[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - /* Make sure the following four names are consistent with the name - used in the fts_init_delete_sql */ - optim->fts_common_table.suffix = fts_common_tables[0]; - fts_get_table_name(&optim->fts_common_table, being_deleted); - pars_info_bind_id(info, fts_common_tables[0], being_deleted); + dberr_t err= DB_SUCCESS; + FTSQueryExecutor executor(optim->trx, nullptr, optim->table); + CommonTableReader reader; - optim->fts_common_table.suffix = fts_common_tables[3]; - fts_get_table_name(&optim->fts_common_table, deleted); - pars_info_bind_id(info, fts_common_tables[3], deleted); + err= executor.read_all_common("DELETED", reader); + if (err != DB_SUCCESS && err != DB_RECORD_NOT_FOUND) return err; - optim->fts_common_table.suffix = fts_common_tables[1]; - fts_get_table_name(&optim->fts_common_table, being_deleted_cache); - pars_info_bind_id(info, fts_common_tables[1], being_deleted_cache); - - optim->fts_common_table.suffix = fts_common_tables[4]; - fts_get_table_name(&optim->fts_common_table, deleted_cache); - pars_info_bind_id(info, fts_common_tables[4], deleted_cache); - - /* Move doc_ids that are to be deleted to state being deleted. */ - graph = fts_parse_sql(NULL, info, fts_init_delete_sql); - - error = fts_eval_sql(optim->trx, graph); - - que_graph_free(graph); + const auto& deleted_doc_ids = reader.get_doc_ids(); + for (doc_id_t doc_id : deleted_doc_ids) + { + err= executor.insert_common_record("BEING_DELETED", doc_id); + if (err != DB_SUCCESS) return err; + } - if (error != DB_SUCCESS) { - fts_sql_rollback(optim->trx); - } else { - fts_sql_commit(optim->trx); - } + reader.clear(); + err= executor.read_all_common("DELETED_CACHE", reader); + if (err != DB_SUCCESS && err != DB_RECORD_NOT_FOUND) return err; - optim->del_list_regenerated = TRUE; + const auto& deleted_cache_doc_ids= reader.get_doc_ids(); + for (doc_id_t doc_id : deleted_cache_doc_ids) + { + err= executor.insert_common_record("BEING_DELETED_CACHE", doc_id); + if (err != DB_SUCCESS) return err; + } - return(error); + optim->del_list_regenerated= TRUE; + return err; } /*********************************************************************//** @@ -2218,29 +1689,23 @@ fts_optimize_read_deleted_doc_id_snapshot( /*======================================*/ fts_optimize_t* optim) /*!< in: optimize instance */ { - dberr_t error; - - optim->fts_common_table.suffix = "BEING_DELETED"; - /* Read the doc_ids to delete. */ - error = fts_table_fetch_doc_ids( - optim->trx, &optim->fts_common_table, optim->to_delete); + dberr_t error = fts_table_fetch_doc_ids( + optim->trx, optim->table, "BEING_DELETED", + optim->to_delete); if (error == DB_SUCCESS) { - optim->fts_common_table.suffix = "BEING_DELETED_CACHE"; - /* Read additional doc_ids to delete. */ error = fts_table_fetch_doc_ids( - optim->trx, &optim->fts_common_table, optim->to_delete); + optim->trx, optim->table, "BEING_DELETED_CACHE", + optim->to_delete); } if (error != DB_SUCCESS) { - fts_doc_ids_free(optim->to_delete); optim->to_delete = NULL; } - return(error); } @@ -2452,7 +1917,10 @@ fts_optimize_table( // rely on DB_DUPLICATE_KEY to handle corrupting the snapshot. /* Check whether there are still records in BEING_DELETED table */ - if (fts_optimize_being_deleted_count(optim) == 0) { + ulint n_rows = 0; + error= fts_optimize_being_deleted_count(optim, &n_rows); + + if (error == DB_SUCCESS && n_rows == 0) { /* Take a snapshot of the deleted document ids, they are copied to the BEING_ tables. */ error = fts_optimize_create_deleted_doc_id_snapshot(optim); diff --git a/storage/innobase/fts/fts0que.cc b/storage/innobase/fts/fts0que.cc index 191678db78188..66329246aad8c 100644 --- a/storage/innobase/fts/fts0que.cc +++ b/storage/innobase/fts/fts0que.cc @@ -67,9 +67,6 @@ struct fts_query_t { trx_t* trx; /*!< The query transaction */ dict_index_t* index; /*!< The FTS index to search */ - /*!< FTS auxiliary common table def */ - - fts_table_t fts_common_table; fts_table_t fts_index_table;/*!< FTS auxiliary index table def */ @@ -270,16 +267,6 @@ struct fts_word_freq_t { double idf; /*!< Inverse document frequency */ }; -/******************************************************************** -Callback function to fetch the rows in an FTS INDEX record. -@return always TRUE */ -static -ibool -fts_query_index_fetch_nodes( -/*========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg); /*!< in: pointer to ib_vector_t */ - /******************************************************************** Read and filter nodes. @return fts_node_t instance */ @@ -310,19 +297,147 @@ fts_ast_visit_sub_exp( fts_ast_callback visitor, void* arg); -#if 0 -/*****************************************************************//*** -Find a doc_id in a word's ilist. -@return TRUE if found. */ -static -ibool -fts_query_find_doc_id( -/*==================*/ - fts_select_t* select, /*!< in/out: search the doc id selected, - update the frequency if found. */ - void* data, /*!< in: doc id ilist */ - ulint len); /*!< in: doc id ilist size */ -#endif +/** Process query records for FTS queries. +@param rec record +@param index index +@param offsets record offsets +@param user_arg user argument +@return true to continue processing, false to stop */ +static bool node_query_processor( + const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets, void* user_arg) +{ + fts_query_t* query= static_cast(user_arg); + ulint word_len; + const byte* word_data = rec_get_nth_field(rec, offsets, 0, &word_len); + if (!word_data || word_len == UNIV_SQL_NULL || + word_len > FTS_MAX_WORD_LEN) + return true; + ut_a(query->cur_node->type == FTS_AST_TERM + || query->cur_node->type == FTS_AST_TEXT + || query->cur_node->type == FTS_AST_PARSER_PHRASE_LIST); + + fts_node_t node; + memset(&node, 0, sizeof(node)); + + fts_string_t term; + byte buf[FTS_MAX_WORD_LEN + 1]; + term.f_str= buf; + + /* Need to consider the wildcard search case, the word frequency + is created on the search string not the actual word. So we need + to assign the frequency on search string behalf. */ + if (query->cur_node->type == FTS_AST_TERM && query->cur_node->term.wildcard) + { + term.f_len = query->cur_node->term.ptr->len; + ut_ad(FTS_MAX_WORD_LEN >= term.f_len); + memcpy(term.f_str, query->cur_node->term.ptr->str, term.f_len); + } + else + { + term.f_len = word_len; + ut_ad(FTS_MAX_WORD_LEN >= word_len); + memcpy(term.f_str, word_data, word_len); + } + + /* Lookup the word in our rb tree, it must exist. */ + ib_rbt_bound_t parent; + int ret= rbt_search(query->word_freqs, &parent, &term); + + ut_a(ret == 0); + fts_word_freq_t* word_freq= rbt_value(fts_word_freq_t, parent.last); + bool skip = false; + + ulint doc_id_len; + const byte* doc_id_data= rec_get_nth_field(rec, offsets, 1, &doc_id_len); + + if (doc_id_data && doc_id_len == 8) + { + node.first_doc_id = fts_read_doc_id(doc_id_data); + skip= (query->oper == FTS_EXIST && query->upper_doc_id > 0 && + node.first_doc_id > query->upper_doc_id); + } + + doc_id_data= rec_get_nth_field(rec, offsets, 4, &doc_id_len); + if (doc_id_data && doc_id_len == 8) + { + node.last_doc_id = fts_read_doc_id(doc_id_data); + skip= (query->oper == FTS_EXIST && query->lower_doc_id > 0 && + node.last_doc_id < query->lower_doc_id); + } + + ulint doc_count_len; + const byte* doc_count_data= rec_get_nth_field(rec, offsets, 5, &doc_count_len); + if (doc_count_data && doc_count_len == 4) + word_freq->doc_count += mach_read_from_4(doc_count_data); + + if (!skip) + { + ulint ilist_len; + const byte* ilist_data= rec_get_nth_field(rec, offsets, 6, &ilist_len); + byte* external_data = nullptr; + mem_heap_t* temp_heap = nullptr; + + if (ilist_data && ilist_len != UNIV_SQL_NULL && ilist_len > 0) + { + /* Check if ilist is stored externally */ + if (rec_offs_nth_extern(offsets, 6)) + { + /* Create temporary heap for external data */ + temp_heap = mem_heap_create(ilist_len + 1000); + /* Fetch the externally stored BLOB data */ + ulint external_len; + external_data = btr_copy_externally_stored_field( + &external_len, ilist_data, + query->index->table->space->zip_size(), + ilist_len, temp_heap); + + if (external_data) + { + ilist_data = external_data; + ilist_len = external_len; + } + else + { + /* Failed to fetch external data, skip this node */ + if (temp_heap) mem_heap_free(temp_heap); + return true; + } + } + + /* Process the ilist data (either inline or external) */ + query->error= fts_query_filter_doc_ids( + query, &word_freq->word, word_freq, &node, + const_cast(ilist_data), ilist_len, FALSE); + + /* Clean up temporary heap if used */ + if (temp_heap) mem_heap_free(temp_heap); + + if (query->error != DB_SUCCESS) return false; + } + } + return true; +} + +/* Comparator that signals how to treat the current record */ +RecordCompareAction doc_id_exact_match_comparator( + const dtuple_t* search_tuple, const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) +{ + const dfield_t* search_field= dtuple_get_nth_field(search_tuple, 0); + const byte* search_data= + static_cast(dfield_get_data(search_field)); + doc_id_t target_doc_id= fts_read_doc_id(search_data); + + ulint len; + const byte* rec_data= rec_get_nth_field(rec, offsets, 0, &len); + if (len != sizeof(doc_id_t)) + return RecordCompareAction::STOP; + doc_id_t rec_doc_id= fts_read_doc_id(rec_data); + return rec_doc_id == target_doc_id + ? RecordCompareAction::PROCESS + : RecordCompareAction::STOP; +} /*************************************************************//** This function implements a simple "blind" query expansion search: @@ -371,107 +486,6 @@ fts_proximity_get_positions( fts_proximity_t* qualified_pos); /*!< out: the position info records ranges containing all matching words. */ -#if 0 -/******************************************************************** -Get the total number of words in a documents. */ -static -ulint -fts_query_terms_in_document( -/*========================*/ - /*!< out: DB_SUCCESS if all go well - else error code */ - fts_query_t* query, /*!< in: FTS query state */ - doc_id_t doc_id, /*!< in: the word to check */ - ulint* total); /*!< out: total words in document */ -#endif - -#if 0 -/*******************************************************************//** -Print the table used for calculating LCS. */ -static -void -fts_print_lcs_table( -/*================*/ - const ulint* table, /*!< in: array to print */ - ulint n_rows, /*!< in: total no. of rows */ - ulint n_cols) /*!< in: total no. of cols */ -{ - ulint i; - - for (i = 0; i < n_rows; ++i) { - ulint j; - - printf("\n"); - - for (j = 0; j < n_cols; ++j) { - - printf("%2lu ", FTS_ELEM(table, n_cols, i, j)); - } - } -} - -/******************************************************************** -Find the longest common subsequence between the query string and -the document. */ -static -ulint -fts_query_lcs( -/*==========*/ - /*!< out: LCS (length) between - two ilists */ - const ulint* p1, /*!< in: word positions of query */ - ulint len_p1, /*!< in: no. of elements in p1 */ - const ulint* p2, /*!< in: word positions within document */ - ulint len_p2) /*!< in: no. of elements in p2 */ -{ - int i; - ulint len = 0; - ulint r = len_p1; - ulint c = len_p2; - ulint size = (r + 1) * (c + 1) * sizeof(ulint); - ulint* table = (ulint*) ut_malloc_nokey(size); - - /* Traverse the table backwards, from the last row to the first and - also from the last column to the first. We compute the smaller - common subsequences first, then use the calculated values to determine - the longest common subsequence. The result will be in TABLE[0][0]. */ - for (i = r; i >= 0; --i) { - int j; - - for (j = c; j >= 0; --j) { - - if (p1[i] == (ulint) -1 || p2[j] == (ulint) -1) { - - FTS_ELEM(table, c, i, j) = 0; - - } else if (p1[i] == p2[j]) { - - FTS_ELEM(table, c, i, j) = FTS_ELEM( - table, c, i + 1, j + 1) + 1; - - } else { - - ulint value; - - value = ut_max( - FTS_ELEM(table, c, i + 1, j), - FTS_ELEM(table, c, i, j + 1)); - - FTS_ELEM(table, c, i, j) = value; - } - } - } - - len = FTS_ELEM(table, c, 0, 0); - - fts_print_lcs_table(table, r, c); - printf("\nLen=" ULINTPF "\n", len); - - ut_free(table); - - return(len); -} -#endif /*******************************************************************//** Compare two fts_ranking_t instance on their rank value and doc ids in @@ -1121,10 +1135,8 @@ fts_query_difference( /* There is nothing we can substract from an empty set. */ if (query->doc_ids && !rbt_empty(query->doc_ids)) { ulint i; - fts_fetch_t fetch; const ib_vector_t* nodes; const fts_index_cache_t*index_cache; - que_t* graph = NULL; fts_cache_t* cache = table->fts->cache; dberr_t error; @@ -1162,21 +1174,21 @@ fts_query_difference( return(query->error); } - /* Setup the callback args for filtering and - consolidating the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, token, &fetch); + trx, query->index, token, query, + node_query_processor, compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); if (error != DB_SUCCESS) { query->error = error; } - - que_graph_free(graph); } /* The size can't increase. */ @@ -1222,10 +1234,8 @@ fts_query_intersect( if (!(rbt_empty(query->doc_ids) && query->multi_exist)) { ulint n_doc_ids = 0; ulint i; - fts_fetch_t fetch; const ib_vector_t* nodes; const fts_index_cache_t*index_cache; - que_t* graph = NULL; fts_cache_t* cache = table->fts->cache; dberr_t error; @@ -1296,13 +1306,15 @@ fts_query_intersect( return(query->error); } - /* Setup the callback args for filtering and - consolidating the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, token, &fetch); + trx, query->index, token, query, + node_query_processor, compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); @@ -1310,8 +1322,6 @@ fts_query_intersect( query->error = error; } - que_graph_free(graph); - if (query->error == DB_SUCCESS) { /* Make the intesection (rb tree) the current doc id set and free the old set. */ @@ -1389,10 +1399,8 @@ fts_query_union( fts_query_t* query, /*!< in: query instance */ fts_string_t* token) /*!< in: token to search */ { - fts_fetch_t fetch; ulint n_doc_ids = 0; trx_t* trx = query->trx; - que_t* graph = NULL; dberr_t error; ut_a(query->oper == FTS_NONE || query->oper == FTS_DECR_RATING || @@ -1417,14 +1425,16 @@ fts_query_union( fts_query_cache(query, token); - /* Setup the callback args for filtering and - consolidating the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } /* Read the nodes from disk. */ error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, token, &fetch); + trx, query->index, token, query, node_query_processor, + compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); @@ -1432,8 +1442,6 @@ fts_query_union( query->error = error; } - que_graph_free(graph); - if (query->error == DB_SUCCESS) { /* The size can't decrease. */ @@ -1950,476 +1958,232 @@ fts_query_match_phrase( return(phrase->found); } -/*****************************************************************//** -Callback function to fetch and search the document. +/** Callback function to fetch and search the document. +@param fts_index fulltext index +@param doc_id document id +@param arg user argument +@param expansion Expansion document @return whether the phrase is found */ static -ibool -fts_query_fetch_document( -/*=====================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ +dberr_t fts_query_fetch_document(dict_index_t *fts_index, + doc_id_t doc_id, + void *arg, bool expansion= false) { - - que_node_t* exp; - sel_node_t* node = static_cast(row); - fts_phrase_t* phrase = static_cast(user_arg); - ulint prev_len = 0; - ulint total_len = 0; - byte* document_text = NULL; - - exp = node->select_list; - - phrase->found = FALSE; - - /* For proximity search, we will need to get the whole document - from all fields, so first count the total length of the document - from all the fields */ - if (phrase->proximity_pos) { - while (exp) { - ulint field_len; - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - - if (dfield_is_ext(dfield)) { - ulint local_len = dfield_get_len(dfield); - - local_len -= BTR_EXTERN_FIELD_REF_SIZE; - - field_len = mach_read_from_4( - data + local_len + BTR_EXTERN_LEN + 4); - } else { - field_len = dfield_get_len(dfield); - } - - if (field_len != UNIV_SQL_NULL) { - total_len += field_len + 1; - } - - exp = que_node_get_next(exp); - } - - document_text = static_cast(mem_heap_zalloc( - phrase->heap, total_len)); - - if (!document_text) { - return(FALSE); - } - } - - exp = node->select_list; - - while (exp) { - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - ulint cur_len; - - if (dfield_is_ext(dfield)) { - data = btr_copy_externally_stored_field( - &cur_len, data, phrase->zip_size, - dfield_get_len(dfield), phrase->heap); - } else { - cur_len = dfield_get_len(dfield); - } - - if (cur_len != UNIV_SQL_NULL && cur_len != 0) { - if (phrase->proximity_pos) { - ut_ad(prev_len + cur_len <= total_len); - memcpy(document_text + prev_len, data, cur_len); - } else { - /* For phrase search */ - phrase->found = - fts_query_match_phrase( - phrase, - static_cast(data), - cur_len, prev_len, - phrase->heap); - } - - /* Document positions are calculated from the beginning - of the first field, need to save the length for each - searched field to adjust the doc position when search - phrases. */ - prev_len += cur_len + 1; - } - - if (phrase->found) { - break; - } - - exp = que_node_get_next(exp); - } - - if (phrase->proximity_pos) { - ut_ad(prev_len <= total_len); - - phrase->found = fts_proximity_is_word_in_range( - phrase, document_text, total_len); - } - - return(phrase->found); -} - -#if 0 -/******************************************************************** -Callback function to check whether a record was found or not. */ -static -ibool -fts_query_select( -/*=============*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ -{ - int i; - que_node_t* exp; - sel_node_t* node = row; - fts_select_t* select = user_arg; - - ut_a(select->word_freq); - ut_a(select->word_freq->doc_freqs); - - exp = node->select_list; - - for (i = 0; exp && !select->found; ++i) { - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - - switch (i) { - case 0: /* DOC_COUNT */ - if (len != UNIV_SQL_NULL && len != 0) { - - select->word_freq->doc_count += - mach_read_from_4(data); - } - break; - - case 1: /* ILIST */ - if (len != UNIV_SQL_NULL && len != 0) { - - fts_query_find_doc_id(select, data, len); - } - break; - - default: - ut_error; - } - - exp = que_node_get_next(exp); - } - - return(FALSE); -} - -/******************************************************************** -Read the rows from the FTS index, that match word and where the -doc id is between first and last doc id. -@return DB_SUCCESS if all go well else error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_query_find_term( -/*================*/ - fts_query_t* query, /*!< in: FTS query state */ - que_t** graph, /*!< in: prepared statement */ - const fts_string_t* word, /*!< in: the word to fetch */ - doc_id_t doc_id, /*!< in: doc id to match */ - ulint* min_pos,/*!< in/out: pos found must be - greater than this minimum value. */ - ibool* found) /*!< out: TRUE if found else FALSE */ -{ - pars_info_t* info; - dberr_t error; - fts_select_t select; - doc_id_t match_doc_id; - trx_t* trx = query->trx; - char table_name[MAX_FULL_NAME_LEN]; - - trx->op_info = "fetching FTS index matching nodes"; - - if (*graph) { - info = (*graph)->info; - } else { - ulint selected; - - info = pars_info_create(); - - selected = fts_select_index(*word->f_str); - query->fts_index_table.suffix = fts_get_suffix(selected); - - fts_get_table_name(&query->fts_index_table, table_name); - pars_info_bind_id(info, "index_table_name", table_name); - } - - select.found = FALSE; - select.doc_id = doc_id; - select.min_pos = *min_pos; - select.word_freq = fts_query_add_word_freq(query, word->f_str); - - pars_info_bind_function(info, "my_func", fts_query_select, &select); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &match_doc_id, doc_id); - - fts_bind_doc_id(info, "min_doc_id", &match_doc_id); - - fts_bind_doc_id(info, "max_doc_id", &match_doc_id); - - if (!*graph) { - - *graph = fts_parse_sql( - &query->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT doc_count, ilist\n" - " FROM $index_table_name\n" - " WHERE word LIKE :word AND" - " first_doc_id <= :min_doc_id AND" - " last_doc_id >= :max_doc_id\n" - " ORDER BY first_doc_id;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - } - - for (;;) { - error = fts_eval_sql(trx, *graph); - - if (error == DB_SUCCESS) { - - break; /* Exit the loop. */ - } else { - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading FTS" - " index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << error - << " while reading FTS index."; - - break; /* Exit the loop. */ - } - } - } - - /* Value to return */ - *found = select.found; - - if (*found) { - *min_pos = select.min_pos; - } - - return(error); -} - -/******************************************************************** -Callback aggregator for int columns. */ -static -ibool -fts_query_sum( -/*==========*/ - /*!< out: always returns TRUE */ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: ulint* */ -{ - - que_node_t* exp; - sel_node_t* node = row; - ulint* total = user_arg; - - exp = node->select_list; - - while (exp) { - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint len = dfield_get_len(dfield); - - if (len != UNIV_SQL_NULL && len != 0) { - *total += mach_read_from_4(data); - } - - exp = que_node_get_next(exp); - } - - return(TRUE); -} - -/******************************************************************** -Calculate the total documents that contain a particular word (term). -@return DB_SUCCESS if all go well else error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_query_total_docs_containing_term( -/*=================================*/ - fts_query_t* query, /*!< in: FTS query state */ - const fts_string_t* word, /*!< in: the word to check */ - ulint* total) /*!< out: documents containing word */ -{ - pars_info_t* info; - dberr_t error; - que_t* graph; - ulint selected; - trx_t* trx = query->trx; - char table_name[MAX_FULL_NAME_LEN] - - trx->op_info = "fetching FTS index document count"; - - *total = 0; - - info = pars_info_create(); - - pars_info_bind_function(info, "my_func", fts_query_sum, total); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - selected = fts_select_index(*word->f_str); - - query->fts_index_table.suffix = fts_get_suffix(selected); - - fts_get_table_name(&query->fts_index_table, table_name); - - pars_info_bind_id(info, "index_table_name", table_name); - - graph = fts_parse_sql( - &query->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT doc_count\n" - " FROM $index_table_name\n" - " WHERE word = :word" - " ORDER BY first_doc_id;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (error == DB_SUCCESS) { - - break; /* Exit the loop. */ - } else { - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading FTS" - " index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << error - << " while reading FTS index."; - - break; /* Exit the loop. */ - } - } - } - - que_graph_free(graph); - - return(error); -} - -/******************************************************************** -Get the total number of words in a documents. -@return DB_SUCCESS if all go well else error code */ -static MY_ATTRIBUTE((nonnull, warn_unused_result)) -dberr_t -fts_query_terms_in_document( -/*========================*/ - fts_query_t* query, /*!< in: FTS query state */ - doc_id_t doc_id, /*!< in: the word to check */ - ulint* total) /*!< out: total words in document */ -{ - pars_info_t* info; - dberr_t error; - que_t* graph; - doc_id_t read_doc_id; - trx_t* trx = query->trx; - char table_name[MAX_FULL_NAME_LEN]; - - trx->op_info = "fetching FTS document term count"; - - *total = 0; - - info = pars_info_create(); - - pars_info_bind_function(info, "my_func", fts_query_sum, total); - - /* Convert to "storage" byte order. */ - fts_write_doc_id((byte*) &read_doc_id, doc_id); - fts_bind_doc_id(info, "doc_id", &read_doc_id); - - query->fts_index_table.suffix = "DOC_ID"; - - fts_get_table_name(&query->fts_index_table, table_name); - - pars_info_bind_id(info, "index_table_name", table_name); - - graph = fts_parse_sql( - &query->fts_index_table, - info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT count\n" - " FROM $index_table_name\n" - " WHERE doc_id = :doc_id" - " BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (error == DB_SUCCESS) { - - break; /* Exit the loop. */ - } else { - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "lock wait timeout reading FTS" - " doc id table. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << error << " while reading FTS" - " doc id table."; - - break; /* Exit the loop. */ - } - } - } - - que_graph_free(graph); - - return(error); -} + trx_t *trx= trx_create(); + trx->op_info= "fetching FTS document for query"; + dict_table_t *user_table= fts_index->table; + dict_index_t *fts_doc_id_index= user_table->fts_doc_id_index; + dict_index_t *clust_index= dict_table_get_first_index(user_table); + ut_a(user_table->fts->doc_col != ULINT_UNDEFINED); + ut_a(fts_doc_id_index); + + QueryExecutor executor(trx); + + /* Map FTS index columns to clustered index field positions */ + ulint *clust_field_nos= static_cast( + mem_heap_alloc(executor.get_heap(), + fts_index->n_user_defined_cols * sizeof(ulint))); + + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + dict_field_t* fts_field= dict_index_get_nth_field(fts_index, i); + clust_field_nos[i]= dict_col_get_index_pos(fts_field->col, clust_index); + } + dfield_t fields[1]; + dtuple_t search_tuple{0, 1, 1, 0, fields, nullptr +#ifdef UNIV_DEBUG + , DATA_TUPLE_MAGIC_N #endif + }; + dict_index_copy_types(&search_tuple, fts_doc_id_index, 1); + dfield_t* dfield= dtuple_get_nth_field(&search_tuple, 0); + doc_id_t write_doc_id; + fts_write_doc_id((byte*) &write_doc_id, doc_id); + dfield_set_data(dfield, &write_doc_id, sizeof(write_doc_id)); + + auto process_expansion_doc= [arg, fts_index, + clust_field_nos](const rec_t* rec, + const dict_index_t *index, + const rec_offs *offsets)-> bool + { + fts_doc_t *result_doc= static_cast(arg); + fts_doc_t doc; + CHARSET_INFO *doc_charset= result_doc->charset; + fts_doc_init(&doc); + doc.found= TRUE; + + ulint doc_len= 0; + ulint field_no= 0; + + /* Process each indexed column content */ + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, + col_pos, &field_len); + + /* NULL column */ + if (field_len == UNIV_SQL_NULL) { + continue; + } + + /* Determine document charset from column if not provided */ + if (!doc_charset) + { + const dict_field_t* ifield= dict_index_get_nth_field(fts_index, i); + doc_charset= fts_get_charset(ifield->col->prtype); + } + + doc.charset= doc_charset; + /* Skip columns stored externally, as in fts_query_expansion_fetch_doc */ + if (rec_offs_nth_extern(offsets, col_pos)) { + continue; + } + + /* Use inline field data */ + doc.text.f_n_char= 0; + doc.text.f_str= const_cast(field_data); + doc.text.f_len= field_len; + + if (field_no == 0) + fts_tokenize_document(&doc, result_doc, result_doc->parser); + else + fts_tokenize_document_next(&doc, doc_len, result_doc, + result_doc->parser); + + /* Next field offset: add 1 for separator if more fields follow */ + doc_len+= ((i + 1) < fts_index->n_user_defined_cols) + ? field_len + 1 + : field_len; + + field_no++; + } + + ut_ad(doc_charset); + if (!result_doc->charset) { + result_doc->charset= doc_charset; + } + + fts_doc_free(&doc); + + return true; /* continue */ + }; + + auto process_doc_query= [arg, user_table, fts_index, + clust_field_nos](const rec_t* rec, + const dict_index_t* index, + const rec_offs* offsets) -> bool + { + ulint prev_len= 0; + ulint total_len= 0; + byte *document_text= nullptr; + + fts_phrase_t *phrase= static_cast(arg); + phrase->found= FALSE; + + /* Extract doc_id from the clustered index record */ + ulint doc_col_pos= dict_col_get_index_pos( + &user_table->cols[user_table->fts->doc_col], index); + + ulint len; + rec_get_nth_field_offs(offsets, doc_col_pos, &len); + if (len != sizeof(doc_id_t)) + return true; + + /* For proximity search, first count total document length */ + if (phrase->proximity_pos) + { + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, + col_pos, &field_len); + if (rec_offs_nth_extern(offsets, col_pos)) + { + ulint local_len= field_len; + local_len-= BTR_EXTERN_FIELD_REF_SIZE; + field_len= mach_read_from_4( + field_data + local_len + BTR_EXTERN_LEN + 4); + } + if (field_len != UNIV_SQL_NULL) + total_len+= field_len + 1; + } + + document_text= + static_cast(mem_heap_zalloc(phrase->heap, total_len)); + if (!document_text) + return false; + } + + /* Process each indexed column content */ + for (ulint i= 0; i < fts_index->n_user_defined_cols; i++) + { + ulint col_pos= clust_field_nos[i]; + ulint field_len; + const byte* field_data= rec_get_nth_field(rec, offsets, + col_pos, &field_len); + byte* data= const_cast(field_data); + ulint cur_len; + + if (rec_offs_nth_extern(offsets, col_pos)) + { + data= btr_copy_externally_stored_field( + &cur_len, const_cast(field_data), phrase->zip_size, + field_len, phrase->heap); + } + else cur_len= field_len; + if (cur_len != UNIV_SQL_NULL && cur_len != 0) + { + if (phrase->proximity_pos) + { + ut_ad(prev_len + cur_len <= total_len); + memcpy(document_text + prev_len, data, cur_len); + } + else + { + /* For phrase search */ + phrase->found= fts_query_match_phrase( + phrase, data, cur_len, prev_len, phrase->heap); + } + + /* Document positions are calculated from the beginning + of the first field, need to save the length for each + searched field to adjust the doc position when search + phrases. */ + prev_len+= cur_len + 1; + } + + if (phrase->found) + break; + } + + if (phrase->proximity_pos) + { + ut_ad(prev_len <= total_len); + phrase->found= fts_proximity_is_word_in_range( + phrase, document_text, total_len); + } + + return !phrase->found; /* Continue only if not found */ + }; + + RecordProcessor proc= expansion + ? RecordProcessor(process_expansion_doc) + : RecordProcessor(process_doc_query); + RecordCallback reader(proc, doc_id_exact_match_comparator); + dberr_t err= DB_SUCCESS; + if (fts_doc_id_index == clust_index) + err= executor.read(user_table, &search_tuple, PAGE_CUR_GE, reader); + else + err= executor.read_by_index(user_table, fts_doc_id_index, + &search_tuple, PAGE_CUR_GE, reader); + trx_commit_for_mysql(trx); + trx->free(); + if (err == DB_RECORD_NOT_FOUND) err= DB_SUCCESS; + return err; +} /*****************************************************************//** Retrieve the document and match the phrase tokens. @@ -2448,9 +2212,8 @@ fts_query_match_document( *found = phrase.found = FALSE; - error = fts_doc_fetch_by_doc_id( - get_doc, match->doc_id, NULL, FTS_FETCH_DOC_BY_ID_EQUAL, - fts_query_fetch_document, &phrase); + error = fts_query_fetch_document( + get_doc->index_cache->index, match->doc_id, &phrase); if (UNIV_UNLIKELY(error != DB_SUCCESS)) { ib::error() << "(" << error << ") matching document."; @@ -2495,21 +2258,14 @@ fts_query_is_in_proximity_range( phrase.proximity_pos = qualified_pos; phrase.found = FALSE; - err = fts_doc_fetch_by_doc_id( - &get_doc, match[0]->doc_id, NULL, FTS_FETCH_DOC_BY_ID_EQUAL, - fts_query_fetch_document, &phrase); + err = fts_query_fetch_document( + get_doc.index_cache->index, match[0]->doc_id, &phrase); if (UNIV_UNLIKELY(err != DB_SUCCESS)) { ib::error() << "(" << err << ") in verification" " phase of proximity search"; } - /* Free the prepared statement. */ - if (get_doc.get_document_graph) { - que_graph_free(get_doc.get_document_graph); - get_doc.get_document_graph = NULL; - } - mem_heap_free(phrase.heap); return(err == DB_SUCCESS && phrase.found); @@ -2622,6 +2378,7 @@ fts_query_phrase_split( ulint len = 0; ulint cur_pos = 0; fts_ast_node_t* term_node = NULL; + CHARSET_INFO* cs = fts_index_get_charset(query->index); if (node->type == FTS_AST_TEXT) { phrase.f_str = node->text.ptr->str; @@ -2645,7 +2402,7 @@ fts_query_phrase_split( } cur_len = innobase_mysql_fts_get_token( - query->fts_index_table.charset, + cs, reinterpret_cast(phrase.f_str) + cur_pos, reinterpret_cast(phrase.f_str) @@ -2668,7 +2425,7 @@ fts_query_phrase_split( result_str.f_str = term_node->term.ptr->str; result_str.f_len = term_node->term.ptr->len; result_str.f_n_char = fts_get_token_size( - query->fts_index_table.charset, + cs, reinterpret_cast(result_str.f_str), result_str.f_len); @@ -2685,8 +2442,7 @@ fts_query_phrase_split( if (fts_check_token( &result_str, - cache->stopword_info.cached_stopword, - query->fts_index_table.charset)) { + cache->stopword_info.cached_stopword, cs)) { /* Add the word to the RB tree so that we can calculate its frequency within a document. */ fts_query_add_word_freq(query, token); @@ -2748,10 +2504,8 @@ fts_query_phrase_search( /* Ignore empty strings. */ if (num_token > 0) { fts_string_t* token = NULL; - fts_fetch_t fetch; trx_t* trx = query->trx; fts_ast_oper_t oper = query->oper; - que_t* graph = NULL; ulint i; dberr_t error; @@ -2786,11 +2540,6 @@ fts_query_phrase_search( } } - /* Setup the callback args for filtering and consolidating - the ilist. */ - fetch.read_arg = query; - fetch.read_record = fts_query_index_fetch_nodes; - for (i = 0; i < num_token; i++) { /* Search for the first word from the phrase. */ token = static_cast( @@ -2801,9 +2550,15 @@ fts_query_phrase_search( query->matched = query->match_array[i]; } + AuxCompareMode compare_mode = AuxCompareMode::EQUAL; + if (query->cur_node->type == FTS_AST_TERM && + query->cur_node->term.wildcard) { + compare_mode = AuxCompareMode::LIKE; + } + error = fts_index_fetch_nodes( - trx, &graph, &query->fts_index_table, - token, &fetch); + trx, query->index, token, query, + node_query_processor, compare_mode); /* DB_FTS_EXCEED_RESULT_CACHE_LIMIT passed by 'query->error' */ ut_ad(!(query->error != DB_SUCCESS && error != DB_SUCCESS)); @@ -2811,9 +2566,6 @@ fts_query_phrase_search( query->error = error; } - que_graph_free(graph); - graph = NULL; - fts_query_cache(query, token); if (!(query->flags & FTS_PHRASE) @@ -2954,12 +2706,11 @@ fts_query_get_token( if (node->term.wildcard) { - token->f_str = static_cast(ut_malloc_nokey(str_len + 2)); - token->f_len = str_len + 1; + token->f_str = static_cast(ut_malloc_nokey(str_len + 1)); + token->f_len = str_len; memcpy(token->f_str, node->term.ptr->str, str_len); - token->f_str[str_len] = '%'; token->f_str[token->f_len] = 0; new_ptr = token->f_str; @@ -3140,78 +2891,6 @@ fts_ast_visit_sub_exp( DBUG_RETURN(error); } -#if 0 -/*****************************************************************//*** -Check if the doc id exists in the ilist. -@return TRUE if doc id found */ -static -ulint -fts_query_find_doc_id( -/*==================*/ - fts_select_t* select, /*!< in/out: contains the doc id to - find, we update the word freq if - document found */ - void* data, /*!< in: doc id ilist */ - ulint len) /*!< in: doc id ilist size */ -{ - byte* ptr = data; - doc_id_t doc_id = 0; - ulint decoded = 0; - - /* Decode the ilist and search for selected doc_id. We also - calculate the frequency of the word in the document if found. */ - while (decoded < len && !select->found) { - ulint freq = 0; - ulint min_pos = 0; - ulint last_pos = 0; - ulint pos = fts_decode_vlc(&ptr); - - /* Add the delta. */ - doc_id += pos; - - while (*ptr) { - ++freq; - last_pos += fts_decode_vlc(&ptr); - - /* Only if min_pos is not set and the current - term exists in a position greater than the - min_pos of the previous term. */ - if (min_pos == 0 && last_pos > select->min_pos) { - min_pos = last_pos; - } - } - - /* Skip the end of word position marker. */ - ++ptr; - - /* Bytes decoded so far. */ - decoded = ptr - (byte*) data; - - /* A word may exist in the document but we only consider a - match if it exists in a position that is greater than the - position of the previous term. */ - if (doc_id == select->doc_id && min_pos > 0) { - fts_doc_freq_t* doc_freq; - - /* Add the doc id to the doc freq rb tree, if - the doc id doesn't exist it will be created. */ - doc_freq = fts_query_add_doc_freq( - select->word_freq->doc_freqs, doc_id); - - /* Avoid duplicating the frequency tally */ - if (doc_freq->freq == 0) { - doc_freq->freq = freq; - } - - select->found = TRUE; - select->min_pos = min_pos; - } - } - - return(select->found); -} -#endif - /*****************************************************************//** Read and filter nodes. @return DB_SUCCESS if all go well, @@ -3332,156 +3011,6 @@ fts_query_filter_doc_ids( } } -/*****************************************************************//** -Read the FTS INDEX row. -@return DB_SUCCESS if all go well. */ -static -dberr_t -fts_query_read_node( -/*================*/ - fts_query_t* query, /*!< in: query instance */ - const fts_string_t* word, /*!< in: current word */ - que_node_t* exp) /*!< in: query graph node */ -{ - int i; - int ret; - fts_node_t node; - ib_rbt_bound_t parent; - fts_word_freq_t* word_freq; - ibool skip = FALSE; - fts_string_t term; - byte buf[FTS_MAX_WORD_LEN + 1]; - dberr_t error = DB_SUCCESS; - - ut_a(query->cur_node->type == FTS_AST_TERM - || query->cur_node->type == FTS_AST_TEXT - || query->cur_node->type == FTS_AST_PARSER_PHRASE_LIST); - - memset(&node, 0, sizeof(node)); - term.f_str = buf; - - /* Need to consider the wildcard search case, the word frequency - is created on the search string not the actual word. So we need - to assign the frequency on search string behalf. */ - if (query->cur_node->type == FTS_AST_TERM - && query->cur_node->term.wildcard) { - - term.f_len = query->cur_node->term.ptr->len; - ut_ad(FTS_MAX_WORD_LEN >= term.f_len); - memcpy(term.f_str, query->cur_node->term.ptr->str, term.f_len); - } else { - term.f_len = word->f_len; - ut_ad(FTS_MAX_WORD_LEN >= word->f_len); - memcpy(term.f_str, word->f_str, word->f_len); - } - - /* Lookup the word in our rb tree, it must exist. */ - ret = rbt_search(query->word_freqs, &parent, &term); - - ut_a(ret == 0); - - word_freq = rbt_value(fts_word_freq_t, parent.last); - - /* Start from 1 since the first column has been read by the caller. - Also, we rely on the order of the columns projected, to filter - out ilists that are out of range and we always want to read - the doc_count irrespective of the suitability of the row. */ - - for (i = 1; exp && !skip; exp = que_node_get_next(exp), ++i) { - - dfield_t* dfield = que_node_get_val(exp); - byte* data = static_cast( - dfield_get_data(dfield)); - ulint len = dfield_get_len(dfield); - - ut_a(len != UNIV_SQL_NULL); - - /* Note: The column numbers below must match the SELECT. */ - - switch (i) { - case 1: /* DOC_COUNT */ - word_freq->doc_count += mach_read_from_4(data); - break; - - case 2: /* FIRST_DOC_ID */ - node.first_doc_id = fts_read_doc_id(data); - - /* Skip nodes whose doc ids are out range. */ - if (query->oper == FTS_EXIST - && query->upper_doc_id > 0 - && node.first_doc_id > query->upper_doc_id) { - skip = TRUE; - } - break; - - case 3: /* LAST_DOC_ID */ - node.last_doc_id = fts_read_doc_id(data); - - /* Skip nodes whose doc ids are out range. */ - if (query->oper == FTS_EXIST - && query->lower_doc_id > 0 - && node.last_doc_id < query->lower_doc_id) { - skip = TRUE; - } - break; - - case 4: /* ILIST */ - - error = fts_query_filter_doc_ids( - query, &word_freq->word, word_freq, - &node, data, len, FALSE); - - break; - - default: - ut_error; - } - } - - if (!skip) { - /* Make sure all columns were read. */ - - ut_a(i == 5); - } - - return error; -} - -/*****************************************************************//** -Callback function to fetch the rows in an FTS INDEX record. -@return always returns TRUE */ -static -ibool -fts_query_index_fetch_nodes( -/*========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: pointer to fts_fetch_t */ -{ - fts_string_t key; - sel_node_t* sel_node = static_cast(row); - fts_fetch_t* fetch = static_cast(user_arg); - fts_query_t* query = static_cast(fetch->read_arg); - que_node_t* exp = sel_node->select_list; - dfield_t* dfield = que_node_get_val(exp); - void* data = dfield_get_data(dfield); - ulint dfield_len = dfield_get_len(dfield); - - key.f_str = static_cast(data); - key.f_len = dfield_len; - - ut_a(dfield_len <= FTS_MAX_WORD_LEN); - - /* Note: we pass error out by 'query->error' */ - query->error = fts_query_read_node(query, &key, que_node_get_next(exp)); - - if (query->error != DB_SUCCESS) { - ut_ad(query->error == DB_FTS_EXCEED_RESULT_CACHE_LIMIT); - return(FALSE); - } else { - return(TRUE); - } -} - /*****************************************************************//** Calculate the inverse document frequency (IDF) for all the terms. */ static @@ -3868,7 +3397,7 @@ fts_query_parse( memset(&state, 0x0, sizeof(state)); - state.charset = query->fts_index_table.charset; + state.charset = fts_index_get_charset(query->index); DBUG_EXECUTE_IF("fts_instrument_query_disable_parser", query->parser = NULL;); @@ -3881,7 +3410,7 @@ fts_query_parse( } else { /* Setup the scanner to use, this depends on the mode flag. */ state.lexer = fts_lexer_create(mode, query_str, query_len); - state.charset = query->fts_index_table.charset; + state.charset = fts_index_get_charset(query->index); error = fts_parse(&state); fts_lexer_free(state.lexer); state.lexer = NULL; @@ -3969,10 +3498,6 @@ fts_query( query.deleted = fts_doc_ids_create(); query.cur_node = NULL; - query.fts_common_table.type = FTS_COMMON_TABLE; - query.fts_common_table.table_id = index->table->id; - query.fts_common_table.table = index->table; - charset = fts_index_get_charset(index); query.fts_index_table.type = FTS_INDEX_TABLE; @@ -4001,20 +3526,16 @@ fts_query( query.total_docs = dict_table_get_n_rows(index->table); - query.fts_common_table.suffix = "DELETED"; - /* Read the deleted doc_ids, we need these for filtering. */ error = fts_table_fetch_doc_ids( - NULL, &query.fts_common_table, query.deleted); + nullptr, index->table, "DELETED", query.deleted); if (error != DB_SUCCESS) { goto func_exit; } - query.fts_common_table.suffix = "DELETED_CACHE"; - error = fts_table_fetch_doc_ids( - NULL, &query.fts_common_table, query.deleted); + nullptr, index->table, "DELETED_CACHE", query.deleted); if (error != DB_SUCCESS) { goto func_exit; @@ -4295,10 +3816,8 @@ fts_expand_query( fetch the original document and parse them. Future optimization could be done here if we support some forms of document-to-word mapping */ - fts_doc_fetch_by_doc_id(NULL, ranking->doc_id, index, - FTS_FETCH_DOC_BY_ID_EQUAL, - fts_query_expansion_fetch_doc, - &result_doc); + fts_query_fetch_document(index, ranking->doc_id, + &result_doc, true); /* Estimate memory used, see fts_process_token and fts_token_t. We ignore token size here. */ diff --git a/storage/innobase/fts/fts0sql.cc b/storage/innobase/fts/fts0sql.cc deleted file mode 100644 index 781d15f2befb0..0000000000000 --- a/storage/innobase/fts/fts0sql.cc +++ /dev/null @@ -1,208 +0,0 @@ -/***************************************************************************** - -Copyright (c) 2007, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2019, 2021, MariaDB Corporation. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA - -*****************************************************************************/ - -/**************************************************//** -@file fts/fts0sql.cc -Full Text Search functionality. - -Created 2007-03-27 Sunny Bains -*******************************************************/ - -#include "que0que.h" -#include "trx0roll.h" -#include "pars0pars.h" -#include "dict0dict.h" -#include "fts0types.h" -#include "fts0priv.h" - -/** SQL statements for creating the ancillary FTS tables. */ - -/** Preamble to all SQL statements. */ -static const char* fts_sql_begin= - "PROCEDURE P() IS\n"; - -/** Postamble to non-committing SQL statements. */ -static const char* fts_sql_end= - "\n" - "END;\n"; - -/******************************************************************//** -Get the table id. -@return number of bytes written */ -int -fts_get_table_id( -/*=============*/ - const fts_table_t* - fts_table, /*!< in: FTS Auxiliary table */ - char* table_id) /*!< out: table id, must be at least - FTS_AUX_MIN_TABLE_ID_LENGTH bytes - long */ -{ - int len; - - ut_a(fts_table->table != NULL); - - switch (fts_table->type) { - case FTS_COMMON_TABLE: - len = fts_write_object_id(fts_table->table_id, table_id); - break; - - case FTS_INDEX_TABLE: - - len = fts_write_object_id(fts_table->table_id, table_id); - - table_id[len] = '_'; - ++len; - table_id += len; - - len += fts_write_object_id(fts_table->index_id, table_id); - break; - - default: - ut_error; - } - - ut_a(len >= 16); - ut_a(len < FTS_AUX_MIN_TABLE_ID_LENGTH); - - return(len); -} - -/** Construct the name of an internal FTS table for the given table. -@param[in] fts_table metadata on fulltext-indexed table -@param[out] table_name a name up to MAX_FULL_NAME_LEN -@param[in] dict_locked whether dict_sys.latch is being held */ -void fts_get_table_name(const fts_table_t* fts_table, char* table_name, - bool dict_locked) -{ - if (!dict_locked) { - dict_sys.freeze(SRW_LOCK_CALL); - } - ut_ad(dict_sys.frozen()); - /* Include the separator as well. */ - const size_t dbname_len = fts_table->table->name.dblen() + 1; - ut_ad(dbname_len > 1); - memcpy(table_name, fts_table->table->name.m_name, dbname_len); - if (!dict_locked) { - dict_sys.unfreeze(); - } - memcpy(table_name += dbname_len, "FTS_", 4); - table_name += 4; - table_name += fts_get_table_id(fts_table, table_name); - *table_name++ = '_'; - strcpy(table_name, fts_table->suffix); -} - -/******************************************************************//** -Parse an SQL string. -@return query graph */ -que_t* -fts_parse_sql( -/*==========*/ - fts_table_t* fts_table, /*!< in: FTS auxiliary table info */ - pars_info_t* info, /*!< in: info struct, or NULL */ - const char* sql) /*!< in: SQL string to evaluate */ -{ - char* str; - que_t* graph; - ibool dict_locked; - - str = ut_str3cat(fts_sql_begin, sql, fts_sql_end); - - dict_locked = (fts_table && fts_table->table->fts - && fts_table->table->fts->dict_locked); - - if (!dict_locked) { - /* The InnoDB SQL parser is not re-entrant. */ - dict_sys.lock(SRW_LOCK_CALL); - } - - graph = pars_sql(info, str); - ut_a(graph); - - if (!dict_locked) { - dict_sys.unlock(); - } - - ut_free(str); - - return(graph); -} - -/******************************************************************//** -Evaluate an SQL query graph. -@return DB_SUCCESS or error code */ -dberr_t -fts_eval_sql( -/*=========*/ - trx_t* trx, /*!< in: transaction */ - que_t* graph) /*!< in: Query graph to evaluate */ -{ - que_thr_t* thr; - - graph->trx = trx; - - ut_a(thr = que_fork_start_command(graph)); - - que_run_threads(thr); - - return(trx->error_state); -} - -/******************************************************************//** -Construct the column specification part of the SQL string for selecting the -indexed FTS columns for the given table. Adds the necessary bound -ids to the given 'info' and returns the SQL string. Examples: - -One indexed column named "text": - - "$sel0", - info/ids: sel0 -> "text" - -Two indexed columns named "subject" and "content": - - "$sel0, $sel1", - info/ids: sel0 -> "subject", sel1 -> "content", -@return heap-allocated WHERE string */ -const char* -fts_get_select_columns_str( -/*=======================*/ - dict_index_t* index, /*!< in: index */ - pars_info_t* info, /*!< in/out: parser info */ - mem_heap_t* heap) /*!< in: memory heap */ -{ - ulint i; - const char* str = ""; - - for (i = 0; i < index->n_user_defined_cols; i++) { - char* sel_str; - - dict_field_t* field = dict_index_get_nth_field(index, i); - - sel_str = mem_heap_printf(heap, "sel%lu", (ulong) i); - - /* Set copy_name to TRUE since it's dynamic. */ - pars_info_bind_id(info, sel_str, field->name); - - str = mem_heap_printf( - heap, "%s%s$%s", str, (*str) ? ", " : "", sel_str); - } - - return(str); -} diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index eacde4912b89e..b1f38c19079c9 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -11449,6 +11449,7 @@ innobase_fts_load_stopword( bool success= fts_load_stopword(table, trx, stopword_table, THDVAR(thd, ft_enable_stopword), false); table->fts->dict_locked= false; + DBUG_EXECUTE_IF("fts_load_stopword_fail", success= false;); return success; } diff --git a/storage/innobase/handler/i_s.cc b/storage/innobase/handler/i_s.cc index 5d260c7f0b072..286447a6a9050 100644 --- a/storage/innobase/handler/i_s.cc +++ b/storage/innobase/handler/i_s.cc @@ -2204,7 +2204,6 @@ i_s_fts_deleted_generic_fill( Field** fields; TABLE* table = (TABLE*) tables->table; trx_t* trx; - fts_table_t fts_table; fts_doc_ids_t* deleted; dict_table_t* user_table; @@ -2235,11 +2234,9 @@ i_s_fts_deleted_generic_fill( trx = trx_create(); trx->op_info = "Select for FTS DELETE TABLE"; - FTS_INIT_FTS_TABLE(&fts_table, - (being_deleted) ? "BEING_DELETED" : "DELETED", - FTS_COMMON_TABLE, user_table); - - fts_table_fetch_doc_ids(trx, &fts_table, deleted); + fts_table_fetch_doc_ids( + nullptr, user_table, + being_deleted ? "BEING_DELETED" : "DELETED", deleted); dict_table_close(user_table, thd, mdl_ticket); @@ -2660,100 +2657,62 @@ struct st_maria_plugin i_s_innodb_ft_index_cache = MariaDB_PLUGIN_MATURITY_STABLE }; -/*******************************************************************//** -Go through a FTS index auxiliary table, fetch its rows and fill +/** Go through a FTS index auxiliary table, fetch its rows and fill FTS word cache structure. +@param index FTS index +@param words vector to hold fetched words +@param selected auxiliary index +@param word word to select @return DB_SUCCESS on success, otherwise error code */ static -dberr_t -i_s_fts_index_table_fill_selected( -/*==============================*/ - dict_index_t* index, /*!< in: FTS index */ - ib_vector_t* words, /*!< in/out: vector to hold - fetched words */ - ulint selected, /*!< in: selected FTS index */ - fts_string_t* word) /*!< in: word to select */ -{ - pars_info_t* info; - fts_table_t fts_table; - trx_t* trx; - que_t* graph; - dberr_t error; - fts_fetch_t fetch; - char table_name[MAX_FULL_NAME_LEN]; - - info = pars_info_create(); - - fetch.read_arg = words; - fetch.read_record = fts_optimize_index_fetch_node; - fetch.total_memory = 0; - - DBUG_EXECUTE_IF("fts_instrument_result_cache_limit", - fts_result_cache_limit = 8192; - ); - - trx = trx_create(); - - trx->op_info = "fetching FTS index nodes"; - - pars_info_bind_function(info, "my_func", fetch.read_record, &fetch); - pars_info_bind_varchar_literal(info, "word", word->f_str, word->f_len); - - FTS_INIT_INDEX_TABLE(&fts_table, fts_get_suffix(selected), - FTS_INDEX_TABLE, index); - fts_get_table_name(&fts_table, table_name); - pars_info_bind_id(info, "table_name", table_name); - - graph = fts_parse_sql( - &fts_table, info, - "DECLARE FUNCTION my_func;\n" - "DECLARE CURSOR c IS" - " SELECT word, doc_count, first_doc_id, last_doc_id," - " ilist\n" - " FROM $table_name WHERE word >= :word;\n" - "BEGIN\n" - "\n" - "OPEN c;\n" - "WHILE 1 = 1 LOOP\n" - " FETCH c INTO my_func();\n" - " IF c % NOTFOUND THEN\n" - " EXIT;\n" - " END IF;\n" - "END LOOP;\n" - "CLOSE c;"); - - for (;;) { - error = fts_eval_sql(trx, graph); - - if (UNIV_LIKELY(error == DB_SUCCESS)) { - fts_sql_commit(trx); - - break; - } else { - fts_sql_rollback(trx); - - if (error == DB_LOCK_WAIT_TIMEOUT) { - ib::warn() << "Lock wait timeout reading" - " FTS index. Retrying!"; - - trx->error_state = DB_SUCCESS; - } else { - ib::error() << "Error occurred while reading" - " FTS index: " << error; - break; - } - } - } - - que_graph_free(graph); - - trx->free(); - - if (fetch.total_memory >= fts_result_cache_limit) { - error = DB_FTS_EXCEED_RESULT_CACHE_LIMIT; - } +dberr_t i_s_fts_index_table_fill_selected( + dict_index_t *index, ib_vector_t *words, ulint selected, + fts_string_t *word) +{ + dberr_t error= DB_SUCCESS; + ulint total_memory= 0; + DBUG_EXECUTE_IF("fts_instrument_result_cache_limit", + fts_result_cache_limit = 8192;); + trx_t* trx= trx_create(); + trx->op_info= "fetching FTS index nodes"; + for (;;) + { + FTSQueryExecutor executor(trx, index, index->table); + AuxRecordReader reader(words, &total_memory); + if (word->f_str == nullptr) + error= executor.read_aux_all((uint8_t) selected, reader); + else + error= executor.read_aux( + (uint8_t) selected, + reinterpret_cast(word->f_str), + PAGE_CUR_GE, reader); - return(error); + if (UNIV_LIKELY(error == DB_SUCCESS || + error == DB_RECORD_NOT_FOUND)) + { + fts_sql_commit(trx); + if (error == DB_RECORD_NOT_FOUND) error = DB_SUCCESS; + break; + } + else + { + fts_sql_rollback(trx); + if (error == DB_LOCK_WAIT_TIMEOUT) + { + ib::warn() << "Lock wait timeout reading FTS index. Retrying!"; + trx->error_state = DB_SUCCESS; + } + else + { + ib::error() << "Error occurred while reading FTS index: " << error; + break; + } + } + } + trx->free(); + if (total_memory >= fts_result_cache_limit) + error= DB_FTS_EXCEED_RESULT_CACHE_LIMIT; + return error; } /*******************************************************************//** @@ -3116,7 +3075,6 @@ i_s_fts_config_fill( Field** fields; TABLE* table = (TABLE*) tables->table; trx_t* trx; - fts_table_t fts_table; dict_table_t* user_table; ulint i = 0; dict_index_t* index = NULL; @@ -3150,8 +3108,6 @@ i_s_fts_config_fill( trx = trx_create(); trx->op_info = "Select for FTS CONFIG TABLE"; - FTS_INIT_FTS_TABLE(&fts_table, "CONFIG", FTS_COMMON_TABLE, user_table); - if (!ib_vector_is_empty(user_table->fts->indexes)) { index = (dict_index_t*) ib_vector_getp_const( user_table->fts->indexes, 0); @@ -3178,7 +3134,7 @@ i_s_fts_config_fill( key_name = (char*) fts_config_key[i]; } - fts_config_get_value(trx, &fts_table, key_name, &value); + fts_config_get_value(trx, user_table, key_name, &value); if (allocated) { ut_free(key_name); diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 53f88cc8ca1f5..9387adb234f24 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -346,7 +346,7 @@ btr_cur_del_mark_set_clust_rec( que_thr_t* thr, /*!< in: query thread */ const dtuple_t* entry, /*!< in: dtuple for the deleting record */ mtr_t* mtr) /*!< in/out: mini-transaction */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); + MY_ATTRIBUTE((nonnull(1,2,3,4,5,7), warn_unused_result)); /*************************************************************//** Tries to compress a page of the tree if it seems useful. It is assumed that mtr holds an x-latch on the tree and on the cursor page. To avoid diff --git a/storage/innobase/include/fts0exec.h b/storage/innobase/include/fts0exec.h new file mode 100644 index 0000000000000..b420fe74972bc --- /dev/null +++ b/storage/innobase/include/fts0exec.h @@ -0,0 +1,306 @@ +/***************************************************************************** + +Copyright (c) 2025, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/fts0exec.h +FTS Query Builder - Abstraction layer for FTS operations + +Created 2025/10/30 +*******************************************************/ + +#ifndef INNOBASE_FTS0QUERY_H +#define INNOBASE_FTS0QUERY_H + +#include "row0query.h" +#include "fts0fts.h" +#include "fts0types.h" +#include "fts0opt.h" +#include "fts0ast.h" +#include + +/** Structure to represent FTS auxiliary table data for insertion */ +struct fts_aux_data_t +{ + /** + CREATE TABLE $FTS_PREFIX_INDEX_[1-6]( + word VARCHAR(FTS_MAX_WORD_LEN), + first_doc_id INT NOT NULL, + last_doc_id UNSIGNED NOT NULL, + doc_count UNSIGNED INT NOT NULL, + ilist VARBINARY NOT NULL, + UNIQUE CLUSTERED INDEX ON (word, first_doc_id)); + */ + const char *word; + ulint word_len; + doc_id_t first_doc_id; + doc_id_t last_doc_id; + uint32_t doc_count; + const byte *ilist; + ulint ilist_len; + + fts_aux_data_t(const char *w, ulint w_len) + : word(w), word_len(w_len) + { + first_doc_id= last_doc_id= doc_count= 0; + ilist= nullptr; + ilist_len= 0; + } + + fts_aux_data_t(const char* w, ulint w_len, doc_id_t first_id, + doc_id_t last_id, uint32_t d_count, const byte* il, + ulint il_len) + : word(w), word_len(w_len), first_doc_id(first_id), + last_doc_id(last_id), doc_count(d_count), ilist(il), + ilist_len(il_len) {} +}; + +/** Abstraction over QueryExecutor for FTS auxiliary/common tables. +Handles table open/lock and provides typed helpers to insert, +delete and read records in FTS INDEX_1..INDEX_6 +and common tables (DELETED, CONFIG, ...) */ +class FTSQueryExecutor +{ +private: + QueryExecutor *m_executor; + bool m_dict_locked; + const dict_index_t *m_fts_index; + const dict_table_t *m_fts_table; + dict_table_t *m_aux_tables[FTS_NUM_AUX_INDEX]; + dict_table_t *m_common_tables[FTS_NUM_AUX_INDEX - 1]; + + /** Table preparation methods */ + + /** Open FTS INDEX_[1..6] table for the given auxiliary index. + @return DB_SUCCESS or error code */ + dberr_t open_aux_table(uint8_t aux_index) noexcept; + + /** Open a common FTS table (DELETED, CONFIG, ...). + @return DB_SUCCESS or error code */ + dberr_t open_common_table(const char *tbl_name) noexcept; + + /** Table lock operation */ + + /** Acquire a lock on an opened INDEX_[1..6] table. + Retries lock wait once via QueryExecutor. + @return DB_SUCCESS or error code */ + dberr_t lock_aux_tables(uint8_t aux_index, lock_mode mode) noexcept; + + /** Acquire a lock on an opened common FTS table. + Retries lock wait once via QueryExecutor. + @return DB_SUCCESS or error code */ + dberr_t lock_common_tables(uint8_t index, lock_mode mode) noexcept; + +public: + /** Create executor bound to trx and FTS table/index. + @param trx transaction (not owned) + @param fts_index FTS index (for INDEX_[1..6] tables) or nullptr + @param fts_table FTS table (for common tables) or nullptr + @param dict_locked true if dict is already locked */ + FTSQueryExecutor(trx_t *trx, const dict_index_t *fts_index, + const dict_table_t *fts_table, bool dict_locked= false); + + /** Release any opened table handles and executor resources. */ + ~FTSQueryExecutor(); + + /** High level DML operation on FTS TABLE */ + + /** Insert a row into auxiliary INDEX_[1..6] table. + Expects (word, first_doc_id, trx_id, roll_ptr, last_doc_id, + doc_count, ilist). + @param aux_index auxiliary table index + @param aux_data data to be inserted + @return DB_SUCCESS or error code */ + dberr_t insert_aux_record(uint8_t aux_index, + const fts_aux_data_t *aux_data) noexcept; + + /** Insert a single doc_id into a common table (e.g. DELETED, ...) + @param tbl_name common table name + @param doc_id document id to be inserted + @return DB_SUCCESS or error code */ + dberr_t insert_common_record(const char *tbl_name, doc_id_t doc_id) noexcept; + + /** Insert a key/value into CONFIG table. + @param key key for the config table + @param value value for the key + @return DB_SUCCESS or error code */ + dberr_t insert_config_record(const char *key, const char *value) noexcept; + + /** Delete one word row from INDEX_[1..6] by (word). + @param aux_index auxiliary table index + @param aux_data auxiliary table record + @return DB_SUCCESS or error code */ + dberr_t delete_aux_record(uint8_t aux_index, + const fts_aux_data_t *aux_data) noexcept; + + /** Delete a single doc_id row from a common table by (doc_id). + @param tbl_name common table name + @param doc_id document id to be deleted + @return DB_SUCCESS or error code */ + dberr_t delete_common_record(const char *tbl_name, doc_id_t doc_id) noexcept; + + /** Delete all rows from a common table. + @return DB_SUCCESS or error code */ + dberr_t delete_all_common_records(const char *tbl_name) noexcept; + + /** Delete a key from CONFIG table by (key). + @return DB_SUCCESS or error code */ + dberr_t delete_config_record(const char *key) noexcept; + + /** Upsert a key/value in CONFIG table. + Replaces 'value' if key exists, inserts otherwise. + @return DB_SUCCESS or error code */ + dberr_t update_config_record(const char *key, const char *value) noexcept; + + /** Full scan of CONFIG table with callback. + @return DB_SUCCESS or error code */ + dberr_t read_all_config(RecordCallback& callback) noexcept; + + /** Read the key from the config table by key + and process the matching record with callback + @param key config key + @param callback callback to process the record + @return DB_SUCCESS or error code */ + dberr_t read_config(const char *key, RecordCallback& callback) noexcept; + + /** Select-for-update CONFIG row by 'key' + @return DB_SUCCESS or error code */ + dberr_t read_config_with_lock( + const char *key, RecordCallback& callback) noexcept; + + /** Read Auxiliary INDEX_[1..6] table rows at (or) after + 'word' with given cursor mode. Callback is invoked for each + row for comparing it with word and process it if there is a match + @return DB_SUCCESS or error code */ + dberr_t read_aux(uint8_t aux_index, const char *word, + page_cur_mode_t mode, RecordCallback& callback) noexcept; + + /** Read all INDEX_[1..6] rows + Callback is invoked for each row for comparing it with word + and process it if it is matching + @return DB_SUCCESS or error code */ + dberr_t read_aux_all(uint8_t aux_index, RecordCallback& callback) noexcept; + + /** Read all rows from given COMMON table + Callback is invoked for processing the record */ + dberr_t read_all_common(const char *tbl_name, + RecordCallback& callback) noexcept; + mem_heap_t* get_heap() const noexcept + { return m_executor->get_heap(); } +}; + +/** Callback class for reading common table records +(DELETED, BEING_DELETED, DELETED_CACHE, BEING_DELETED_CACHE) */ +class CommonTableReader : public RecordCallback +{ +private: + std::vector doc_ids; + +public: + CommonTableReader(); + + const std::vector& get_doc_ids() const { return doc_ids; } + void clear() { doc_ids.clear(); } +}; + +/** Callback class for reading FTS config table records */ +class ConfigReader : public RecordCallback +{ +public: + std::vector> config_pairs; + ConfigReader(); +}; + +/** Type alias for FTS record processor function */ +using FTSRecordProcessor= std::function< + bool(const rec_t*, const dict_index_t*, const rec_offs*, void*)>; + +/** Comparison modes for AuxRecordReader */ +enum class AuxCompareMode +{ + /* >= comparison (range scan from word) */ + GREATER_EQUAL, + /* > comparison (exclude exact match) */ + GREATER, + /* LIKE pattern matching (prefix match) */ + LIKE, + /* = comparison (exact match) */ + EQUAL +}; + +/** Callback class for reading FTS auxiliary index table records */ +class AuxRecordReader : public RecordCallback +{ +private: + void* user_arg; + ulint* total_memory; + AuxCompareMode compare_mode; + +private: + /** FTS-specific record comparison logic */ + RecordCompareAction compare_record( + const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index, const rec_offs* offsets) noexcept; + +public: + /** Default word processor for FTS auxiliary table records */ + bool default_word_processor(const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets, void* user_arg); + + /* Constructor with custom processor */ + template + AuxRecordReader(void* user_data, + ProcessorFunc proc_func, + AuxCompareMode mode= AuxCompareMode::GREATER_EQUAL) + : RecordCallback( + [this, proc_func](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> bool + { + return proc_func(rec, index, offsets, this->user_arg); + }, + [this](const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index, const rec_offs* offsets) + -> RecordCompareAction + { + return this->compare_record(search_tuple, rec, index, offsets); + } + ), + user_arg(user_data), total_memory(nullptr), + compare_mode(mode) {} + + /* Different constructor with default word processing */ + AuxRecordReader(void* user_data, ulint* memory_counter, + AuxCompareMode mode= AuxCompareMode::GREATER_EQUAL) + : RecordCallback( + [this](const rec_t* rec, const dict_index_t* index, + const rec_offs* offsets) -> bool + { + return this->default_word_processor(rec, index, offsets, + this->user_arg); + }, + [this](const dtuple_t* search_tuple, const rec_t* rec, + const dict_index_t* index, const rec_offs* offsets) + -> RecordCompareAction + { + return this->compare_record(search_tuple, rec, index, offsets); + } + ), + user_arg(user_data), total_memory(memory_counter), + compare_mode(mode) {} +}; + +#endif /* INNOBASE_FTS0QUERY_H */ diff --git a/storage/innobase/include/fts0fts.h b/storage/innobase/include/fts0fts.h index 80b0ab1fd1abd..4c1a7ffd4d466 100644 --- a/storage/innobase/include/fts0fts.h +++ b/storage/innobase/include/fts0fts.h @@ -784,14 +784,6 @@ fts_tokenize_document_internal( const char* doc, /*!< in: document to tokenize */ int len); /*!< in: document length */ -/*********************************************************************//** -Fetch COUNT(*) from specified table. -@return the number of rows in the table */ -ulint -fts_get_rows_count( -/*===============*/ - fts_table_t* fts_table); /*!< in: fts table to read */ - /*************************************************************//** Get maximum Doc ID in a table if index "FTS_DOC_ID_INDEX" exists @return max Doc ID or 0 if index "FTS_DOC_ID_INDEX" does not exist */ @@ -824,16 +816,17 @@ fts_load_stopword( bool reload); /*!< in: Whether it is during reload of FTS table */ -/****************************************************************//** -Read the rows from the FTS index -@return DB_SUCCESS if OK */ -dberr_t -fts_table_fetch_doc_ids( -/*====================*/ - trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: aux table */ - fts_doc_ids_t* doc_ids); /*!< in: For collecting - doc ids */ + +/** Read the rows from the fulltext index +@param trx transaction +@param table Fulltext table +@param tbl_name table name +@param doc_ids collecting doc ids +@return DB_SUCCESS or error code */ +dberr_t fts_table_fetch_doc_ids(trx_t *trx, dict_table_t *table, + const char *tbl_name, + fts_doc_ids_t *doc_ids) noexcept; + /****************************************************************//** This function brings FTS index in sync when FTS index is first used. There are documents that have not yet sync-ed to auxiliary @@ -935,3 +928,28 @@ fts_update_sync_doc_id(const dict_table_t *table, /** Sync the table during commit phase @param[in] table table to be synced */ void fts_sync_during_ddl(dict_table_t* table); + +/** Tokenize a document. +@param[in,out] doc document to tokenize +@param[out] result tokenization result +@param[in] parser pluggable parser */ +void fts_tokenize_document( + fts_doc_t* doc, + fts_doc_t* result, + st_mysql_ftparser* parser); + +/** Continue to tokenize a document. +@param[in,out] doc document to tokenize +@param[in] add_pos add this position to all tokens from this tokenization +@param[out] result tokenization result +@param[in] parser pluggable parser */ +void fts_tokenize_document_next( + fts_doc_t* doc, + ulint add_pos, + fts_doc_t* result, + st_mysql_ftparser* parser); + +/** Get a character set based on precise type. +@param prtype precise type +@return the corresponding character set */ +CHARSET_INFO* fts_get_charset(ulint prtype); diff --git a/storage/innobase/include/fts0priv.h b/storage/innobase/include/fts0priv.h index 1bb1b2a27e519..66fed5c57dabc 100644 --- a/storage/innobase/include/fts0priv.h +++ b/storage/innobase/include/fts0priv.h @@ -28,10 +28,10 @@ Created 2011/09/02 Sunny Bains #define INNOBASE_FTS0PRIV_H #include "dict0dict.h" -#include "pars0pars.h" #include "que0que.h" #include "que0types.h" #include "fts0types.h" +#include "fts0exec.h" /* The various states of the FTS sub system pertaining to a table with FTS indexes defined on it. */ @@ -112,26 +112,6 @@ component. /** Maximum length of an integer stored in the config table value column. */ #define FTS_MAX_INT_LEN 32 -/******************************************************************//** -Parse an SQL string. %s is replaced with the table's id. -@return query graph */ -que_t* -fts_parse_sql( -/*==========*/ - fts_table_t* fts_table, /*!< in: FTS aux table */ - pars_info_t* info, /*!< in: info struct, or NULL */ - const char* sql) /*!< in: SQL string to evaluate */ - MY_ATTRIBUTE((nonnull(3), malloc, warn_unused_result)); -/******************************************************************//** -Evaluate a parsed SQL statement -@return DB_SUCCESS or error code */ -dberr_t -fts_eval_sql( -/*=========*/ - trx_t* trx, /*!< in: transaction */ - que_t* graph) /*!< in: Parsed statement */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); - /** Construct the name of an internal FTS table for the given table. @param[in] fts_table metadata on fulltext-indexed table @param[out] table_name a name up to MAX_FULL_NAME_LEN @@ -139,77 +119,15 @@ fts_eval_sql( void fts_get_table_name(const fts_table_t* fts_table, char* table_name, bool dict_locked = false) MY_ATTRIBUTE((nonnull)); -/******************************************************************//** -Construct the column specification part of the SQL string for selecting the -indexed FTS columns for the given table. Adds the necessary bound -ids to the given 'info' and returns the SQL string. Examples: - -One indexed column named "text": - "$sel0", - info/ids: sel0 -> "text" - -Two indexed columns named "subject" and "content": - - "$sel0, $sel1", - info/ids: sel0 -> "subject", sel1 -> "content", -@return heap-allocated WHERE string */ -const char* -fts_get_select_columns_str( -/*=======================*/ - dict_index_t* index, /*!< in: FTS index */ - pars_info_t* info, /*!< in/out: parser info */ - mem_heap_t* heap) /*!< in: memory heap */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); - -/** define for fts_doc_fetch_by_doc_id() "option" value, defines whether -we want to get Doc whose ID is equal to or greater or smaller than supplied -ID */ -#define FTS_FETCH_DOC_BY_ID_EQUAL 1 -#define FTS_FETCH_DOC_BY_ID_LARGE 2 -#define FTS_FETCH_DOC_BY_ID_SMALL 3 - -/*************************************************************//** -Fetch document (= a single row's indexed text) with the given -document id. -@return: DB_SUCCESS if fetch is successful, else error */ -dberr_t -fts_doc_fetch_by_doc_id( -/*====================*/ - fts_get_doc_t* get_doc, /*!< in: state */ - doc_id_t doc_id, /*!< in: id of document to fetch */ - dict_index_t* index_to_use, /*!< in: caller supplied FTS index, - or NULL */ - ulint option, /*!< in: search option, if it is - greater than doc_id or equal */ - fts_sql_callback - callback, /*!< in: callback to read - records */ - void* arg) /*!< in: callback arg */ - MY_ATTRIBUTE((nonnull(6))); - -/*******************************************************************//** -Callback function for fetch that stores the text of an FTS document, -converting each column to UTF-16. -@return always FALSE */ -ibool -fts_query_expansion_fetch_doc( -/*==========================*/ - void* row, /*!< in: sel_node_t* */ - void* user_arg) /*!< in: fts_doc_t* */ - MY_ATTRIBUTE((nonnull)); -/******************************************************************** -Write out a single word's data as new entry/entries in the INDEX table. -@return DB_SUCCESS if all OK. */ -dberr_t -fts_write_node( -/*===========*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: query graph */ - fts_table_t* fts_table, /*!< in: the FTS aux index */ - fts_string_t* word, /*!< in: word in UTF-8 */ - fts_node_t* node) /*!< in: node columns */ - MY_ATTRIBUTE((nonnull, warn_unused_result)); +/** Write out a single word's data as new entry/entries in the INDEX table. +@param executor FTS Query Executor +@param selected auxiliary index number +@param aux_data auxiliary table data +@return DB_SUCCESS if all OK or error code */ +dberr_t fts_write_node(FTSQueryExecutor *executor, uint8_t selected, + const fts_aux_data_t *aux_data) + MY_ATTRIBUTE((nonnull, warn_unused_result)); /** Check if a fts token is a stopword or less than fts_min_token_size or greater than fts_max_token_size. @@ -258,35 +176,36 @@ fts_word_free( /*==========*/ fts_word_t* word) /*!< in: instance to free.*/ MY_ATTRIBUTE((nonnull)); -/******************************************************************//** -Read the rows from the FTS inde -@return DB_SUCCESS or error code */ -dberr_t -fts_index_fetch_nodes( -/*==================*/ - trx_t* trx, /*!< in: transaction */ - que_t** graph, /*!< in: prepared statement */ - fts_table_t* fts_table, /*!< in: FTS aux table */ - const fts_string_t* - word, /*!< in: the word to fetch */ - fts_fetch_t* fetch) /*!< in: fetch callback.*/ - MY_ATTRIBUTE((nonnull)); + +/** Read the rows from FTS index +@param trx transaction +@param index fulltext index +@param word word to fetch +@param user_arg user argument +@param processor custom processor to filter the word from record +@param compare_mode comparison mode for record matching +@return error code or DB_SUCCESS */ +dberr_t fts_index_fetch_nodes(trx_t *trx, dict_index_t *index, + const fts_string_t *word, + void *user_arg, + FTSRecordProcessor processor, + AuxCompareMode compare_mode) + MY_ATTRIBUTE((nonnull)); + #define fts_sql_commit(trx) trx_commit_for_mysql(trx) #define fts_sql_rollback(trx) (trx)->rollback() -/******************************************************************//** -Get value from config table. The caller must ensure that enough -space is allocated for value to hold the column contents + +/** Get value from the config table. The caller must ensure that enough +space is allocated for value to hold the column contents. +@param trx transaction +@param table Indexed fts table +@param name name of the key +@param value value of the key @return DB_SUCCESS or error code */ -dberr_t -fts_config_get_value( -/*=================*/ - trx_t* trx, /* transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ - const char* name, /*!< in: get config value for - this parameter name */ - fts_string_t* value) /*!< out: value read from - config table */ - MY_ATTRIBUTE((nonnull)); +dberr_t fts_config_get_value(trx_t *trx, const dict_table_t *table, + const char *name, fts_string_t *value) + MY_ATTRIBUTE((nonnull)); + /******************************************************************//** Get value specific to an FTS index from the config table. The caller must ensure that enough space is allocated for value to hold the @@ -309,7 +228,7 @@ dberr_t fts_config_set_value( /*=================*/ trx_t* trx, /*!< transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ + const dict_table_t* table, /*!< in: the indexed FTS table */ const char* name, /*!< in: get config value for this parameter name */ const fts_string_t* @@ -322,7 +241,7 @@ dberr_t fts_config_set_ulint( /*=================*/ trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ + const dict_table_t* table, /*!< in: the indexed FTS table */ const char* name, /*!< in: param name */ ulint int_value) /*!< in: value */ MY_ATTRIBUTE((nonnull, warn_unused_result)); @@ -372,7 +291,7 @@ dberr_t fts_config_get_ulint( /*=================*/ trx_t* trx, /*!< in: transaction */ - fts_table_t* fts_table, /*!< in: the indexed FTS table */ + const dict_table_t* table, /*!< in: the indexed FTS table */ const char* name, /*!< in: param name */ ulint* int_value) /*!< out: value */ MY_ATTRIBUTE((nonnull)); diff --git a/storage/innobase/include/fts0types.h b/storage/innobase/include/fts0types.h index 8db88bfc1765f..53eec73629f61 100644 --- a/storage/innobase/include/fts0types.h +++ b/storage/innobase/include/fts0types.h @@ -315,7 +315,7 @@ fts_get_suffix( @param[in] len string length in bytes @return the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index( const CHARSET_INFO* cs, const byte* str, diff --git a/storage/innobase/include/fts0types.inl b/storage/innobase/include/fts0types.inl index c4e7bfed7ffde..d1081bfd0a44d 100644 --- a/storage/innobase/include/fts0types.inl +++ b/storage/innobase/include/fts0types.inl @@ -102,13 +102,13 @@ inline bool fts_is_charset_cjk(const CHARSET_INFO* cs) @param[in] len string length @retval the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index_by_range( const CHARSET_INFO* cs, const byte* str, ulint len) { - ulint selected = 0; + uint8_t selected = 0; ulint value = innobase_strnxfrm(cs, str, len); while (fts_index_selector[selected].value != 0) { @@ -136,7 +136,7 @@ fts_select_index_by_range( @param[in] len string length @retval the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index_by_hash( const CHARSET_INFO* cs, const byte* str, @@ -163,7 +163,7 @@ fts_select_index_by_hash( /* Get collation hash code */ my_ci_hash_sort(cs, str, char_len, &nr1, &nr2); - return(nr1 % FTS_NUM_AUX_INDEX); + return static_cast(nr1 % FTS_NUM_AUX_INDEX); } /** Select the FTS auxiliary index for the given character. @@ -172,21 +172,17 @@ fts_select_index_by_hash( @param[in] len string length in bytes @retval the index to use for the string */ UNIV_INLINE -ulint +uint8_t fts_select_index( const CHARSET_INFO* cs, const byte* str, ulint len) { - ulint selected; - if (fts_is_charset_cjk(cs)) { - selected = fts_select_index_by_hash(cs, str, len); - } else { - selected = fts_select_index_by_range(cs, str, len); + return fts_select_index_by_hash(cs, str, len); } - return(selected); + return fts_select_index_by_range(cs, str, len); } /******************************************************************//** diff --git a/storage/innobase/include/row0query.h b/storage/innobase/include/row0query.h new file mode 100644 index 0000000000000..bb35a1921b764 --- /dev/null +++ b/storage/innobase/include/row0query.h @@ -0,0 +1,258 @@ +/***************************************************************************** + +Copyright (c) 2025, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file include/row0query.h +General Query Executor + +Created 2025/10/30 +*******************************************************/ + +#ifndef INNOBASE_ROW0QUERY_H +#define INNOBASE_ROW0QUERY_H + +#include "btr0pcur.h" +#include +#include "dict0types.h" +#include "data0types.h" +#include "db0err.h" +#include "lock0types.h" +#include "rem0rec.h" + +/** Comparator action for deciding how to treat a record */ +enum class RecordCompareAction +{ + /** Do not process this record, continue traversal */ + SKIP, + /** Process this record via process_record */ + PROCESS, + /** Stop traversal immediately */ + STOP +}; + +using RecordProcessor= std::function; + +using RecordComparator= std::function< + RecordCompareAction(const dtuple_t*, const rec_t*, + const dict_index_t*, const rec_offs*)>; + +/** Record processing callback interface using std::function. +Can be used by FTS, stats infrastructure, and other components +that need to process database records with custom logic. */ +class RecordCallback +{ +public: + /** Constructor with processor function and optional comparator + @param[in] processor Function to process each record + @param[in] comparator Optional function to filter records (default: accept all) */ + RecordCallback( + RecordProcessor processor, + RecordComparator comparator= [](const dtuple_t*, const rec_t*, + const dict_index_t*, const rec_offs*) + { return RecordCompareAction::PROCESS; }) + : process_record(std::move(processor)), + compare_record(std::move(comparator)) {} + + virtual ~RecordCallback() = default; + + /** Called for each matching record */ + RecordProcessor process_record; + + /** Comparison function for custom filtering */ + RecordComparator compare_record; +}; + +/** General-purpose MVCC-aware record traversal and basic +DML executor. Provides a thin abstraction over B-tree cursors for +reading and mutating records with consistent-read (MVCC) handling, +and callback API. +- Open and iterate clustered/secondary indexes with page cursors. +- Build consistent-read versions when needed via transaction +read views. +- Filter and process records using RecordCallback: + - compare_record: decide SKIP/PROCESS/STOP for each record + - process_record: handle visible records; return whether to continue +- Basic DML helpers (insert/delete/replace) and table locking. */ +class QueryExecutor +{ +private: + trx_t *m_trx; + que_thr_t *m_thr; + btr_pcur_t m_pcur; + mtr_t m_mtr; + mem_heap_t *m_heap; + bool m_mtr_active; + + /** Lookup clustered index record from secondary index record + @param table table containing the indexes + @param sec_index secondary index + @param clust_index clustered index + @param sec_rec secondary index record + @param callback callback to process the clustered record + @param match_count counter for processed records + @return true if should continue processing, false to break */ + bool lookup_clustered_record(dict_table_t *table, + dict_index_t *sec_index, + dict_index_t *clust_index, + const rec_t *sec_rec, + RecordCallback& callback, + ulint& match_count) noexcept; + + /** Process a record with MVCC visibility checking and + version building + @param table table containing the record + @param index index of the record + @param rec record to process + @param offsets record offsets + @param callback callback to process the record + @param mtr mini-transaction for version building + @param match_count counter for processed records + @return true if should continue processing, false to break */ + bool process_record_with_mvcc(dict_table_t *table, + dict_index_t *index, + const rec_t *rec, + rec_offs *offsets, + RecordCallback& callback, + mtr_t *mtr, + ulint& match_count) noexcept; +public: + QueryExecutor(trx_t *trx); + ~QueryExecutor(); + + /** Insert a record in clustered index of the table + @param table table to be inserted + @param tuple tuple to be inserted + @return DB_SUCCESS on success, error code on failure */ + dberr_t insert_record(dict_table_t *table, dtuple_t *tuple) noexcept; + + /** Delete a record from the clustered index of the table + @param table table to be inserted + @param tuple tuple to be inserted + @return DB_SUCCESS on success, error code on failure */ + dberr_t delete_record(dict_table_t *table, dtuple_t *tuple) noexcept; + + /** Delete all records from the clustered index of the table + @param table table be be deleted + @return DB_SUCCESS on success, error code on failure */ + dberr_t delete_all(dict_table_t *table) noexcept; + + /** Acquire and lock a single clustered record for update + Performs a keyed lookup on the clustered index, validates MVCC visibility, + and acquires an X lock on the matching record. + @param[in] table Table containing the record + @param[in] search_tuple Exact key for clustered index lookup + @param[in] callback Optional record callback + @return DB_SUCCESS on successful lock + DB_RECORD_NOT_FOUND if no visible matching record + DB_LOCK_WAIT if waiting was required + error code on failure */ + dberr_t select_for_update(dict_table_t *table, dtuple_t *search_tuple, + RecordCallback *callback= nullptr) noexcept; + + /** Update the currently selected clustered record within an active mtr. + Attempts in-place update; falls back to optimistic/pessimistic update if needed, + including external field storage when required. + select_for_update() has positioned and locked m_pcur on the target row. + @param[in] table target table + @param[in] update update descriptor (fields, new values) + @return DB_SUCCESS on success + DB_OVERFLOW/DB_UNDERFLOW during size-changing paths + error_code on failures */ + dberr_t update_record(dict_table_t *table, const upd_t *update) noexcept; + + /** Commit the current modification mtr. Commits any pending page + changes from an update flow and clears the active mtr state. + @returns DB_SUCCESS if an mtr was active and is now committed, + DB_ERROR if no active mtr */ + dberr_t commit_update() noexcept; + + /** Roll back the current modification mtr. + Rollback is implemented as an mtr commit without persisting + an update, because page-level undo is managed by the + higher-level InnoDB machinery (undo logs). + @return DB_SUCCESS if an mtr was active + DB_ERROR if no active mtr */ + dberr_t rollback_update() noexcept; + + /** Try to update a record by key or insert if not found. + Performs a SELECT ... FOR UPDATE using search_tuple; + if found, updates the row; otherwise inserts a new record. + Note: + On update path, commits or rolls back the active mtr as needed. + On insert path, no active mtr remains upon return + @param[in] table target table + @param[in] search_tuple key identifying the target row + @param[in] update update descriptor (applied when found) + @param[in] insert_tuple tuple to insert when not found + @return DB_SUCCESS on successful update or insert + @retval DB_LOCK_WAIT to be retried, + @return error code on failure */ + dberr_t replace_record(dict_table_t *table, dtuple_t *search_tuple, + const upd_t *update, dtuple_t *insert_tuple) noexcept; + + /** Iterate clustered index records and process via callback. + Handles full table scan and index scan for range/select queries + Calls callback.compare_record() to decide SKIP/PROCESS/STOP for + each matching record. On PROCESS, invokes + callback.process_record() on an MVCC-visible version. + @param table table to read + @param tuple optional search key (range/point). nullptr => full scan + @param mode B-tree search mode (e.g., PAGE_CUR_GE) + @param callback record comparator/processor + @return DB_SUCCESS if at least one record was processed + @retval DB_RECORD_NOT_FOUND if no record matched + @return error code on failure */ + dberr_t read(dict_table_t *table, dtuple_t *tuple, page_cur_mode_t mode, + RecordCallback& callback) noexcept; + + /** Read records via a secondary index and process corresponding + clustered rows. Performs a range or point scan on the given secondary index, + filters secondary records with callback.compare_record(), then looks up + the matching clustered record and invokes callback.process_record() + on a MVCC-visible version. + + @param table Table to read + @param sec_index Secondary index used for traversal + @param search_tuple search key or nullptr for full scan + @param mode Cursor search mode + @param callback RecordCallback with comparator+processor + @return DB_SUCCESS on success + DB_RECORD_NOT_FOUND if no matching record was processed + error code on failure */ + dberr_t read_by_index(dict_table_t *table, dict_index_t *sec_index, + dtuple_t *search_tuple, page_cur_mode_t mode, + RecordCallback& callback) noexcept; + + /** Acquire a table lock in the given mode for transaction. + @param table table to lock + @param mode lock mode + @return DB_SUCCESS, DB_LOCK_WAIT or error code */ + dberr_t lock_table(dict_table_t *table, lock_mode mode) noexcept; + + /** Handle a lock wait for the current transaction and thread context. + @param err the lock-related error to handle (e.g., DB_LOCK_WAIT) + @param table_lock true if the wait originated from table lock, else row lock + @return DB_SUCCESS if the wait completed successfully and lock was granted + @retval DB_LOCK_WAIT_TIMEOUT if timed out */ + dberr_t handle_wait(dberr_t err, bool table_lock) noexcept; + mem_heap_t *get_heap() const { return m_heap; } + trx_t *get_trx() const { return m_trx; } +}; + +#endif /* INNOBASE_ROW0QUERY_H */ diff --git a/storage/innobase/include/ut0new.h b/storage/innobase/include/ut0new.h index 398dd0dcc9ecd..ae2a6ca871ae0 100644 --- a/storage/innobase/include/ut0new.h +++ b/storage/innobase/include/ut0new.h @@ -852,6 +852,7 @@ constexpr const char* const auto_event_names[] = "fts0ast", "fts0blex", "fts0config", + "fts0exec", "fts0file", "fts0fts", "fts0opt", diff --git a/storage/innobase/row/row0query.cc b/storage/innobase/row/row0query.cc new file mode 100644 index 0000000000000..826c9325ac6b7 --- /dev/null +++ b/storage/innobase/row/row0query.cc @@ -0,0 +1,634 @@ +/***************************************************************************** + +Copyright (c) 2025, MariaDB Corporation. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Fifth Floor, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file row/row0query.cc +General Query Executor + +Created 2025/10/30 +*******************************************************/ + +#include "row0query.h" +#include "pars0pars.h" +#include "dict0dict.h" +#include "row0ins.h" +#include "row0upd.h" +#include "row0row.h" +#include "row0vers.h" +#include "mem0mem.h" +#include "que0que.h" +#include "lock0lock.h" +#include "rem0rec.h" +#include "btr0pcur.h" +#include "btr0cur.h" + +QueryExecutor::QueryExecutor(trx_t *trx) + : m_trx(trx), m_mtr(trx), m_mtr_active(false) +{ + m_heap= mem_heap_create(256); + m_thr= pars_complete_graph_for_exec(nullptr, m_trx, m_heap, nullptr); + btr_pcur_init(&m_pcur); +} + +QueryExecutor::~QueryExecutor() +{ + btr_pcur_close(&m_pcur); + if (m_heap) mem_heap_free(m_heap); +} + +dberr_t QueryExecutor::insert_record(dict_table_t *table, + dtuple_t *tuple) noexcept +{ + dict_index_t* index= dict_table_get_first_index(table); + return row_ins_clust_index_entry(index, tuple, m_thr, 0); +} + +dberr_t QueryExecutor::lock_table(dict_table_t *table, lock_mode mode) noexcept +{ + trx_start_if_not_started(m_trx, true); + return ::lock_table(table, nullptr, mode, m_thr); +} + +dberr_t QueryExecutor::handle_wait(dberr_t err, bool table_lock) noexcept +{ + m_trx->error_state= err; + if (table_lock) m_thr->lock_state= QUE_THR_LOCK_TABLE; + else m_thr->lock_state= QUE_THR_LOCK_ROW; + if (m_trx->lock.wait_thr) + { + dberr_t wait_err= lock_wait(m_thr); + if (wait_err == DB_LOCK_WAIT_TIMEOUT) err= wait_err; + if (wait_err == DB_SUCCESS) + { + m_thr->lock_state= QUE_THR_LOCK_NOLOCK; + return DB_SUCCESS; + } + } + return err; +} + +dberr_t QueryExecutor::delete_record(dict_table_t *table, + dtuple_t *tuple) noexcept +{ + dict_index_t *index= dict_table_get_first_index(table); + btr_pcur_t pcur; + mtr_t mtr(m_trx); + ulint deleted_count= 0; + + mtr.start(); + mtr.set_named_space(table->space); + + pcur.btr_cur.page_cur.index= index; + dberr_t err= btr_pcur_open(tuple, PAGE_CUR_GE, BTR_MODIFY_LEAF, + &pcur, &mtr); + if (err != DB_SUCCESS) + { + mtr.commit(); + return err; + } + while (!btr_pcur_is_after_last_on_page(&pcur) && + !btr_pcur_is_after_last_in_tree(&pcur)) + { + rec_t* rec= btr_pcur_get_rec(&pcur); + if (!rec) break; + + if (rec_get_deleted_flag(rec, dict_table_is_comp(table))) + { + if (!btr_pcur_move_to_next(&pcur, &mtr)) break; + continue; + } + + rec_offs* offsets= rec_get_offsets(rec, index, nullptr, + index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + + uint16_t matched_fields= 0; + int cmp= cmp_dtuple_rec_with_match(tuple, rec, index, + offsets, &matched_fields); + if (cmp != 0) break; + err= lock_clust_rec_read_check_and_lock( + 0, btr_pcur_get_block(&pcur), rec, index, offsets, LOCK_X, + LOCK_REC_NOT_GAP, m_thr); + if (err == DB_LOCK_WAIT) + { + mtr.commit(); + err= handle_wait(err, false); + if (err != DB_SUCCESS) return err; + mtr.start(); + continue; + } + else if (err != DB_SUCCESS && err != DB_SUCCESS_LOCKED_REC) + { + mtr.commit(); + return err; + } + + err= btr_cur_del_mark_set_clust_rec(btr_pcur_get_block(&pcur), + rec, index, offsets, m_thr, + nullptr, &mtr); + if (err != DB_SUCCESS) break; + deleted_count++; + if (!btr_pcur_move_to_next(&pcur, &mtr)) break; + } + mtr.commit(); + return (deleted_count > 0) ? DB_SUCCESS : DB_RECORD_NOT_FOUND; +} + +dberr_t QueryExecutor::delete_all(dict_table_t *table) noexcept +{ + dict_index_t *index= dict_table_get_first_index(table); + btr_pcur_t pcur; + mtr_t mtr(m_trx); + mtr.start(); + mtr.set_named_space(table->space); + + dberr_t err= pcur.open_leaf(true, index, BTR_MODIFY_LEAF, &mtr); + if (err == DB_SUCCESS) btr_pcur_move_to_next(&pcur, &mtr); + if (err != DB_SUCCESS) + { + mtr.commit(); + return err; + } + + while (!btr_pcur_is_after_last_on_page(&pcur) && + !btr_pcur_is_after_last_in_tree(&pcur)) + { + rec_t* rec= btr_pcur_get_rec(&pcur); + if (!rec) break; + if (rec_get_deleted_flag(rec, dict_table_is_comp(table))) + { + if (!btr_pcur_move_to_next(&pcur, &mtr)) break; + continue; + } + + if (rec_get_info_bits( + rec, dict_table_is_comp(table)) & REC_INFO_MIN_REC_FLAG) + { + if (!btr_pcur_move_to_next(&pcur, &mtr)) break; + continue; + } + rec_offs* offsets= rec_get_offsets(rec, index, nullptr, + index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + err= lock_clust_rec_read_check_and_lock( + 0, btr_pcur_get_block(&pcur), rec, index, offsets, LOCK_X, + LOCK_REC_NOT_GAP, m_thr); + + if (err == DB_LOCK_WAIT) + { + mtr.commit(); + err= handle_wait(err, false); + if (err != DB_SUCCESS) return err; + mtr.start(); + continue; + } + else if (err != DB_SUCCESS && err != DB_SUCCESS_LOCKED_REC) + { + mtr.commit(); + return err; + } + + err= btr_cur_del_mark_set_clust_rec(btr_pcur_get_block(&pcur), + const_cast(rec), index, + offsets, m_thr, nullptr, &mtr); + if (err || !btr_pcur_move_to_next(&pcur, &mtr)) break; + } + + mtr.commit(); + return err; +} + +dberr_t QueryExecutor::select_for_update(dict_table_t *table, + dtuple_t *search_tuple, + RecordCallback *callback) noexcept +{ + dict_index_t *index= dict_table_get_first_index(table); + if (m_mtr_active) + { + m_mtr.commit(); + m_mtr_active= false; + } + + m_mtr.start(); + m_mtr.set_named_space(table->space); + m_mtr_active= true; + + if (m_trx && !m_trx->read_view.is_open()) + { + trx_start_if_not_started(m_trx, false); + m_trx->read_view.open(m_trx); + } + m_pcur.btr_cur.page_cur.index= index; + dberr_t err= btr_pcur_open(search_tuple, PAGE_CUR_GE, BTR_MODIFY_LEAF, + &m_pcur, &m_mtr); + if (err != DB_SUCCESS) + { + m_mtr.commit(); + m_mtr_active= false; + return err; + } + + if (btr_pcur_is_after_last_on_page(&m_pcur) || + btr_pcur_is_after_last_in_tree(&m_pcur)) + { + m_mtr.commit(); + m_mtr_active= false; + return DB_RECORD_NOT_FOUND; + } + rec_t* rec= btr_pcur_get_rec(&m_pcur); + if (!rec) + { + m_mtr.commit(); + m_mtr_active= false; + return DB_RECORD_NOT_FOUND; + } + + rec_offs* offsets= rec_get_offsets(rec, index, nullptr, + index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + + if (m_trx && m_trx->read_view.is_open()) + { + trx_id_t rec_trx_id= row_get_rec_trx_id(rec, index, offsets); + if (rec_trx_id && !m_trx->read_view.changes_visible(rec_trx_id)) + { + m_mtr.commit(); + m_mtr_active= false; + return DB_RECORD_NOT_FOUND; + } + } + uint16_t matched_fields= 0; + int cmp= cmp_dtuple_rec_with_match(search_tuple, rec, index, + offsets, &matched_fields); + if (cmp != 0) + { + m_mtr.commit(); + m_mtr_active= false; + return DB_RECORD_NOT_FOUND; + } + + err= lock_clust_rec_read_check_and_lock( + 0, btr_pcur_get_block(&m_pcur), rec, index, offsets, LOCK_X, + LOCK_REC_NOT_GAP, m_thr); + + if (err == DB_LOCK_WAIT) + { + m_mtr.commit(); + m_mtr_active= false; + err= handle_wait(err, false); + if (err != DB_SUCCESS) return err; + return DB_LOCK_WAIT; + } + else if (err != DB_SUCCESS && err != DB_SUCCESS_LOCKED_REC) + { + m_mtr.commit(); + m_mtr_active= false; + return err; + } + + if (callback) + { + RecordCompareAction action= + callback->compare_record(search_tuple, rec, index, offsets); + if (action == RecordCompareAction::PROCESS) + callback->process_record(rec, index, offsets); + m_mtr.commit(); + m_mtr_active= false; + if (action == RecordCompareAction::SKIP) + return DB_RECORD_NOT_FOUND; + } + return DB_SUCCESS; +} + +dberr_t QueryExecutor::update_record(dict_table_t *table, + const upd_t *update) noexcept +{ + if (!m_mtr_active) return DB_ERROR; + dict_index_t *index= dict_table_get_first_index(table); + rec_t *rec= btr_pcur_get_rec(&m_pcur); + if (!rec) return DB_RECORD_NOT_FOUND; + mtr_x_lock_index(index, &m_mtr); + rec_offs *offsets= rec_get_offsets(rec, index, nullptr, + index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + + dberr_t err= DB_SUCCESS; + ulint cmpl_info= UPD_NODE_NO_ORD_CHANGE | UPD_NODE_NO_SIZE_CHANGE; + for (ulint i = 0; i < update->n_fields; i++) + { + const upd_field_t *upd_field= &update->fields[i]; + ulint field_no= upd_field->field_no; + if (field_no < rec_offs_n_fields(offsets)) + { + ulint old_len= rec_offs_nth_size(offsets, field_no); + ulint new_len= upd_field->new_val.len; + if (new_len != UNIV_SQL_NULL && new_len != old_len) + { + cmpl_info &= ~UPD_NODE_NO_SIZE_CHANGE; + err= DB_OVERFLOW; + break; + } + } + } + + if (cmpl_info & UPD_NODE_NO_SIZE_CHANGE) + err= btr_cur_update_in_place(BTR_NO_LOCKING_FLAG, + btr_pcur_get_btr_cur(&m_pcur), + offsets, const_cast(update), 0, + m_thr, m_trx->id, &m_mtr); + if (err == DB_OVERFLOW) + { + big_rec_t *big_rec= nullptr; + err= btr_cur_optimistic_update(BTR_NO_LOCKING_FLAG, + btr_pcur_get_btr_cur(&m_pcur), + &offsets, &m_heap, + const_cast(update), + cmpl_info, m_thr, m_trx->id, &m_mtr); + + if (err == DB_OVERFLOW || err == DB_UNDERFLOW) + { + mem_heap_t* offsets_heap= nullptr; + err= btr_cur_pessimistic_update(BTR_NO_LOCKING_FLAG, + btr_pcur_get_btr_cur(&m_pcur), + &offsets, &offsets_heap, m_heap, + &big_rec, const_cast(update), + cmpl_info, m_thr, m_trx->id, &m_mtr); + + if (err == DB_SUCCESS && big_rec) + { + err= btr_store_big_rec_extern_fields(&m_pcur, offsets, big_rec, &m_mtr, + BTR_STORE_UPDATE); + dtuple_big_rec_free(big_rec); + } + if (offsets_heap) mem_heap_free(offsets_heap); + } + } + return err; +} + +dberr_t QueryExecutor::commit_update() noexcept +{ + if (m_mtr_active) + { + m_mtr.commit(); + m_mtr_active= false; + return DB_SUCCESS; + } + return DB_ERROR; +} + +dberr_t QueryExecutor::rollback_update() noexcept +{ + if (m_mtr_active) + { + m_mtr.commit(); + m_mtr_active= false; + return DB_SUCCESS; + } + return DB_ERROR; +} + +dberr_t QueryExecutor::replace_record( + dict_table_t *table, dtuple_t *search_tuple, + const upd_t *update, dtuple_t *insert_tuple) noexcept +{ +retry_again: + dberr_t err= select_for_update(table, search_tuple); + if (err == DB_SUCCESS) + { + err= update_record(table, update); + if (err == DB_SUCCESS) err= commit_update(); + else rollback_update(); + return err; + } + else if (err == DB_RECORD_NOT_FOUND) + { + if (m_mtr_active) + { + m_mtr.commit(); + m_mtr_active= false; + } + err= insert_record(table, insert_tuple); + return err; + } + else if (err == DB_LOCK_WAIT) + goto retry_again; + return err; +} + +dberr_t QueryExecutor::read(dict_table_t *table, dtuple_t *tuple, + page_cur_mode_t mode, + RecordCallback& callback) noexcept +{ + ut_ad(table); + dict_index_t *index= dict_table_get_first_index(table); + if (!index) return DB_ERROR; + + m_mtr.start(); + if (m_trx && !m_trx->read_view.is_open()) + { + trx_start_if_not_started(m_trx, false); + m_trx->read_view.open(m_trx); + } + m_pcur.btr_cur.page_cur.index= index; + dberr_t err= DB_SUCCESS; + if (tuple) err= btr_pcur_open(tuple, mode, BTR_SEARCH_LEAF, &m_pcur, &m_mtr); + else + { + err= m_pcur.open_leaf(true, index, BTR_SEARCH_LEAF, &m_mtr); + if (err == DB_SUCCESS) btr_pcur_move_to_next(&m_pcur, &m_mtr); + } + if (err != DB_SUCCESS) + { + m_mtr.commit(); + return err; + } + ulint match_count= 0; + while (btr_pcur_is_on_user_rec(&m_pcur)) + { + const rec_t *rec= btr_pcur_get_rec(&m_pcur); + rec_offs* offsets= rec_get_offsets(rec, index, nullptr, + index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + RecordCompareAction action= callback.compare_record( + tuple, rec, index, offsets); + if (action == RecordCompareAction::PROCESS) + { + bool continue_processing= process_record_with_mvcc(table, index, rec, + offsets, callback, + &m_mtr, match_count); + if (!continue_processing) break; + } + else if (action == RecordCompareAction::STOP) + break; + if (!btr_pcur_move_to_next(&m_pcur, &m_mtr)) break; + } + m_mtr.commit(); + return (match_count > 0 || !tuple) ? DB_SUCCESS : DB_RECORD_NOT_FOUND; +} + +dberr_t QueryExecutor::read_by_index(dict_table_t *table, + dict_index_t *sec_index, + dtuple_t *search_tuple, + page_cur_mode_t mode, + RecordCallback& callback) noexcept +{ + ut_ad(table); + ut_ad(sec_index); + ut_ad(sec_index->table == table); + ut_ad(!dict_index_is_clust(sec_index)); + + dict_index_t *clust_index= dict_table_get_first_index(table); + if (!clust_index) return DB_ERROR; + + m_mtr.start(); + if (m_trx && !m_trx->read_view.is_open()) + { + trx_start_if_not_started(m_trx, false); + m_trx->read_view.open(m_trx); + } + m_pcur.btr_cur.page_cur.index= sec_index; + + dberr_t err= DB_SUCCESS; + if (search_tuple) + err= btr_pcur_open(search_tuple, mode, BTR_SEARCH_LEAF, &m_pcur, &m_mtr); + else + { + err= m_pcur.open_leaf(true, sec_index, BTR_SEARCH_LEAF, &m_mtr); + if (err == DB_SUCCESS) btr_pcur_move_to_next(&m_pcur, &m_mtr); + } + + if (err != DB_SUCCESS) + { + m_mtr.commit(); + return err; + } + + ulint match_count= 0; + while (btr_pcur_is_on_user_rec(&m_pcur)) + { + const rec_t *sec_rec= btr_pcur_get_rec(&m_pcur); + rec_offs* sec_offsets= rec_get_offsets(sec_rec, sec_index, nullptr, + sec_index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + /* Check if secondary record matches our search criteria */ + RecordCompareAction action= callback.compare_record(search_tuple, sec_rec, + sec_index, sec_offsets); + if (action == RecordCompareAction::PROCESS) + { + /* Lookup clustered record and process it */ + if (!lookup_clustered_record(table, sec_index, clust_index, + sec_rec, callback, match_count)) + break; + } + else if (action == RecordCompareAction::STOP) + break; + if (!btr_pcur_move_to_next(&m_pcur, &m_mtr)) break; + } + m_mtr.commit(); + return (match_count > 0 || !search_tuple) ? DB_SUCCESS : DB_RECORD_NOT_FOUND; +} + +bool QueryExecutor::lookup_clustered_record(dict_table_t *table, + dict_index_t *sec_index, + dict_index_t *clust_index, + const rec_t *sec_rec, + RecordCallback& callback, + ulint& match_count) noexcept +{ + /* Extract primary key from secondary index record */ + dtuple_t *clust_tuple= row_build_row_ref(ROW_COPY_DATA, sec_index, + sec_rec, m_heap); + if (!clust_tuple) return false; + /* Now lookup the complete row using clustered index */ + btr_pcur_t clust_pcur; + clust_pcur.btr_cur.page_cur.index= clust_index; + + mtr_t clust_mtr{m_trx}; + clust_mtr.start(); + bool continue_processing= true; + bool record_processed= false; + dberr_t clust_err= btr_pcur_open(clust_tuple, PAGE_CUR_LE, + BTR_SEARCH_LEAF, &clust_pcur, + &clust_mtr); + if (clust_err == DB_SUCCESS && btr_pcur_is_on_user_rec(&clust_pcur)) + { + const rec_t *clust_rec= btr_pcur_get_rec(&clust_pcur); + rec_offs* clust_offsets= rec_get_offsets(clust_rec, clust_index, + nullptr, + clust_index->n_core_fields, + ULINT_UNDEFINED, &m_heap); + /* Verify this is the exact record we want */ + if (!cmp_dtuple_rec(clust_tuple, clust_rec, clust_index, clust_offsets)) + { + ulint prev_match_count= match_count; + continue_processing= process_record_with_mvcc( + table, clust_index, clust_rec, clust_offsets, callback, &clust_mtr, + match_count); + record_processed= (match_count > prev_match_count); + } + } + clust_mtr.commit(); + return record_processed ? continue_processing : true; +} + +bool QueryExecutor::process_record_with_mvcc(dict_table_t *table, + dict_index_t *index, + const rec_t *rec, + rec_offs *offsets, + RecordCallback& callback, + mtr_t *mtr, + ulint& match_count) noexcept +{ + bool is_deleted= rec_get_deleted_flag(rec, dict_table_is_comp(table)); + rec_t* version_rec= const_cast(rec); + rec_offs* version_offsets= offsets; + mem_heap_t* version_heap= nullptr; + bool should_process_record= false; + bool result= true; + if (m_trx && m_trx->read_view.is_open()) + { + trx_id_t rec_trx_id= row_get_rec_trx_id(rec, index, offsets); + if (rec_trx_id && !m_trx->read_view.changes_visible(rec_trx_id)) + { + version_heap= mem_heap_create(1024); + dberr_t vers_err= row_vers_build_for_consistent_read( + rec, mtr, index, &offsets, &m_trx->read_view, &version_heap, + version_heap, &version_rec, nullptr); + if (vers_err == DB_SUCCESS && version_rec) + { + version_offsets= rec_get_offsets(version_rec, index, nullptr, + index->n_core_fields, + ULINT_UNDEFINED, &version_heap); + is_deleted= rec_get_deleted_flag(version_rec, dict_table_is_comp(table)); + should_process_record= !is_deleted; + } + } + else should_process_record= !is_deleted; + } + else should_process_record= !is_deleted && version_rec; + + if (should_process_record) + { + result= callback.process_record(version_rec, index, version_offsets); + match_count++; + } + + if (version_heap) mem_heap_free(version_heap); + return result; +}