From fc8a889c8a1506e24e44a7fd2084cafaea95e0f5 Mon Sep 17 00:00:00 2001 From: Ilya Eremin Date: Tue, 19 Nov 2024 16:09:32 +0300 Subject: [PATCH 1/6] Add AUTO RELEASE TEMP BLOBID transaction option It makes the transaction release temporary ID of user BLOB just after its materialization. It's useful for massive insertions of records with user-defined BLOBs because it eliminates the memory overhead caused by creating and keeping temporary IDs until the transaction ends. This option is used during the database restore. --- doc/sql.extensions/README.set_transaction.txt | 6 ++++ src/burp/BurpTasks.cpp | 1 + src/burp/restore.epp | 28 +++++++++---------- src/common/ParserTokens.h | 2 ++ src/dsql/StmtNodes.cpp | 3 ++ src/dsql/StmtNodes.h | 2 ++ src/dsql/parse.y | 6 ++++ src/gpre/cmp.cpp | 2 ++ src/gpre/gpre.h | 3 +- src/gpre/hsh.h | 1 + src/gpre/sql.cpp | 6 ++++ src/gpre/words.h | 1 + src/include/firebird/impl/consts_pub.h | 1 + src/include/gen/Firebird.pas | 1 + src/jrd/Monitoring.cpp | 3 ++ src/jrd/blb.cpp | 7 ++++- src/jrd/blb.h | 1 + src/jrd/names.h | 1 + src/jrd/relations.h | 1 + src/jrd/tra.cpp | 4 +++ src/jrd/tra.h | 3 +- 21 files changed, 66 insertions(+), 17 deletions(-) diff --git a/doc/sql.extensions/README.set_transaction.txt b/doc/sql.extensions/README.set_transaction.txt index 48e6463cb23..ebba0c04b49 100644 --- a/doc/sql.extensions/README.set_transaction.txt +++ b/doc/sql.extensions/README.set_transaction.txt @@ -27,6 +27,12 @@ commit fails. This option is mostly used by gfix. LOCK TIMEOUT nonneg_short_integer: it's the time (measured in seconds) that a transaction waits for a lock in a record before giving up and reporting an error. +AUTO RELEASE TEMP BLOBID: makes the transaction release temporary ID of user BLOB +just after its materialization. It's useful for massive insertions of records with +user-defined BLOBs because it eliminates the memory overhead caused by creating and +keeping temporary IDs until the transaction ends. This option is used during the +database restore. + Author: Claudio Valderrama C. diff --git a/src/burp/BurpTasks.cpp b/src/burp/BurpTasks.cpp index 1d236f87757..ca85d6800ca 100644 --- a/src/burp/BurpTasks.cpp +++ b/src/burp/BurpTasks.cpp @@ -898,6 +898,7 @@ void RestoreRelationTask::initItem(BurpGlobals* tdgbl, Item& item) ClumpletWriter tpb(ClumpletReader::Tpb, 128, isc_tpb_version3); tpb.insertTag(isc_tpb_concurrency); tpb.insertTag(isc_tpb_no_auto_undo); + tpb.insertTag(isc_tpb_auto_release_temp_blobid); item.m_tra = item.m_att->startTransaction(&status, tpb.getBufferLength(), tpb.getBuffer()); diff --git a/src/burp/restore.epp b/src/burp/restore.epp index 1bc95c3df0e..e7f69423517 100644 --- a/src/burp/restore.epp +++ b/src/burp/restore.epp @@ -421,7 +421,7 @@ int RESTORE_restore (const TEXT* file_name, const TEXT* database_name) // Block added to verbose index creation by Toni Martir // Always try to activate deferred indices - it helps for some broken backups, // and in normal cases doesn't take much time to look for such indices. AP-2008. - EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO; + EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; @@ -445,7 +445,7 @@ int RESTORE_restore (const TEXT* file_name, const TEXT* database_name) general_on_error (); END_ERROR; - EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO; + EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; @@ -481,7 +481,7 @@ int RESTORE_restore (const TEXT* file_name, const TEXT* database_name) END_ERROR; } - EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO; + EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; @@ -745,7 +745,7 @@ void add_files(BurpGlobals* tdgbl, const char* file_name) END_ERROR; END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; + EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; } @@ -3166,7 +3166,7 @@ static void commit_relation_data(BurpGlobals* tdgbl, burp_rel* relation) } // end of while END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; + EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; } @@ -7777,7 +7777,7 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t general_on_error (); END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; + EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; } @@ -8004,7 +8004,7 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t general_on_error (); END_ERROR; END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; + EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; } @@ -8041,7 +8041,7 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t general_on_error (); END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; + EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; @@ -9085,7 +9085,7 @@ bool get_trigger_old (BurpGlobals* tdgbl, burp_rel* relation) general_on_error (); END_ERROR; END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; + EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; } @@ -9441,7 +9441,7 @@ bool get_trigger(BurpGlobals* tdgbl) general_on_error (); END_ERROR; END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; + EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; } @@ -9537,7 +9537,7 @@ bool get_trigger_message(BurpGlobals* tdgbl) general_on_error (); END_ERROR; END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; + EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; } @@ -10391,7 +10391,7 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file create_database(tdgbl, provider, database_name); - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; + EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; @@ -10751,7 +10751,7 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file ON_ERROR general_on_error (); END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; + EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; flag = false; @@ -10875,7 +10875,7 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file ON_ERROR general_on_error (); END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO; + EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; } diff --git a/src/common/ParserTokens.h b/src/common/ParserTokens.h index ad2b6827228..c345f98b7c2 100644 --- a/src/common/ParserTokens.h +++ b/src/common/ParserTokens.h @@ -97,6 +97,7 @@ PARSER_TOKEN(TOK_BINARY, "BINARY", false) PARSER_TOKEN(TOK_BIND, "BIND", true) PARSER_TOKEN(TOK_BIT_LENGTH, "BIT_LENGTH", false) PARSER_TOKEN(TOK_BLOB, "BLOB", false) +PARSER_TOKEN(TOK_BLOBID, "BLOBID", true) PARSER_TOKEN(TOK_BLOB_APPEND, "BLOB_APPEND", true) PARSER_TOKEN(TOK_BLOCK, "BLOCK", true) PARSER_TOKEN(TOK_BODY, "BODY", true) @@ -492,6 +493,7 @@ PARSER_TOKEN(TOK_TAN, "TAN", true) PARSER_TOKEN(TOK_TANH, "TANH", true) PARSER_TOKEN(TOK_TARGET, "TARGET", true) PARSER_TOKEN(TOK_TEMPORARY, "TEMPORARY", true) +PARSER_TOKEN(TOK_TEMP, "TEMP", true) PARSER_TOKEN(TOK_THEN, "THEN", false) PARSER_TOKEN(TOK_TIES, "TIES", true) PARSER_TOKEN(TOK_TIME, "TIME", false) diff --git a/src/dsql/StmtNodes.cpp b/src/dsql/StmtNodes.cpp index 50b861b73e4..3252da179b1 100644 --- a/src/dsql/StmtNodes.cpp +++ b/src/dsql/StmtNodes.cpp @@ -9720,6 +9720,9 @@ SetTransactionNode* SetTransactionNode::dsqlPass(DsqlCompilerScratch* dsqlScratc if (autoCommit.isAssigned()) dsqlScratch->appendUChar(isc_tpb_autocommit); + if (autoReleaseTempBlobID.isAssigned()) + dsqlScratch->appendUChar(isc_tpb_auto_release_temp_blobid); + if (lockTimeout.has_value()) { dsqlScratch->appendUChar(isc_tpb_lock_timeout); diff --git a/src/dsql/StmtNodes.h b/src/dsql/StmtNodes.h index 265b13dd6e6..c6cede470d9 100644 --- a/src/dsql/StmtNodes.h +++ b/src/dsql/StmtNodes.h @@ -1607,6 +1607,7 @@ class SetTransactionNode : public TransactionNode NODE_PRINT(printer, ignoreLimbo); NODE_PRINT(printer, restartRequests); NODE_PRINT(printer, autoCommit); + NODE_PRINT(printer, autoReleaseTempBlobID); NODE_PRINT(printer, lockTimeout); //// FIXME-PRINT: NODE_PRINT(printer, reserveList); NODE_PRINT(printer, tpb); @@ -1633,6 +1634,7 @@ class SetTransactionNode : public TransactionNode Firebird::TriState ignoreLimbo; Firebird::TriState restartRequests; Firebird::TriState autoCommit; + Firebird::TriState autoReleaseTempBlobID; }; diff --git a/src/dsql/parse.y b/src/dsql/parse.y index 6e2c5a31fd8..c2012b3dd58 100644 --- a/src/dsql/parse.y +++ b/src/dsql/parse.y @@ -163,6 +163,7 @@ using namespace Firebird; %token BEGIN %token BETWEEN %token BLOB +%token BLOBID %token BY %token CAST %token CHARACTER @@ -308,6 +309,7 @@ using namespace Firebird; %token SUSPEND %token SUM %token TABLE +%token TEMP %token THEN %token TO %token TRANSACTION @@ -5938,6 +5940,8 @@ tran_option($setTransactionNode) { setClause($setTransactionNode->restartRequests, "RESTART REQUESTS", true); } | AUTO COMMIT { setClause($setTransactionNode->autoCommit, "AUTO COMMIT", true); } + | AUTO RELEASE TEMP BLOBID + { setClause($setTransactionNode->autoReleaseTempBlobID, "AUTO RELEASE TEMP BLOBID", true); } // timeout | LOCK TIMEOUT nonneg_short_integer { setClause($setTransactionNode->lockTimeout, "LOCK TIMEOUT", (USHORT) $3); } @@ -9749,10 +9753,12 @@ non_reserved_word // added in FB 4.0.2 | BLOB_APPEND // added in FB 5.0 + | BLOBID | LOCKED | OPTIMIZE | QUARTER | TARGET + | TEMP | TIMEZONE_NAME | UNICODE_CHAR | UNICODE_VAL diff --git a/src/gpre/cmp.cpp b/src/gpre/cmp.cpp index e8bca853c71..2d2ddc77300 100644 --- a/src/gpre/cmp.cpp +++ b/src/gpre/cmp.cpp @@ -408,6 +408,8 @@ void CMP_t_start( gpre_tra* trans) *text++ = isc_tpb_autocommit; if (trans->tra_flags & TRA_no_auto_undo) *text++ = isc_tpb_no_auto_undo; + if (trans->tra_flags & TRA_auto_release_temp_blobid) + *text++ = isc_tpb_auto_release_temp_blobid; *text = 0; const USHORT tpb_len = text - tpb_buffer; diff --git a/src/gpre/gpre.h b/src/gpre/gpre.h index 9d7832eea64..9702f557264 100644 --- a/src/gpre/gpre.h +++ b/src/gpre/gpre.h @@ -1414,7 +1414,8 @@ enum tra_flags_vals { TRA_read_committed = 32, TRA_autocommit = 64, TRA_rec_version = 128, - TRA_no_auto_undo = 256 + TRA_no_auto_undo = 256, + TRA_auto_release_temp_blobid = 512 }; const int MAX_TRA_OPTIONS = 8; diff --git a/src/gpre/hsh.h b/src/gpre/hsh.h index f7facbb5ac8..540f3e5b87c 100644 --- a/src/gpre/hsh.h +++ b/src/gpre/hsh.h @@ -306,6 +306,7 @@ {"PROCEDURE", KW_PROCEDURE}, {"PROTECTED", KW_PROTECTED}, {"PUBLIC", KW_PUBLIC}, + {"AUTO_RELEASE_TEMP_BLOBID", KW_AUTO_RELEASE_TEMP_BLOBID}, {"PUT_SEGMENT", KW_PUT_SEGMENT}, {"PUT_SLICE", KW_PUT_SLICE}, {"QUADWORD", KW_QUAD}, diff --git a/src/gpre/sql.cpp b/src/gpre/sql.cpp index 5ae99b8830a..5f73dc7eba1 100644 --- a/src/gpre/sql.cpp +++ b/src/gpre/sql.cpp @@ -4424,6 +4424,12 @@ static act* act_set_transaction() continue; } + if (MSC_match(KW_AUTO_RELEASE_TEMP_BLOBID)) + { + trans->tra_flags |= TRA_auto_release_temp_blobid; + continue; + } + break; } diff --git a/src/gpre/words.h b/src/gpre/words.h index afd856f54cc..ef04277eac1 100644 --- a/src/gpre/words.h +++ b/src/gpre/words.h @@ -270,6 +270,7 @@ enum kwwords_t { KW_PRIVILEGES, KW_PROTECTED, KW_PUBLIC, + KW_AUTO_RELEASE_TEMP_BLOBID, KW_QUAD, KW_RAW_PARTITIONS, // ??? KW_READ, diff --git a/src/include/firebird/impl/consts_pub.h b/src/include/firebird/impl/consts_pub.h index 645db9d4864..d7702aac71d 100644 --- a/src/include/firebird/impl/consts_pub.h +++ b/src/include/firebird/impl/consts_pub.h @@ -268,6 +268,7 @@ #define isc_tpb_lock_timeout 21 #define isc_tpb_read_consistency 22 #define isc_tpb_at_snapshot_number 23 +#define isc_tpb_auto_release_temp_blobid 24 /************************/ diff --git a/src/include/gen/Firebird.pas b/src/include/gen/Firebird.pas index 9c1be4d79af..e446fc37a00 100644 --- a/src/include/gen/Firebird.pas +++ b/src/include/gen/Firebird.pas @@ -4111,6 +4111,7 @@ IProfilerStatsImpl = class(IProfilerStats) isc_tpb_lock_timeout = byte(21); isc_tpb_read_consistency = byte(22); isc_tpb_at_snapshot_number = byte(23); + isc_tpb_auto_release_temp_blobid = byte(24); isc_bpb_version1 = byte(1); isc_bpb_source_type = byte(1); isc_bpb_target_type = byte(2); diff --git a/src/jrd/Monitoring.cpp b/src/jrd/Monitoring.cpp index 032f725407c..704c1168654 100644 --- a/src/jrd/Monitoring.cpp +++ b/src/jrd/Monitoring.cpp @@ -1138,6 +1138,9 @@ void Monitoring::putTransaction(SnapshotData::DumpRecord& record, const jrd_tra* // statistics const int stat_id = fb_utils::genUniqueId(); record.storeGlobalId(f_mon_tra_stat_id, getGlobalId(stat_id)); + // auto release temp blobid flag + temp = (transaction->tra_flags & TRA_auto_release_temp_blobid) ? 1 : 0; + record.storeInteger(f_mon_tra_auto_release_temp_blobid, temp); record.write(); diff --git a/src/jrd/blb.cpp b/src/jrd/blb.cpp index 0cf895dead5..923161dc8b4 100644 --- a/src/jrd/blb.cpp +++ b/src/jrd/blb.cpp @@ -326,6 +326,9 @@ blb* blb::create2(thread_db* tdbb, blb* blob = allocate_blob(tdbb, transaction); + if (userBlob) + blob->blb_flags |= BLB_user; + if (type & isc_bpb_type_stream) blob->blb_flags |= BLB_stream; @@ -1296,7 +1299,9 @@ void blb::move(thread_db* tdbb, dsc* from_desc, dsc* to_desc, array->arr_request = own_request; } - blob->destroy(!materialized_blob); + const bool purgeBlob = !materialized_blob || + ((transaction->tra_flags & TRA_auto_release_temp_blobid) && (blob->blb_flags & BLB_user)); + blob->destroy(purgeBlob); } diff --git a/src/jrd/blb.h b/src/jrd/blb.h index 26176b5e695..cad5e0d836c 100644 --- a/src/jrd/blb.h +++ b/src/jrd/blb.h @@ -181,6 +181,7 @@ const int BLB_seek = 32; // Seek is pending const int BLB_large_scan = 64; // Blob is larger than page buffer cache const int BLB_close_on_read = 128; // Temporary blob is not closed until read const int BLB_bulk = 256; // Blob created by bulk insert operation +const int BLB_user = 512; // User-defined blob /* Blob levels are: diff --git a/src/jrd/names.h b/src/jrd/names.h index ae061ea7f7d..be65d348554 100644 --- a/src/jrd/names.h +++ b/src/jrd/names.h @@ -287,6 +287,7 @@ NAME("MON$ATTACHMENT_ID", nam_mon_att_id) NAME("MON$ATTACHMENT_NAME", nam_mon_att_name) NAME("MON$AUTH_METHOD", nam_mon_auth_method) NAME("MON$AUTO_COMMIT", nam_mon_auto_commit) +NAME("MON$AUTO_RELEASE_TEMP_BLOBID", nam_mon_auto_release_temp_blobid) NAME("MON$AUTO_UNDO", nam_mon_auto_undo) NAME("MON$BACKUP_STATE", nam_mon_backup_state) NAME("MON$BACKVERSION_READS", nam_mon_bkversion_reads) diff --git a/src/jrd/relations.h b/src/jrd/relations.h index b10de39449b..b2a7aaf57f9 100644 --- a/src/jrd/relations.h +++ b/src/jrd/relations.h @@ -550,6 +550,7 @@ RELATION(nam_mon_transactions, rel_mon_transactions, ODS_11_1, rel_virtual) FIELD(f_mon_tra_auto_commit, nam_mon_auto_commit, fld_flag_nullable, 0, ODS_11_1) FIELD(f_mon_tra_auto_undo, nam_mon_auto_undo, fld_flag_nullable, 0, ODS_11_1) FIELD(f_mon_tra_stat_id, nam_mon_stat_id, fld_stat_id, 0, ODS_11_1) + FIELD(f_mon_tra_auto_release_temp_blobid, nam_mon_auto_release_temp_blobid, fld_flag_nullable, 0, ODS_14_0) END_RELATION // Relation 36 (MON$STATEMENTS) diff --git a/src/jrd/tra.cpp b/src/jrd/tra.cpp index 3acd8489761..a8149d79889 100644 --- a/src/jrd/tra.cpp +++ b/src/jrd/tra.cpp @@ -3162,6 +3162,10 @@ static void transaction_options(thread_db* tdbb, transaction->tra_flags |= TRA_no_auto_undo; break; + case isc_tpb_auto_release_temp_blobid: + transaction->tra_flags |= TRA_auto_release_temp_blobid; + break; + case isc_tpb_lock_write: // Cannot set a R/W table reservation if the whole txn is R/O. if (read_only.asBool()) diff --git a/src/jrd/tra.h b/src/jrd/tra.h index dea0202e96c..35a1797b27d 100644 --- a/src/jrd/tra.h +++ b/src/jrd/tra.h @@ -435,10 +435,11 @@ const ULONG TRA_read_consistency = 0x40000L; // ensure read consistency in this const ULONG TRA_ex_restart = 0x80000L; // Exception was raised to restart request const ULONG TRA_replicating = 0x100000L; // transaction is allowed to be replicated const ULONG TRA_no_blob_check = 0x200000L; // disable blob access checking +const ULONG TRA_auto_release_temp_blobid = 0x400000L; // remove temp ids of materialized user blobs from tra_blobs // flags derived from TPB, see also transaction_options() at tra.cpp const ULONG TRA_OPTIONS_MASK = (TRA_degree3 | TRA_readonly | TRA_ignore_limbo | TRA_read_committed | - TRA_autocommit | TRA_rec_version | TRA_read_consistency | TRA_no_auto_undo | TRA_restart_requests); + TRA_autocommit | TRA_rec_version | TRA_read_consistency | TRA_no_auto_undo | TRA_restart_requests | TRA_auto_release_temp_blobid); const int TRA_MASK = 3; //const int TRA_BITS_PER_TRANS = 2; From 7c47747a0b3f9d993ae74ab76c589526d290f47d Mon Sep 17 00:00:00 2001 From: Ilya Eremin Date: Sat, 23 Nov 2024 12:04:21 +0300 Subject: [PATCH 2/6] Place tokens in the correct sections --- src/common/ParserTokens.h | 2 +- src/dsql/parse.y | 4 ++-- src/gpre/hsh.h | 2 +- src/gpre/words.h | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/common/ParserTokens.h b/src/common/ParserTokens.h index c345f98b7c2..a45da1c81aa 100644 --- a/src/common/ParserTokens.h +++ b/src/common/ParserTokens.h @@ -492,8 +492,8 @@ PARSER_TOKEN(TOK_TAGS, "TAGS", true) PARSER_TOKEN(TOK_TAN, "TAN", true) PARSER_TOKEN(TOK_TANH, "TANH", true) PARSER_TOKEN(TOK_TARGET, "TARGET", true) -PARSER_TOKEN(TOK_TEMPORARY, "TEMPORARY", true) PARSER_TOKEN(TOK_TEMP, "TEMP", true) +PARSER_TOKEN(TOK_TEMPORARY, "TEMPORARY", true) PARSER_TOKEN(TOK_THEN, "THEN", false) PARSER_TOKEN(TOK_TIES, "TIES", true) PARSER_TOKEN(TOK_TIME, "TIME", false) diff --git a/src/dsql/parse.y b/src/dsql/parse.y index c2012b3dd58..bcfbd508337 100644 --- a/src/dsql/parse.y +++ b/src/dsql/parse.y @@ -163,7 +163,6 @@ using namespace Firebird; %token BEGIN %token BETWEEN %token BLOB -%token BLOBID %token BY %token CAST %token CHARACTER @@ -309,7 +308,6 @@ using namespace Firebird; %token SUSPEND %token SUM %token TABLE -%token TEMP %token THEN %token TO %token TRANSACTION @@ -345,10 +343,12 @@ using namespace Firebird; %token ACTION %token ADMIN +%token BLOBID %token CASCADE %token FREE_IT // ISC SQL extension %token RESTRICT %token ROLE +%token TEMP // New tokens added v6.0 diff --git a/src/gpre/hsh.h b/src/gpre/hsh.h index 540f3e5b87c..ae86373066d 100644 --- a/src/gpre/hsh.h +++ b/src/gpre/hsh.h @@ -43,6 +43,7 @@ {"AT", KW_AT}, {"AUTO", KW_AUTO}, {"AUTOCOMMIT", KW_AUTOCOMMIT}, + {"AUTO_RELEASE_TEMP_BLOBID", KW_AUTO_RELEASE_TEMP_BLOBID}, {"AVERAGE", KW_AVERAGE}, {"AVG", KW_AVERAGE}, {"\\", KW_BACK_SLASH}, @@ -306,7 +307,6 @@ {"PROCEDURE", KW_PROCEDURE}, {"PROTECTED", KW_PROTECTED}, {"PUBLIC", KW_PUBLIC}, - {"AUTO_RELEASE_TEMP_BLOBID", KW_AUTO_RELEASE_TEMP_BLOBID}, {"PUT_SEGMENT", KW_PUT_SEGMENT}, {"PUT_SLICE", KW_PUT_SLICE}, {"QUADWORD", KW_QUAD}, diff --git a/src/gpre/words.h b/src/gpre/words.h index ef04277eac1..5e89a60657d 100644 --- a/src/gpre/words.h +++ b/src/gpre/words.h @@ -113,6 +113,7 @@ enum kwwords_t { KW_ASTERISK, KW_AUTO, KW_AUTOCOMMIT, + KW_AUTO_RELEASE_TEMP_BLOBID, KW_AVERAGE, KW_BASE_NAME, KW_BETWEEN, @@ -270,7 +271,6 @@ enum kwwords_t { KW_PRIVILEGES, KW_PROTECTED, KW_PUBLIC, - KW_AUTO_RELEASE_TEMP_BLOBID, KW_QUAD, KW_RAW_PARTITIONS, // ??? KW_READ, From 03d36f58feab7404b9b0c87afc8d7d2766791b0c Mon Sep 17 00:00:00 2001 From: Ilya Eremin Date: Sat, 23 Nov 2024 13:06:30 +0300 Subject: [PATCH 3/6] Avoid repeated attempts to start a transaction with options that are not supported by the target server --- src/burp/BurpTasks.cpp | 4 +- src/burp/burp.h | 3 ++ src/burp/restore.epp | 103 +++++++++++++++++++++++++++-------------- 3 files changed, 73 insertions(+), 37 deletions(-) diff --git a/src/burp/BurpTasks.cpp b/src/burp/BurpTasks.cpp index ca85d6800ca..5b1c6c9137a 100644 --- a/src/burp/BurpTasks.cpp +++ b/src/burp/BurpTasks.cpp @@ -869,6 +869,8 @@ void RestoreRelationTask::initItem(BurpGlobals* tdgbl, Item& item) tdgbl->verboseInterval = m_masterGbl->verboseInterval; tdgbl->RESTORE_format = m_masterGbl->RESTORE_format; tdgbl->runtimeODS = m_masterGbl->runtimeODS; + tdgbl->gbl_use_no_auto_undo = m_masterGbl->gbl_use_no_auto_undo; + tdgbl->gbl_use_auto_release_temp_blobid = m_masterGbl->gbl_use_auto_release_temp_blobid; if (item.m_ownAttach) { @@ -893,7 +895,7 @@ void RestoreRelationTask::initItem(BurpGlobals* tdgbl, Item& item) if (status->getState() & IStatus::STATE_ERRORS) BURP_abort(&status); - // SET TRANSACTION NO_AUTO_UNDO, see at the end of get_data() + // SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID, see at the end of get_data() ClumpletWriter tpb(ClumpletReader::Tpb, 128, isc_tpb_version3); tpb.insertTag(isc_tpb_concurrency); diff --git a/src/burp/burp.h b/src/burp/burp.h index d174c84f487..cb791efa7e4 100644 --- a/src/burp/burp.h +++ b/src/burp/burp.h @@ -1227,6 +1227,9 @@ class BurpGlobals : public Firebird::ThreadData, public GblPool bool gbl_stat_header; // true, if stats header was printed bool gbl_stat_done; // true, if main process is done, stop to collect db-level stats SINT64 gbl_stats[LAST_COUNTER]; + + bool gbl_use_no_auto_undo = true; + bool gbl_use_auto_release_temp_blobid = true; }; // CVC: This aux routine declared here to not force inclusion of burp.h with burp_proto.h diff --git a/src/burp/restore.epp b/src/burp/restore.epp index e7f69423517..ebfde1ca058 100644 --- a/src/burp/restore.epp +++ b/src/burp/restore.epp @@ -183,6 +183,7 @@ void update_ownership(BurpGlobals* tdgbl); void update_view_dbkey_lengths(BurpGlobals* tdgbl); void fix_missing_privileges(BurpGlobals* tdgbl); void fix_system_generators(BurpGlobals* tdgbl); +void set_transaction(BurpGlobals* tdgbl); void general_on_error(); #ifdef DEBUG UCHAR debug_on = 0; // able to turn this on in the debugger @@ -421,7 +422,7 @@ int RESTORE_restore (const TEXT* file_name, const TEXT* database_name) // Block added to verbose index creation by Toni Martir // Always try to activate deferred indices - it helps for some broken backups, // and in normal cases doesn't take much time to look for such indices. AP-2008. - EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; + EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; @@ -445,7 +446,7 @@ int RESTORE_restore (const TEXT* file_name, const TEXT* database_name) general_on_error (); END_ERROR; - EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; + EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; @@ -481,7 +482,7 @@ int RESTORE_restore (const TEXT* file_name, const TEXT* database_name) END_ERROR; } - EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; + EXEC SQL SET TRANSACTION ISOLATION LEVEL READ COMMITTED NO_AUTO_UNDO; if (gds_status->hasData()) EXEC SQL SET TRANSACTION; @@ -745,9 +746,7 @@ void add_files(BurpGlobals* tdgbl, const char* file_name) END_ERROR; END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + set_transaction(tdgbl); } } @@ -3166,9 +3165,7 @@ static void commit_relation_data(BurpGlobals* tdgbl, burp_rel* relation) } // end of while END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + set_transaction(tdgbl); } // We have a corrupt backup, save the restore process from becoming useless. @@ -7777,9 +7774,7 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t general_on_error (); END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + set_transaction(tdgbl); } // Pick up relation attributes @@ -8004,9 +7999,8 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t general_on_error (); END_ERROR; END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + + set_transaction(tdgbl); } return true; @@ -8041,9 +8035,7 @@ bool get_relation(BurpGlobals* tdgbl, Coordinator* coord, RestoreRelationTask* t general_on_error (); END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + set_transaction(tdgbl); // If we're only doing meta-data, ignore data records @@ -9085,9 +9077,8 @@ bool get_trigger_old (BurpGlobals* tdgbl, burp_rel* relation) general_on_error (); END_ERROR; END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + + set_transaction(tdgbl); } return true; @@ -9441,9 +9432,8 @@ bool get_trigger(BurpGlobals* tdgbl) general_on_error (); END_ERROR; END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + + set_transaction(tdgbl); } return true; @@ -9537,9 +9527,8 @@ bool get_trigger_message(BurpGlobals* tdgbl) general_on_error (); END_ERROR; END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + + set_transaction(tdgbl); } return true; @@ -10391,9 +10380,7 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file create_database(tdgbl, provider, database_name); - EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + set_transaction(tdgbl); // For V4.0, start a read commited transaction. This will be used // to create blobs for global fields and update the record in the @@ -10751,9 +10738,8 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file ON_ERROR general_on_error (); END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + + set_transaction(tdgbl); flag = false; } if (!get_relation_data(tdgbl, &coord, &task)) @@ -10875,9 +10861,8 @@ bool restore(BurpGlobals* tdgbl, Firebird::IProvider* provider, const TEXT* file ON_ERROR general_on_error (); END_ERROR; - EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; - if (gds_status->hasData()) - EXEC SQL SET TRANSACTION; + + set_transaction(tdgbl); } // put validation clauses for global fields @@ -11625,6 +11610,52 @@ void fix_system_generators(BurpGlobals* tdgbl) } } +void set_transaction(BurpGlobals* tdgbl) +{ +/************************************** + * + * s e t _ t r a n s a c t i o n + * + ************************************** + * + * Functional description + * Start a transaction with options + * supported by the target server. + * + **************************************/ + + while (true) + { + if (tdgbl->gbl_use_auto_release_temp_blobid && tdgbl->gbl_use_no_auto_undo) + { + EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; + if (gds_status->hasData()) + { + // First try to disable AUTO_RELEASE_TEMP_BLOBID transaction + // option because it was implemented later than NO_AUTO_UNDO + tdgbl->gbl_use_auto_release_temp_blobid = false; + continue; + } + } + else if (tdgbl->gbl_use_no_auto_undo) + { + EXEC SQL SET TRANSACTION NO_AUTO_UNDO; + if (gds_status->hasData()) + { + tdgbl->gbl_use_no_auto_undo = false; + continue; + } + } + else + { + fb_assert(!tdgbl->gbl_use_auto_release_temp_blobid); + EXEC SQL SET TRANSACTION; + } + + break; + } +} + } // namespace namespace Burp From eab3d1eb0863336fcd302c300e420e09e1a35630 Mon Sep 17 00:00:00 2001 From: Ilya Eremin Date: Sat, 23 Nov 2024 15:07:12 +0300 Subject: [PATCH 4/6] Correct AUTO RELEASE TEMP BLOBID description --- doc/sql.extensions/README.set_transaction.txt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/doc/sql.extensions/README.set_transaction.txt b/doc/sql.extensions/README.set_transaction.txt index ebba0c04b49..2b086167c48 100644 --- a/doc/sql.extensions/README.set_transaction.txt +++ b/doc/sql.extensions/README.set_transaction.txt @@ -27,11 +27,12 @@ commit fails. This option is mostly used by gfix. LOCK TIMEOUT nonneg_short_integer: it's the time (measured in seconds) that a transaction waits for a lock in a record before giving up and reporting an error. -AUTO RELEASE TEMP BLOBID: makes the transaction release temporary ID of user BLOB -just after its materialization. It's useful for massive insertions of records with -user-defined BLOBs because it eliminates the memory overhead caused by creating and -keeping temporary IDs until the transaction ends. This option is used during the -database restore. +AUTO RELEASE TEMP BLOBID: makes the transaction release a temporary ID of a user +BLOB just after its materialization. It's useful for massive insertions of records +with user-defined BLOBs because it eliminates the memory overhead caused by creating +and keeping temporary IDs until the transaction ends. This option should be used +with care and only if there is no need to access a materialized BLOB via a temporary +ID obtained after its creation. It's used during the database restore. Author: From 51e224ff9909c06bca2d77acd2719d5ee6018a6f Mon Sep 17 00:00:00 2001 From: Ilya Eremin Date: Sat, 23 Nov 2024 18:37:48 +0300 Subject: [PATCH 5/6] Check bad_tpb_form error for more reliable detection of unsupported transaction options --- src/burp/restore.epp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/burp/restore.epp b/src/burp/restore.epp index ebfde1ca058..f992afd2b2e 100644 --- a/src/burp/restore.epp +++ b/src/burp/restore.epp @@ -11629,7 +11629,7 @@ void set_transaction(BurpGlobals* tdgbl) if (tdgbl->gbl_use_auto_release_temp_blobid && tdgbl->gbl_use_no_auto_undo) { EXEC SQL SET TRANSACTION NO_AUTO_UNDO AUTO_RELEASE_TEMP_BLOBID; - if (gds_status->hasData()) + if (gds_status->hasData() && fb_utils::containsErrorCode(gds_status->getErrors(), isc_bad_tpb_form)) { // First try to disable AUTO_RELEASE_TEMP_BLOBID transaction // option because it was implemented later than NO_AUTO_UNDO @@ -11640,7 +11640,7 @@ void set_transaction(BurpGlobals* tdgbl) else if (tdgbl->gbl_use_no_auto_undo) { EXEC SQL SET TRANSACTION NO_AUTO_UNDO; - if (gds_status->hasData()) + if (gds_status->hasData() && fb_utils::containsErrorCode(gds_status->getErrors(), isc_bad_tpb_form)) { tdgbl->gbl_use_no_auto_undo = false; continue; From 6c87a5ccb75cf94e8ca0f958e995521464acf863 Mon Sep 17 00:00:00 2001 From: Ilya Eremin Date: Sat, 23 Nov 2024 18:45:42 +0300 Subject: [PATCH 6/6] Do not use unsupported options for transactions of parallel workers performing RestoreRelationTask --- src/burp/BurpTasks.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/burp/BurpTasks.cpp b/src/burp/BurpTasks.cpp index 5b1c6c9137a..396e1d7ea53 100644 --- a/src/burp/BurpTasks.cpp +++ b/src/burp/BurpTasks.cpp @@ -899,8 +899,12 @@ void RestoreRelationTask::initItem(BurpGlobals* tdgbl, Item& item) ClumpletWriter tpb(ClumpletReader::Tpb, 128, isc_tpb_version3); tpb.insertTag(isc_tpb_concurrency); - tpb.insertTag(isc_tpb_no_auto_undo); - tpb.insertTag(isc_tpb_auto_release_temp_blobid); + + if (tdgbl->gbl_use_no_auto_undo) + tpb.insertTag(isc_tpb_no_auto_undo); + + if (tdgbl->gbl_use_auto_release_temp_blobid) + tpb.insertTag(isc_tpb_auto_release_temp_blobid); item.m_tra = item.m_att->startTransaction(&status, tpb.getBufferLength(), tpb.getBuffer());