New built-in function BLOB_APPEND

src/common/keywords.cpp

@@ -100,6 +100,7 @@ static const TOK tokens[] =
 	{TOK_BIND, "BIND", true},
 	{TOK_BIT_LENGTH, "BIT_LENGTH", false},
 	{TOK_BLOB, "BLOB", false},
+	{TOK_BLOB_APPEND, "BLOB_APPEND", true},
 	{TOK_BLOCK, "BLOCK", true},
 	{TOK_BODY, "BODY", true},
 	{TOK_BOOLEAN, "BOOLEAN", false},

src/dsql/parse.y

@@ -681,6 +681,7 @@ using namespace Firebird;
 
 %token <metaNamePtr> DEBUG
 %token <metaNamePtr> PKCS_1_5
+%token <metaNamePtr> BLOB_APPEND
 
 // precedence declarations for expression evaluation
 
@@ -8106,6 +8107,7 @@ system_function_std_syntax
 	| BIN_SHL
 	| BIN_SHR
 	| BIN_XOR
+	| BLOB_APPEND
 	| CEIL
 	| CHAR_TO_UUID
 	| COS
@@ -9062,6 +9064,7 @@ non_reserved_word
 	| ZONE
 	| DEBUG				// added in FB 4.0.1
 	| PKCS_1_5
+	| BLOB_APPEND
 	;
 
 %%

src/jrd/SysFunction.cpp

@@ -222,6 +222,7 @@ bool dscHasData(const dsc* param);
 // specific setParams functions
 void setParamsAsciiVal(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args);
 void setParamsBin(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args);
+void setParamsBlobAppend(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args);
 void setParamsCharToUuid(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args);
 void setParamsDateAdd(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args);
 void setParamsDateDiff(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, int argsCount, dsc** args);
@@ -256,6 +257,7 @@ void makeAbs(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* r
 void makeAsciiChar(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
 void makeBin(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
 void makeBinShift(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
+void makeBlobAppend(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
 void makeCeilFloor(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
 void makeDateAdd(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
 void makeDecode64(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, int argsCount, const dsc** args);
@@ -293,6 +295,7 @@ dsc* evlAsciiVal(thread_db* tdbb, const SysFunction* function, const NestValueAr
 dsc* evlAtan2(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
 dsc* evlBin(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
 dsc* evlBinShift(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
+dsc* evlBlobAppend(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
 dsc* evlCeil(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
 dsc* evlCharToUuid(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
 dsc* evlDateAdd(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, impure_value* impure);
@@ -603,6 +606,19 @@ void setParamsAsciiVal(DataTypeUtilBase*, const SysFunction*, int argsCount, dsc
 }
 
 
+void setParamsBlobAppend(DataTypeUtilBase*, const SysFunction*, int argsCount, dsc** args)
+{
+	if (argsCount >= 1 && args[0]->isUnknown())
+		args[0]->makeBlob(isc_blob_text, CS_dynamic);
+
+	for (int i = 1; i < argsCount; ++i)
+	{
+		if (args[i]->isUnknown())
+			args[i]->makeVarying(80, args[0]->getTextType());
+	}
+}
+
+
 void setParamsCharToUuid(DataTypeUtilBase*, const SysFunction*, int argsCount, dsc** args)
 {
 	if (argsCount >= 1 && args[0]->isUnknown())
@@ -1217,6 +1233,18 @@ void makeBinShift(DataTypeUtilBase*, const SysFunction* function, dsc* result,
 }
 
 
+void makeBlobAppend(DataTypeUtilBase* dataTypeUtil, const SysFunction* function, dsc* result, 
+	int argsCount, const dsc** args)
+{
+	USHORT ttype = CS_dynamic;
+
+	if (argsCount > 0 && args[0])
+		ttype = args[0]->getTextType();
+
+	result->makeBlob(isc_blob_text, ttype);
+}
+
+
 void makeCeilFloor(DataTypeUtilBase*, const SysFunction* function, dsc* result,
 	int argsCount, const dsc** args)
 {
@@ -2203,6 +2231,146 @@ HUGEINT getScale(impure_value* impure)
 }
 
 
+static void appendFromBlob(thread_db* tdbb, jrd_tra* transaction, blb* blob, 
+	const dsc* blobDsc, const dsc* srcDsc)
+{
+	if (!srcDsc->dsc_address)
+		return;
+
+	bid* srcBlobID = (bid*)srcDsc->dsc_address;
+	if (srcBlobID->isEmpty())
+		return;
+
+	if (memcmp(blobDsc->dsc_address, srcDsc->dsc_address, sizeof(bid)) == 0)
+		status_exception::raise(Arg::Gds(isc_random) << Arg::Str("Can not append blob to itself"));
+
+	UCharBuffer bpb;
+	BLB_gen_bpb_from_descs(srcDsc, blobDsc, bpb);
+
+	AutoBlb srcBlob(tdbb, blb::open2(tdbb, transaction, srcBlobID, bpb.getCount(), bpb.begin()));
+
+	Database* dbb = tdbb->getDatabase();
+
+	HalfStaticArray<UCHAR, BUFFER_LARGE> buffer;
+	const SLONG buffSize = (srcBlob->getLevel() == 0) ? 
+		MAX(BUFFER_LARGE, srcBlob->blb_length) : dbb->dbb_page_size - BLP_SIZE;
+
+	UCHAR* buff = buffer.getBuffer(buffSize);
+	while (!(srcBlob->blb_flags & BLB_eof))
+	{
+		const SLONG len = srcBlob->BLB_get_data(tdbb, buff, buffSize, false);
+		if (len)
+			blob->BLB_put_data(tdbb, buff, len);
+	}
+}
+
+
+dsc* evlBlobAppend(thread_db* tdbb, const SysFunction* function, const NestValueArray& args, 
+	impure_value* impure)
+{
+	jrd_req* request = tdbb->getRequest();
+	jrd_tra* transaction = request ? request->req_transaction : tdbb->getTransaction();
+	transaction = transaction->getOuter();
+
+	USHORT ttype = tdbb->getCharSet();
+
+	blb* blob = NULL;
+	bid blob_id;
+	dsc blobDsc;
+
+	blob_id.clear();
+	blobDsc.clear();
+
+	const dsc* argDsc = EVL_expr(tdbb, request, args[0]);
+	const bool arg0_null = (request->req_flags & req_null) || (argDsc == NULL);
+
+	if (!arg0_null && argDsc->isBlob())
+		blob_id = *reinterpret_cast<bid*>(argDsc->dsc_address);
+
+	const dsc* declDsc = argDsc;
+	if (!declDsc)
+		declDsc = EVL_assign_to(tdbb, args[0]);
+
+	if (declDsc && declDsc->isBlob())
+	{
+		ttype = declDsc->getTextType();
+		blobDsc.makeBlob(declDsc->getBlobSubType(), ttype, (ISC_QUAD*)&blob_id);
+	}
+	else
+	{
+		if (declDsc && declDsc->isText())
+			ttype = declDsc->getTextType();
+
+		blobDsc.makeBlob(isc_blob_text, ttype, (ISC_QUAD*) &blob_id);
+	}
+
+	bool copyBlob = !blob_id.isEmpty();
+	if (copyBlob)
+	{
+		if (!blob_id.bid_internal.bid_relation_id)
+		{
+			if (!transaction->tra_blobs->locate(blob_id.bid_temp_id()))
+				status_exception::raise(Arg::Gds(isc_bad_segstr_id));
+
+			BlobIndex blobIdx = transaction->tra_blobs->current();
+			if (!blobIdx.bli_materialized && (blobIdx.bli_blob_object->blb_flags & BLB_close_on_read))
+			{
+				blob = blobIdx.bli_blob_object;
+				copyBlob = false;
+			}
+		}
+	}
+
+	if (!blob)
+	{
+		UCharBuffer bpb;
+		BLB_gen_bpb_from_descs(&blobDsc, &blobDsc, bpb);
+		bpb.push(isc_bpb_storage);
+		bpb.push(1);
+		bpb.push(isc_bpb_storage_temp);
+
+		blob = blb::create2(tdbb, transaction, &blob_id, bpb.getCount(), bpb.begin());
+		blob->blb_flags |= BLB_stream | BLB_close_on_read;
+	}
+
+//	if (copyBlob && argDsc && argDsc->isBlob())
+//		appendFromBlob(tdbb, transaction, blob, &blobDsc, argDsc);
+
+	EVL_make_value(tdbb, &blobDsc, impure);
+
+	for (FB_SIZE_T i = 0; i < args.getCount(); i++)
+	{
+		if (i == 0)
+		{
+			if (arg0_null || argDsc->isBlob() && !copyBlob)
+				continue;
+		}
+		else
+		{
+			argDsc = EVL_expr(tdbb, request, args[i]);
+			if ((request->req_flags & req_null) || !argDsc)
+				continue;
+		}
+
+		if (!argDsc->isBlob())
+		{
+			MoveBuffer temp;
+			UCHAR* addr = NULL;
+			SLONG len = MOV_make_string2(tdbb, argDsc, ttype, &addr, temp);
+
+			if (addr)
+				blob->BLB_put_data(tdbb, addr, len);
+		}
+		else
+		{
+			appendFromBlob(tdbb, transaction, blob, &blobDsc, argDsc);
+		}
+	}
+
+	return &impure->vlu_desc;
+}
+
+
 dsc* evlCeil(thread_db* tdbb, const SysFunction*, const NestValueArray& args,
 	impure_value* impure)
 {
@@ -6447,6 +6615,7 @@ const SysFunction SysFunction::functions[] =
 		{"BIN_SHL_ROT", 2, 2, setParamsInteger, makeBinShift, evlBinShift, (void*) funBinShlRot},
 		{"BIN_SHR_ROT", 2, 2, setParamsInteger, makeBinShift, evlBinShift, (void*) funBinShrRot},
 		{"BIN_XOR", 2, -1, setParamsBin, makeBin, evlBin, (void*) funBinXor},
+		{"BLOB_APPEND", 2, -1, setParamsBlobAppend, makeBlobAppend, evlBlobAppend, NULL},
 		{"CEIL", 1, 1, setParamsDblDec, makeCeilFloor, evlCeil, NULL},
 		{"CEILING", 1, 1, setParamsDblDec, makeCeilFloor, evlCeil, NULL},
 		{"CHAR_TO_UUID", 1, 1, setParamsCharToUuid, makeUuid, evlCharToUuid, NULL},

src/jrd/blb.cpp

@@ -59,6 +59,7 @@
 #include "../jrd/dpm_proto.h"
 #include "../jrd/err_proto.h"
 #include "../jrd/evl_proto.h"
+#include "../jrd/exe_proto.h"
 #include "../jrd/filte_proto.h"
 #include "../yvalve/gds_proto.h"
 #include "../jrd/intl_proto.h"
@@ -113,7 +114,12 @@ void blb::BLB_cancel(thread_db* tdbb)
 	// Release filter control resources
 
 	if (blb_flags & BLB_temporary)
+	{
+		if (!(blb_flags & BLB_closed))
+			blb_transaction->tra_temp_blobs_count--;
+
 		delete_blob(tdbb, 0);
+	}
 
 	destroy(true);
 }
@@ -189,11 +195,14 @@ bool blb::BLB_close(thread_db* tdbb)
 
 	SET_TDBB(tdbb);
 
+	const bool alreadyClosed = (blb_flags & BLB_closed);
+
 	// Release filter control resources
 
 	if (blb_filter)
 		BLF_close_blob(tdbb, &blb_filter);
 
+	blb_flags &= ~BLB_close_on_read;
 	blb_flags |= BLB_closed;
 
 	if (!(blb_flags & BLB_temporary))
@@ -202,6 +211,9 @@ bool blb::BLB_close(thread_db* tdbb)
 		return true;
 	}
 
+	if (!alreadyClosed)
+		blb_transaction->tra_temp_blobs_count--;
+
 	if (blb_level == 0)
 	{
 		//Database* dbb = tdbb->getDatabase();
@@ -267,6 +279,40 @@ blb* blb::create2(thread_db* tdbb,
 	Database* dbb = tdbb->getDatabase();
 	CHECK_DBB(dbb);
 
+	const int maxTempBlobs = MAX_TEMP_BLOBS;
+	if (maxTempBlobs > 0 && transaction->tra_temp_blobs_count >= maxTempBlobs)
+	{
+		const jrd_req* request = tdbb->getRequest();
+		string info;
+
+		if (userBlob)
+		{
+			Attachment* att = tdbb->getAttachment();
+			info = "By user application";
+			if (att->att_remote_process.hasData())
+			{
+				info += string(" (") + att->att_remote_process.c_str() + ")";
+			}
+		}
+		else if (request)
+		{
+			const JrdStatement* const statement = request->getStatement();
+			if (statement && statement->sqlText) 
+				info = string("By query: ") + *statement->sqlText;
+
+			string stack;
+			if (EXE_get_stack_trace(request, stack))
+			{
+				info += "\n";
+				info += stack;
+			}
+		}
+
+		gds__log("Too many temporary blobs (%i allowed)\n%s", maxTempBlobs, info.c_str());
+
+		ERR_post(Arg::Gds(isc_random) << Arg::Str("Too many temporary blobs"));
+	}
+
 	// Create a blob large enough to hold a single data page
 	SSHORT from, to;
 	SSHORT from_charset, to_charset;
@@ -322,6 +368,7 @@ blb* blb::create2(thread_db* tdbb,
 
 	blob->blb_space_remaining = blob->blb_clump_size;
 	blob->blb_flags |= BLB_temporary;
+	blob->blb_transaction->tra_temp_blobs_count++;
 
 	if (filter_required)
 	{
@@ -1163,7 +1210,10 @@ void blb::move(thread_db* tdbb, dsc* from_desc, dsc* to_desc, const record_param
 
 				if (!blob || !(blob->blb_flags & BLB_closed))
 				{
-					ERR_post(Arg::Gds(isc_bad_segstr_id));
+					if (blob && (blob->blb_flags & BLB_close_on_read))
+						blob->BLB_close(tdbb);
+					else
+						ERR_post(Arg::Gds(isc_bad_segstr_id));
 				}
 
 				if (blob->blb_level && (blob->blb_pg_space_id != relPages->rel_pg_space_id))
@@ -1327,7 +1377,7 @@ blb* blb::open2(thread_db* tdbb,
 			 */
 
 			// Search the index of transaction blobs for a match
-			const blb* new_blob = NULL;
+			blb* new_blob = NULL;
 			if (transaction->tra_blobs->locate(blobId.bid_temp_id()))
 			{
 				current = &transaction->tra_blobs->current();
@@ -1342,7 +1392,10 @@ blb* blb::open2(thread_db* tdbb,
 				if (!new_blob || !(new_blob->blb_flags & BLB_temporary) ||
 					!(new_blob->blb_flags & BLB_closed))
 				{
-					ERR_post(Arg::Gds(isc_bad_segstr_id));
+					if (new_blob && (new_blob->blb_flags & BLB_close_on_read))
+						new_blob->BLB_close(tdbb);
+					else
+						ERR_post(Arg::Gds(isc_bad_segstr_id));
 				}
 
 				blob->blb_lead_page = new_blob->blb_lead_page;
@@ -1542,7 +1595,7 @@ void blb::BLB_put_segment(thread_db* tdbb, const void* seg, USHORT segment_lengt
 
 	// Make sure blob is a temporary blob.  If not, complain bitterly.
 
-	if (!(blb_flags & BLB_temporary))
+	if (!(blb_flags & BLB_temporary) || (blb_flags & BLB_closed))
 		ERR_post(Arg::Gds(isc_cannot_update_old_blob));
 
 	if (blb_filter)

src/jrd/blb.h

@@ -172,6 +172,7 @@ const int BLB_closed		= 8;		// Temporary blob has been closed
 const int BLB_damaged		= 16;		// Blob is busted
 const int BLB_seek			= 32;		// Seek is pending
 const int BLB_large_scan	= 64;		// Blob is larger than page buffer cache
+const int BLB_close_on_read = 128;		// Temporary blob is not closed until read
 
 /* Blob levels are: