diff --git a/storage-resize-images/CHANGELOG.md b/storage-resize-images/CHANGELOG.md index 3cd98058a..0e7d219ff 100644 --- a/storage-resize-images/CHANGELOG.md +++ b/storage-resize-images/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.3.0 + +fix! - remove backfill, due to architectural flaws. + ## Version 0.2.10 feat - added param for adjusting backfill max batch size diff --git a/storage-resize-images/README.md b/storage-resize-images/README.md index 689e080c1..69d5944a9 100644 --- a/storage-resize-images/README.md +++ b/storage-resize-images/README.md @@ -144,7 +144,7 @@ You can find more information about this extension in the following articles: * Sizes of resized images: What sizes of images would you like (in pixels)? Enter the sizes as a comma-separated list of WIDTHxHEIGHT values. Learn more about [how this parameter works](https://firebase.google.com/products/extensions/storage-resize-images). -* Deletion of original file: Do you want to automatically delete the original file from the Cloud Storage bucket? Warning: these deletions cannot be undone, and if you reconfigure this instance to use different image dimensions, you won't be able to backfill deleted images. +* Deletion of original file: Do you want to automatically delete the original file from the Cloud Storage bucket? Warning: these deletions cannot be undone. * Make resized images public: Do you want to make the resized images public automatically? So you can access them by URL. For example: https://storage.googleapis.com/{bucket}/{path} @@ -182,13 +182,6 @@ Leave this field empty if you do not want to store failed images in a separate d * Cloud Function memory: Memory of the function responsible of resizing images. Choose how much memory to give to the function that resize images. (For animated GIF => GIF we recommend using a minimum of 2GB). -* Backfill existing images: Should existing, unresized images in the Storage bucket be resized as well? - - -* Backfill batch size: The maximum number of images to resize in a single batch. By default, the function is configured to resize 3 images at a time. This is a conservative default to work with smallest memory option (512 MB). -WARNING: Ensure your function has enough memory to handle the batch size. - - * Assign new access token: Should resized images have a new access token assigned to them, different from the original image? @@ -207,8 +200,6 @@ WARNING: Ensure your function has enough memory to handle the batch size. * **generateResizedImage:** Listens for new images uploaded to your specified Cloud Storage bucket, resizes the images, then stores the resized images in the same bucket. Optionally keeps or deletes the original images. -* **backfillResizedImages:** Handles tasks from startBackfill to resize existing images. - **APIs Used**: diff --git a/storage-resize-images/extension.yaml b/storage-resize-images/extension.yaml index 1e593295e..7cbe78f11 100644 --- a/storage-resize-images/extension.yaml +++ b/storage-resize-images/extension.yaml @@ -13,7 +13,7 @@ # limitations under the License. name: storage-resize-images -version: 0.2.10 +version: 0.3.10 specVersion: v1beta displayName: Resize Images @@ -65,14 +65,15 @@ resources: eventTrigger: eventType: google.storage.object.finalize resource: projects/_/buckets/${param:IMG_BUCKET} - - name: backfillResizedImages - type: firebaseextensions.v1beta.function - description: >- - Handles tasks from startBackfill to resize existing images. - properties: - runtime: nodejs20 - availableMemoryMb: ${param:FUNCTION_MEMORY} - taskQueueTrigger: {} + # Backfill feature disabled - commented out to preserve code + # - name: backfillResizedImages + # type: firebaseextensions.v1beta.function + # description: >- + # Handles tasks from startBackfill to resize existing images. + # properties: + # runtime: nodejs20 + # availableMemoryMb: ${param:FUNCTION_MEMORY} + # taskQueueTrigger: {} params: - param: IMG_BUCKET @@ -111,9 +112,7 @@ params: label: Deletion of original file description: >- Do you want to automatically delete the original file from the Cloud - Storage bucket? Warning: these deletions cannot be undone, and if you - reconfigure this instance to use different image dimensions, you won't be - able to backfill deleted images. + Storage bucket? Warning: these deletions cannot be undone. type: select options: - label: Don't delete @@ -316,32 +315,33 @@ params: required: true immutable: false - - param: DO_BACKFILL - label: Backfill existing images - description: > - Should existing, unresized images in the Storage bucket be resized as - well? - type: select - required: true - options: - - label: Yes - value: true - - label: No - value: false - - - param: BACKFILL_BATCH_SIZE - label: Backfill batch size - description: > - The maximum number of images to resize in a single batch. By default, the - function is configured to resize 3 images at a time. This is a - conservative default to work with smallest memory option (512 MB). - - WARNING: Ensure your function has enough memory to handle the batch size. - type: string - default: 3 - validationRegex: ^[1-9][0-9]*$ - validationErrorMessage: Please provide a valid number. - required: false + # Backfill parameters disabled - commented out to preserve code + # - param: DO_BACKFILL + # label: Backfill existing images + # description: > + # Should existing, unresized images in the Storage bucket be resized as + # well? + # type: select + # required: true + # options: + # - label: Yes + # value: true + # - label: No + # value: false + + # - param: BACKFILL_BATCH_SIZE + # label: Backfill batch size + # description: > + # The maximum number of images to resize in a single batch. By default, the + # function is configured to resize 3 images at a time. This is a + # conservative default to work with smallest memory option (512 MB). + # + # WARNING: Ensure your function has enough memory to handle the batch size. + # type: string + # default: 3 + # validationRegex: ^[1-9][0-9]*$ + # validationErrorMessage: Please provide a valid number. + # required: false - param: REGENERATE_TOKEN label: Assign new access token @@ -426,14 +426,14 @@ events: description: Occurs when the function is settled. Provides no customized data other than the context. - -lifecycleEvents: - onInstall: - function: backfillResizedImages - processingMessage: Resizing existing images in ${param:IMG_BUCKET} - onUpdate: - function: backfillResizedImages - processingMessage: Resizing existing images in ${param:IMG_BUCKET} - onConfigure: - function: backfillResizedImages - processingMessage: Resizing existing images in ${param:IMG_BUCKET} +# Lifecycle events disabled - backfill feature commented out +# lifecycleEvents: +# onInstall: +# function: backfillResizedImages +# processingMessage: Resizing existing images in ${param:IMG_BUCKET} +# onUpdate: +# function: backfillResizedImages +# processingMessage: Resizing existing images in ${param:IMG_BUCKET} +# onConfigure: +# function: backfillResizedImages +# processingMessage: Resizing existing images in ${param:IMG_BUCKET} diff --git a/storage-resize-images/functions/__tests__/__mocks__/src/config.ts b/storage-resize-images/functions/__tests__/__mocks__/src/config.ts index 5e81e04b8..c98e07a3c 100644 --- a/storage-resize-images/functions/__tests__/__mocks__/src/config.ts +++ b/storage-resize-images/functions/__tests__/__mocks__/src/config.ts @@ -51,7 +51,7 @@ export const config = { bucket: "extensions-testing.appspot.com", cacheControlHeader: undefined, - doBackfill: false, + // doBackfill: false, imageSizes: ["200x200"], regenerateToken: false, makePublic: false, diff --git a/storage-resize-images/functions/__tests__/__snapshots__/config.test.ts.snap b/storage-resize-images/functions/__tests__/__snapshots__/config.test.ts.snap index 64999bd9e..20f63350a 100644 --- a/storage-resize-images/functions/__tests__/__snapshots__/config.test.ts.snap +++ b/storage-resize-images/functions/__tests__/__snapshots__/config.test.ts.snap @@ -3,13 +3,11 @@ exports[`extension configuration detected from environment variables 1`] = ` Object { "animated": false, - "backfillBatchSize": 3, "bucket": "extensions-testing.appspot.com", "cacheControlHeader": undefined, "contentFilterLevel": null, "customFilterPrompt": null, "deleteOriginalFile": 0, - "doBackfill": false, "excludePathList": undefined, "failedImagesPath": undefined, "imageSizes": Array [ diff --git a/storage-resize-images/functions/__tests__/filters.test.ts b/storage-resize-images/functions/__tests__/filters.test.ts index 8db9b27f0..401a95467 100644 --- a/storage-resize-images/functions/__tests__/filters.test.ts +++ b/storage-resize-images/functions/__tests__/filters.test.ts @@ -11,7 +11,7 @@ jest.mock("../src/config", () => { imgSizes: ["200x200"], resizedImagesPath: undefined, deleteOriginalFile: "true", - backfillBatchSize: 3, + // backfillBatchSize: 3, }, }; }); diff --git a/storage-resize-images/functions/__tests__/function.test.ts b/storage-resize-images/functions/__tests__/function.test.ts index 7111f9284..8d869c5bd 100644 --- a/storage-resize-images/functions/__tests__/function.test.ts +++ b/storage-resize-images/functions/__tests__/function.test.ts @@ -27,7 +27,7 @@ jest.mock("../src/config", () => { imgSizes: ["200x200"], resizedImagesPath: undefined, deleteOriginalFile: "true", - backfillBatchSize: 3, + // backfillBatchSize: 3, }, }; }); diff --git a/storage-resize-images/functions/src/config.ts b/storage-resize-images/functions/src/config.ts index e5436fd90..863ae77b7 100644 --- a/storage-resize-images/functions/src/config.ts +++ b/storage-resize-images/functions/src/config.ts @@ -68,7 +68,8 @@ export const convertHarmBlockThreshold: ( export const config = { bucket: process.env.IMG_BUCKET, cacheControlHeader: process.env.CACHE_CONTROL_HEADER, - doBackfill: process.env.DO_BACKFILL === "true", + // Backfill feature disabled - commented out to preserve code + // doBackfill: process.env.DO_BACKFILL === "true", imageSizes: process.env.IMG_SIZES.split(","), regenerateToken: process.env.REGENERATE_TOKEN == "true", makePublic: process.env.MAKE_PUBLIC === "true", @@ -88,7 +89,7 @@ export const config = { ), customFilterPrompt: process.env.CUSTOM_FILTER_PROMPT || null, placeholderImagePath: process.env.PLACEHOLDER_IMAGE_PATH || null, - backfillBatchSize: Number(process.env.BACKFILL_BATCH_SIZE) || 3, + // backfillBatchSize: Number(process.env.BACKFILL_BATCH_SIZE) || 3, }; export type Config = typeof config; diff --git a/storage-resize-images/functions/src/index.ts b/storage-resize-images/functions/src/index.ts index 9a5cd8f86..5cb7034f5 100644 --- a/storage-resize-images/functions/src/index.ts +++ b/storage-resize-images/functions/src/index.ts @@ -156,80 +156,80 @@ export const generateResizedImage = functions.storage /** * */ -export const backfillResizedImages = functions.tasks - .taskQueue() - .onDispatch(async (data) => { - const runtime = getExtensions().runtime(); - if (!config.doBackfill) { - await runtime.setProcessingState( - "PROCESSING_COMPLETE", - "Existing images were not resized because 'Backfill existing images' was configured to false." + - " If you want to resize existing images, reconfigure this instance." - ); - return; - } - if (data?.nextPageQuery == undefined) { - logs.startBackfill(); - } - - const bucket = admin.storage().bucket(process.env.IMG_BUCKET); - const query = data.nextPageQuery || { - autoPaginate: false, - maxResults: config.backfillBatchSize, - }; - const [files, nextPageQuery] = await bucket.getFiles(query); - const filesToResize = files.filter((f: File) => { - logs.continueBackfill(f.metadata.name); - return shouldResize(convertToObjectMetadata(f.metadata)); - }); - - const filePromises = filesToResize.map((f) => { - return generateResizedImageHandler( - convertToObjectMetadata(f.metadata), - /*verbose=*/ false - ); - }); - const results = await Promise.allSettled(filePromises); - - const pageErrorsCount = results.filter( - (r) => r.status === "rejected" - ).length; - const pageSuccessCount = results.filter( - (r) => r.status === "fulfilled" - ).length; - const oldErrorsCount = Number(data.errorsCount) || 0; - const oldSuccessCount = Number(data.successCount) || 0; - const errorsCount = pageErrorsCount + oldErrorsCount; - const successCount = pageSuccessCount + oldSuccessCount; - - if (nextPageQuery) { - const queue = getFunctions().taskQueue( - `locations/${config.location}/functions/backfillResizedImages`, - process.env.EXT_INSTANCE_ID - ); - await queue.enqueue({ - nextPageQuery, - errorsCount, - successCount, - }); - } else { - logs.backfillComplete(successCount, errorsCount); - if (errorsCount == 0) { - await runtime.setProcessingState( - "PROCESSING_COMPLETE", - `Successfully resized ${successCount} images.` - ); - } else if (errorsCount > 0 && successCount > 0) { - await runtime.setProcessingState( - "PROCESSING_WARNING", - `Successfully resized ${successCount} images, failed to resize ${errorsCount} images. See function logs for error details.` - ); - } - if (errorsCount > 0 && successCount == 0) { - await runtime.setProcessingState( - "PROCESSING_FAILED", - `Successfully resized ${successCount} images, failed to resize ${errorsCount} images. See function logs for error details.` - ); - } - } - }); +// export const backfillResizedImages = functions.tasks +// .taskQueue() +// .onDispatch(async (data) => { +// const runtime = getExtensions().runtime(); +// if (!config.doBackfill) { +// await runtime.setProcessingState( +// "PROCESSING_COMPLETE", +// "Existing images were not resized because 'Backfill existing images' was configured to false." + +// " If you want to resize existing images, reconfigure this instance." +// ); +// return; +// } +// if (data?.nextPageQuery == undefined) { +// logs.startBackfill(); +// } + +// const bucket = admin.storage().bucket(process.env.IMG_BUCKET); +// const query = data.nextPageQuery || { +// autoPaginate: false, +// maxResults: config.backfillBatchSize, +// }; +// const [files, nextPageQuery] = await bucket.getFiles(query); +// const filesToResize = files.filter((f: File) => { +// logs.continueBackfill(f.metadata.name); +// return shouldResize(convertToObjectMetadata(f.metadata)); +// }); + +// const filePromises = filesToResize.map((f) => { +// return generateResizedImageHandler( +// convertToObjectMetadata(f.metadata), +// /*verbose=*/ false +// ); +// }); +// const results = await Promise.allSettled(filePromises); + +// const pageErrorsCount = results.filter( +// (r) => r.status === "rejected" +// ).length; +// const pageSuccessCount = results.filter( +// (r) => r.status === "fulfilled" +// ).length; +// const oldErrorsCount = Number(data.errorsCount) || 0; +// const oldSuccessCount = Number(data.successCount) || 0; +// const errorsCount = pageErrorsCount + oldErrorsCount; +// const successCount = pageSuccessCount + oldSuccessCount; + +// if (nextPageQuery) { +// const queue = getFunctions().taskQueue( +// `locations/${config.location}/functions/backfillResizedImages`, +// process.env.EXT_INSTANCE_ID +// ); +// await queue.enqueue({ +// nextPageQuery, +// errorsCount, +// successCount, +// }); +// } else { +// logs.backfillComplete(successCount, errorsCount); +// if (errorsCount == 0) { +// await runtime.setProcessingState( +// "PROCESSING_COMPLETE", +// `Successfully resized ${successCount} images.` +// ); +// } else if (errorsCount > 0 && successCount > 0) { +// await runtime.setProcessingState( +// "PROCESSING_WARNING", +// `Successfully resized ${successCount} images, failed to resize ${errorsCount} images. See function logs for error details.` +// ); +// } +// if (errorsCount > 0 && successCount == 0) { +// await runtime.setProcessingState( +// "PROCESSING_FAILED", +// `Successfully resized ${successCount} images, failed to resize ${errorsCount} images. See function logs for error details.` +// ); +// } +// } +// });