[IXE-10075] Enhancements of EBS Snapshot Lamda

lambda/ebs.js

@@ -1,9 +1,11 @@
 var utils = require('./utils');
 
-var Promise = require('bluebird');
 var AWS = require('aws-sdk');
-var ec2 = new AWS.EC2(utils.getRegionObject());
+AWS.config.update(utils.getRegionObject());
+var ec2 = new AWS.EC2(utils.getApiVersionObject());
+var ses = new AWS.SES(utils.getApiVersionObject());
 var config = require('./config.json');
+var promisesToPurgeSnapshotsInBatches = []
 
 var getPurgeDate = function(tags) {
   var purgeDate = new Date();
@@ -62,49 +64,121 @@ var snapshotVolumes = function () {
     ]
   };
 
-  var snapshotPromises = ec2.describeVolumes(getVolumesParam)
+  return ec2.describeVolumes(getVolumesParam)
     .promise()
-    .then(function(data) {
-      return data.Volumes.map(function(volume) {
-        return createSnapshot(volume.VolumeId)
-          .then(function(data) {
-            return tagSnapshot(volume, data.SnapshotId);
-          })
-      });
-    });
-
-    return Promise.all(snapshotPromises);
+    .then(data => Promise.all(
+      data.Volumes.map(volume =>
+        createSnapshot(volume.VolumeId)
+          .then(data => tagSnapshot(volume, data.SnapshotId))
+      )
+    ))
+    .catch(notifyFailure);
+
 };
 
-var deleteSnapshot = function(snapshotId) {
+var deleteSnapshot = function(SnapshotId) {
   var params = {
-    SnapshotId: snapshotId,
+    SnapshotId,
     DryRun: false
   };
-  return ec2.deleteSnapshot(params).promise();
+
+  console.log(">>> Deleting "+ SnapshotId + " ...");
+  return ec2.deleteSnapshot(params).promise()
+    .catch(err => {
+      if (err.statusCode == 400 && err.code == 'InvalidSnapshot.InUse') {
+        console.log(">>> Skipping ERROR on deleting "+ SnapshotId +" in use ...");
+        return Promise.resolve({});
+      }
+      return Promise.reject();
+    });
+};
+
+var checkSnapshotPurgeStatus = function(snapshot) {
+
+  if (!snapshot.State || snapshot.State == 'completed')   // Empty-response OR snapshot-delete-status obj considered successful here.
+    return Promise.resolve();
+
+  console.log('>>> ' + snapshot.SnapshotId + ' purge state is ' + snapshot.state + '. Retrying purge once more ...');
+  return retryDeleteSnapshot(snapshot.SnapshotId);
 };
 
-var purgeSnapshots = function() {
-  var today = utils.getDate(new Date());
-  var snapshotsParams = {
+var retryDeleteSnapshot = deleteSnapshot;
+
+var getSnapshots = function(MaxResults, NextToken) {
+  return ec2.describeSnapshots({
     DryRun: false,
     Filters: [
       {
         Name: "tag:PurgeDate",
-        Values: [today]
-      },
-    ]
-  };
-
-  var snapshotDeletePromises = ec2.describeSnapshots(snapshotsParams).promise()
-    .then(function(data) {
-      return data.Snapshots.map(function(snapshot) {
-        return deleteSnapshot(snapshot.SnapshotId);
-      });
+        Values: [process.argv[2] || utils.getDate(new Date())]
+      }
+    ],
+    // MaxResults,  //--
+    // NextToken,   //-- This pagination is not working
+  }).promise();
+};
+
+var purgeSnapshots = (MaxResults, NextToken) => getSnapshots(MaxResults, NextToken)
+  .then(data => Promise.all(
+
+      data.Snapshots.map(snapshot => deleteSnapshot(snapshot.SnapshotId)
+        .then(checkSnapshotPurgeStatus, err => retryDeleteSnapshot(snapshot.SnapshotId))
+      )
+
+    ).then(() => Promise.resolve(
+
+      (data.Snapshots && data.Snapshots.length)   // If there are more snapshots
+      ? data.NextToken                            // PASS NextToken
+      : null                                      // Else signal a NULL
+
+    ))
+
+  ).catch(notifyFailure);
+
+var purgeSnapshotsInBatches = function(BATCH_SIZE, next) {
+
+  return purgeSnapshots(BATCH_SIZE, next)
+    .then(function(nextToken) {
+      if (nextToken)
+        return promisesToPurgeSnapshotsInBatches.push(purgeSnapshotsInBatches(BATCH_SIZE, nextToken));
+
+      console.log('>>> Purge snapshot activity for '+ (process.argv[2] || utils.getDate(new Date())) +' completed in '+ (promisesToPurgeSnapshotsInBatches.length + 1) +' batches.');
+      return Promise.all(promisesToPurgeSnapshotsInBatches);
     });
-
-  return Promise.all(snapshotDeletePromises);
 };
 
+var notifyFailure = function(error) {
+
+  if (!(process.env.TO_EMAIL_ADDRESS && process.env.SENDER_EMAIL_ADDRESS))
+    return Promise.reject(error);
+
+  return ses.sendEmail({
+    Destination: {
+      CcAddresses: [
+        process.env.CC_EMAIL_ADDRESS || process.env.TO_EMAIL_ADDRESS,
+      ],
+      ToAddresses: [
+        process.env.TO_EMAIL_ADDRESS,
+      ]
+    },
+    Message: {
+      Body: {
+        Text: {
+         Charset: "UTF-8",
+         Data: error.message || error
+        }
+       },
+       Subject: {
+        Charset: 'UTF-8',
+        Data: 'EBS Snapshot Lambda Failed'
+       }
+      },
+    Source: process.env.SENDER_EMAIL_ADDRESS,
+  }).promise();
+
+}
+
 exports.snapshotVolumes = snapshotVolumes;
-exports.purgeSnapshots = purgeSnapshots;
+exports.purgeSnapshots = purgeSnapshots;
+exports.purgeSnapshotsInBatches = purgeSnapshotsInBatches;
+exports.notifyFailure = notifyFailure;

lambda/index.js

@@ -0,0 +1,22 @@
+var ebs = require('./ebs');
+
+
+var handler = (event, context, callback) =>
+
+  ebs.purgeSnapshotsInBatches(process.env.BATCH_SIZE || 10)
+  .then(ebs.snapshotVolumes, ebs.snapshotVolumes)
+  .then(()=> callback(null, 'Finished'))
+  .catch(callback);
+
+
+exports.handler = handler
+
+// // Uncomment below to test locally
+// exports.handler(null, null, function(e, s) {
+//   if(e) {
+//     console.log("[ERROR] ", e);
+//     return;
+//   }
+
+//   console.log(s);
+// });

lambda/package.json

@@ -2,11 +2,8 @@
   "name": "ebs_snapshot_lambda",
   "version": "0.1.0",
   "description": "AWS Lambda function to create and purge EBS snapshots",
-  "main": "ebs_snapshot_lambda.js",
   "scripts": {},
   "author": "manojlds <[email protected]> (https://stacktoheap.com/)",
   "license": "MIT",
-  "dependencies": {
-    "bluebird": "3.4.1"
-  }
+  "dependencies": {}
 }

lambda/utils.js

@@ -1,11 +1,20 @@
 var region = process.env.AWS_DEFAULT_REGION || "us-east-1";
+var apiVersion = process.env.AWS_DEFAULT_API_VERSION || "2016-11-15";
 
 var getRegion = function () {
   return region;
 }
 
 var getRegionObject = function() {
-  return { region: region };
+  return { region };
+}
+
+var getApiVersion = function() {
+  return apiVersion;
+}
+
+var getApiVersionObject = function() {
+  return { apiVersion };
 }
 
 var getTags = function(tags) {
@@ -21,5 +30,7 @@ var getDate = function(date) {
 
 exports.getRegion = getRegion;
 exports.getRegionObject = getRegionObject;
+exports.getApiVersion = getApiVersion;
+exports.getApiVersionObject = getApiVersionObject;
 exports.getTags = getTags;
 exports.getDate = getDate;