fix error handling for missing instances

Author: haugenj

pkg/monitor/sqsevent/sqs-monitor.go

@@ -17,9 +17,9 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-
 	"github.com/aws/aws-node-termination-handler/pkg/monitor"
 	"github.com/aws/aws-sdk-go/aws"
+	"github.com/aws/aws-sdk-go/aws/awserr"
 	"github.com/aws/aws-sdk-go/service/autoscaling"
 	"github.com/aws/aws-sdk-go/service/autoscaling/autoscalingiface"
 	"github.com/aws/aws-sdk-go/service/ec2"
@@ -88,7 +88,7 @@ func (m SQSMonitor) Monitor() error {
 	}
 
 	if len(messages) > 0 && failedEvents == len(messages) {
-		return fmt.Errorf("All of the waiting queue events could not be processed")
+		return fmt.Errorf("none of the waiting queue events could be processed")
 	}
 
 	return nil
@@ -189,10 +189,14 @@ func (m SQSMonitor) retrieveNodeName(instanceID string) (string, error) {
 		},
 	})
 	if err != nil {
+		if aerr, ok := err.(awserr.Error); ok && aerr.Code() == "InvalidInstanceID.NotFound" {
+			log.Warn().Msgf("No instance found with instance-id %s", instanceID)
+			return "", ErrNodeStateNotRunning
+		}
 		return "", err
 	}
 	if len(result.Reservations) == 0 || len(result.Reservations[0].Instances) == 0 {
-		log.Info().Msgf("No instance found with instance-id %s", instanceID)
+		log.Warn().Msgf("No instance found with instance-id %s", instanceID)
 		return "", ErrNodeStateNotRunning
 	}
 

pkg/monitor/sqsevent/sqs-monitor_test.go

@@ -485,6 +485,42 @@ func TestMonitor_EC2NoInstances(t *testing.T) {
 	}
 }
 
+func TestMonitor_DescribeInstancesError(t *testing.T) {
+	for _, event := range []sqsevent.EventBridgeEvent{spotItnEvent, asgLifecycleEvent} {
+		msg, err := getSQSMessageFromEvent(event)
+		h.Ok(t, err)
+		messages := []*sqs.Message{
+			&msg,
+		}
+		sqsMock := h.MockedSQS{
+			ReceiveMessageResp: sqs.ReceiveMessageOutput{Messages: messages},
+			ReceiveMessageErr:  nil,
+		}
+		ec2Mock := h.MockedEC2{
+			DescribeInstancesResp: ec2.DescribeInstancesOutput{},
+			DescribeInstancesErr:  awserr.New("InvalidInstanceID.NotFound",  "The instance ID 'i-0d6bd3ce2bf8a6751' does not exist\n\tstatus code: 400, request id: 6a5c30e2-922d-464c-946c-a1ec76e5920b", fmt.Errorf("original error")),
+		}
+		drainChan := make(chan monitor.InterruptionEvent, 1)
+
+		sqsMonitor := sqsevent.SQSMonitor{
+			SQS:              sqsMock,
+			EC2:              ec2Mock,
+			QueueURL:         "https://test-queue",
+			InterruptionChan: drainChan,
+		}
+
+		err = sqsMonitor.Monitor()
+		h.Ok(t, err)
+
+		select {
+		case <-drainChan:
+			h.Ok(t, fmt.Errorf("Expected no events"))
+		default:
+			h.Ok(t, nil)
+		}
+	}
+}
+
 func TestMonitor_EC2NoDNSName(t *testing.T) {
 	msg, err := getSQSMessageFromEvent(asgLifecycleEvent)
 	h.Ok(t, err)