diff --git a/pkg/kubeapiserver/authenticator/config.go b/pkg/kubeapiserver/authenticator/config.go
index 5913036804843..523387a4a36b3 100644
--- a/pkg/kubeapiserver/authenticator/config.go
+++ b/pkg/kubeapiserver/authenticator/config.go
@@ -140,7 +140,7 @@ func (config Config) New(serverLifecycle context.Context) (authenticator.Request
 		tokenAuthenticators = append(tokenAuthenticators, serviceAccountAuth)
 	}
 	if len(config.ServiceAccountIssuers) > 0 && config.ServiceAccountPublicKeysGetter != nil {
-		serviceAccountAuth, err := newServiceAccountAuthenticator(config.ServiceAccountIssuers, config.ServiceAccountPublicKeysGetter, config.APIAudiences, config.ServiceAccountTokenGetter)
+		serviceAccountAuth, err := newServiceAccountAuthenticator(config.ServiceAccountIssuers, config.ServiceAccountPublicKeysGetter, config.APIAudiences, config.ServiceAccountTokenGetter, config.ServiceAccountLookup)
 		if err != nil {
 			return nil, nil, nil, nil, err
 		}
@@ -351,11 +351,11 @@ func newLegacyServiceAccountAuthenticator(publicKeysGetter serviceaccount.Public
 }
 
 // newServiceAccountAuthenticator returns an authenticator.Token or an error
-func newServiceAccountAuthenticator(issuers []string, publicKeysGetter serviceaccount.PublicKeysGetter, apiAudiences authenticator.Audiences, serviceAccountGetter serviceaccount.ServiceAccountTokenClusterGetter) (authenticator.Token, error) {
+func newServiceAccountAuthenticator(issuers []string, publicKeysGetter serviceaccount.PublicKeysGetter, apiAudiences authenticator.Audiences, serviceAccountGetter serviceaccount.ServiceAccountTokenGetter, lookup bool) (authenticator.Token, error) {
 	if publicKeysGetter == nil {
 		return nil, fmt.Errorf("no public key getter provided")
 	}
-	tokenAuthenticator := serviceaccount.JWTTokenAuthenticator(issuers, publicKeysGetter, apiAudiences, serviceaccount.NewValidator(serviceAccountGetter))
+	tokenAuthenticator := serviceaccount.JWTTokenAuthenticator(issuers, publicKeysGetter, apiAudiences, serviceaccount.NewValidator(lookup, serviceAccountGetter))
 	return tokenAuthenticator, nil
 }
 
diff --git a/pkg/serviceaccount/claims.go b/pkg/serviceaccount/claims.go
index 8ff9e35b854b1..7b5f34b3f2287 100644
--- a/pkg/serviceaccount/claims.go
+++ b/pkg/serviceaccount/claims.go
@@ -133,14 +133,16 @@ func Claims(sa core.ServiceAccount, pod *core.Pod, secret *core.Secret, node *co
 	return sc, pc, nil
 }
 
-func NewValidator(getter ServiceAccountTokenClusterGetter) Validator[privateClaims] {
+func NewValidator(lookup bool, getter ServiceAccountTokenGetter) Validator[privateClaims] {
 	return &validator{
+		lookup: lookup,
 		getter: getter,
 	}
 }
 
 type validator struct {
-	getter ServiceAccountTokenClusterGetter
+	lookup bool
+	getter ServiceAccountTokenGetter
 }
 
 var _ = Validator[privateClaims](&validator{})
@@ -181,7 +183,8 @@ func (v *validator) Validate(ctx context.Context, _ string, public *jwt.Claims,
 	podref := private.Kubernetes.Pod
 	noderef := private.Kubernetes.Node
 	secref := private.Kubernetes.Secret
-	// Make sure service account still exists (name and UID)
+if v.lookup {	
+// Make sure service account still exists (name and UID)
 	serviceAccount, err := v.getter.Cluster(clusterName).GetServiceAccount(namespace, saref.Name)
 	if err != nil {
 		klog.V(4).Infof("Could not retrieve service account %s/%s: %v", namespace, saref.Name, err)
@@ -197,20 +200,21 @@ func (v *validator) Validate(ctx context.Context, _ string, public *jwt.Claims,
 		return nil, fmt.Errorf("service account %s/%s has been deleted", namespace, saref.Name)
 	}
 
-	if secref != nil {
+	if v.lookup && secref != nil {
 		// Make sure token hasn't been invalidated by deletion of the secret
 		secret, err := v.getter.Cluster(clusterName).GetSecret(namespace, secref.Name)
 		if err != nil {
-			klog.V(4).Infof("Could not retrieve bound secret %s/%s for service account %s/%s: %v", namespace, secref.Name, namespace, saref.Name, err)
-			return nil, errors.New("service account token has been invalidated")
+			klog.V(4).Infof("Could not retrieve service account %s/%s: %v", namespace, saref.Name, err)
+			return nil, err
 		}
-		if secref.UID != string(secret.UID) {
-			klog.V(4).Infof("Secret UID no longer matches %s/%s: %q != %q", namespace, secref.Name, string(secret.UID), secref.UID)
-			return nil, fmt.Errorf("secret UID (%s) does not match service account secret ref claim (%s)", secret.UID, secref.UID)
+
+		if string(serviceAccount.UID) != saref.UID {
+			klog.V(4).Infof("Service account UID no longer matches %s/%s: %q != %q", namespace, saref.Name, string(serviceAccount.UID), saref.UID)
+			return nil, fmt.Errorf("service account UID (%s) does not match claim (%s)", serviceAccount.UID, saref.UID)
 		}
-		if secret.DeletionTimestamp != nil && secret.DeletionTimestamp.Time.Before(invalidIfDeletedBefore) {
-			klog.V(4).Infof("Bound secret is deleted and awaiting removal: %s/%s for service account %s/%s", namespace, secref.Name, namespace, saref.Name)
-			return nil, errors.New("service account token has been invalidated")
+		if serviceAccount.DeletionTimestamp != nil && serviceAccount.DeletionTimestamp.Time.Before(invalidIfDeletedBefore) {
+			klog.V(4).Infof("Service account has been deleted %s/%s", namespace, saref.Name)
+			return nil, fmt.Errorf("service account %s/%s has been deleted", namespace, saref.Name)
 		}
 	}
 
diff --git a/pkg/serviceaccount/claims_test.go b/pkg/serviceaccount/claims_test.go
index 3cc869f4ff01c..2d5f090b5682d 100644
--- a/pkg/serviceaccount/claims_test.go
+++ b/pkg/serviceaccount/claims_test.go
@@ -346,12 +346,13 @@ type deletionTestCase struct {
 }
 
 type claimTestCase struct {
-	name      string
-	getter    ServiceAccountTokenGetter
-	private   *privateClaims
-	expiry    jwt.NumericDate
-	notBefore jwt.NumericDate
-	expectErr string
+	name          string
+	getter        ServiceAccountTokenGetter
+	private       *privateClaims
+	disableLookup bool
+	expiry        jwt.NumericDate
+	notBefore     jwt.NumericDate
+	expectErr     string
 }
 
 func TestValidatePrivateClaims(t *testing.T) {
@@ -455,6 +456,12 @@ func TestValidatePrivateClaims(t *testing.T) {
 			private:   &privateClaims{Kubernetes: kubernetes{Svcacct: ref{Name: "saname", UID: "sauid"}, Pod: &ref{Name: "podname", UID: "poduidold"}, Namespace: "ns"}},
 			expectErr: "pod UID (poduid) does not match service account pod ref claim (poduidold)",
 		},
+		{
+			name:          "disabled lookup",
+			getter:        fakeGetter{serviceAccount, nil, nil, nil},
+			private:       &privateClaims{Kubernetes: kubernetes{Svcacct: ref{Name: "not-here"}, Namespace: "ns"}},
+			disableLookup: true,
+		},
 	}
 
 	for _, deletionTestCase := range deletionTestCases {
@@ -505,7 +512,10 @@ func TestValidatePrivateClaims(t *testing.T) {
 
 	for _, tc := range testcases {
 		t.Run(tc.name, func(t *testing.T) {
-			v := &validator{getter: tc.getter}
+			v := &validator{
+				lookup: !tc.disableLookup,
+				getter: tc.getter,
+			}
 			expiry := jwt.NumericDate(nowUnix)
 			if tc.expiry != 0 {
 				expiry = tc.expiry