fix(backend): Support storing multiple local keys in JWK cache (#6993)

Author: nikosdouvlis

.changeset/fix-multi-instance-jwt-caching.md

@@ -0,0 +1,14 @@
+---
+'@clerk/backend': patch
+'@clerk/shared': patch
+---
+
+Fixed JWT public key caching in `verifyToken()` to support multi-instance scenarios. Public keys are now correctly cached per `kid` from the token header instead of using a single shared cache key.
+
+**What was broken:**
+
+When verifying JWT tokens with the `jwtKey` option (PEM public key), all keys were cached under the same cache key. This caused verification failures in multi-instance scenarios.
+
+**What's fixed:**
+
+JWT public keys are now cached using the `kid` value from each token's header.

packages/backend/src/fixtures/index.ts

@@ -52,7 +52,7 @@ export const mockPEMKey =
   'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA8Z1oLQbaYkakUSIYRvjmOoeXMDFFjynGP2+gVy0mQJHYgVhgo34RsQgZoz7rSNm/EOL+l/mHTqQAhwaf9Ef8X5vsPX8vP3RNRRm3XYpbIGbOcANJaHihJZwnzG9zIGYF8ki+m55zftO7pkOoXDtIqCt+5nIUQjGJK5axFELrnWaz2qcR03A7rYKQc3F1gut2Ru1xfmiJVUlQe0tLevQO/FzfYpWu7+691q+ZRUGxWvGc0ays4ACa7JXElCIKXRv/yb3Vc1iry77HRAQ28J7Fqpj5Cb+sxfFI+Vhf1GB1bNeOLPR10nkSMJ74HB0heHi/SsM83JiGekv0CpZPCC8jcQIDAQAB';
 
 export const mockPEMJwk = {
-  kid: 'local',
+  kid: 'local-test-kid',
   kty: 'RSA',
   alg: 'RS256',
   n: '8Z1oLQbaYkakUSIYRvjmOoeXMDFFjynGP2-gVy0mQJHYgVhgo34RsQgZoz7rSNm_EOL-l_mHTqQAhwaf9Ef8X5vsPX8vP3RNRRm3XYpbIGbOcANJaHihJZwnzG9zIGYF8ki-m55zftO7pkOoXDtIqCt-5nIUQjGJK5axFELrnWaz2qcR03A7rYKQc3F1gut2Ru1xfmiJVUlQe0tLevQO_FzfYpWu7-691q-ZRUGxWvGc0ays4ACa7JXElCIKXRv_yb3Vc1iry77HRAQ28J7Fqpj5Cb-sxfFI-Vhf1GB1bNeOLPR10nkSMJ74HB0heHi_SsM83JiGekv0CpZPCC8jcQ',

packages/backend/src/tokens/__tests__/keys.test.ts

@@ -12,11 +12,13 @@ import {
   mockRsaJwkKid,
 } from '../../fixtures';
 import { server, validateHeaders } from '../../mock-server';
-import { loadClerkJWKFromLocal, loadClerkJWKFromRemote } from '../keys';
+import { loadClerkJwkFromPem, loadClerkJWKFromRemote } from '../keys';
+
+const MOCK_KID = 'test-kid';
 
 describe('tokens.loadClerkJWKFromLocal(localKey)', () => {
   it('throws an error if no key has been provided', () => {
-    expect(() => loadClerkJWKFromLocal()).toThrow(
+    expect(() => loadClerkJwkFromPem({ kid: MOCK_KID })).toThrow(
       new TokenVerificationError({
         action: TokenVerificationErrorAction.SetClerkJWTKey,
         message: 'Missing local JWK.',
@@ -26,14 +28,57 @@ describe('tokens.loadClerkJWKFromLocal(localKey)', () => {
   });
 
   it('loads the local key', () => {
-    const jwk = loadClerkJWKFromLocal(mockPEMKey);
+    const jwk = loadClerkJwkFromPem({ kid: MOCK_KID, pem: mockPEMKey });
     expect(jwk).toMatchObject(mockPEMJwk);
   });
 
   it('loads the local key in PEM format', () => {
-    const jwk = loadClerkJWKFromLocal(mockPEMJwtKey);
+    const jwk = loadClerkJwkFromPem({ kid: MOCK_KID, pem: mockPEMJwtKey });
     expect(jwk).toMatchObject(mockPEMJwk);
   });
+
+  it('caches PEM keys separately for different kids', () => {
+    const jwk1 = loadClerkJwkFromPem({ kid: 'ins_1', pem: mockPEMKey }) as JsonWebKey & { kid: string };
+    expect(jwk1.kid).toBe('local-ins_1');
+    expect(jwk1.n).toBe(mockPEMJwk.n);
+
+    const jwk2 = loadClerkJwkFromPem({ kid: 'ins_2', pem: mockPEMJwtKey }) as JsonWebKey & { kid: string };
+    expect(jwk2.kid).toBe('local-ins_2');
+    expect(jwk2.n).toBe(mockPEMJwk.n);
+
+    // Verify both are cached independently
+    const jwk1Cached = loadClerkJwkFromPem({ kid: 'ins_1', pem: mockPEMKey });
+    const jwk2Cached = loadClerkJwkFromPem({ kid: 'ins_2', pem: mockPEMJwtKey });
+
+    expect(jwk1Cached).toBe(jwk1);
+    expect(jwk2Cached).toBe(jwk2); // Same object reference means its cached
+  });
+
+  it('returns cached JWK on subsequent calls with same kid', () => {
+    const jwk1 = loadClerkJwkFromPem({ kid: 'cache-test', pem: mockPEMKey });
+    const jwk2 = loadClerkJwkFromPem({ kid: 'cache-test', pem: mockPEMKey });
+    // Should return the exact same reference
+    expect(jwk1).toBe(jwk2);
+  });
+
+  it('uses "local-" prefix to avoid cache collision with remote keys', () => {
+    const localJwk = loadClerkJwkFromPem({ kid: 'test-kid', pem: mockPEMKey }) as JsonWebKey & { kid: string };
+    expect(localJwk.kid).toBe('local-test-kid');
+  });
+
+  it('creates separate cache entries for different kids even with same PEM', () => {
+    // Two JWT keys might theoretically use the same PEM (unlikely but possible)
+    const jwkA = loadClerkJwkFromPem({ kid: 'ins_key_a', pem: mockPEMKey }) as JsonWebKey & { kid: string };
+    const jwkB = loadClerkJwkFromPem({ kid: 'ins_key_b', pem: mockPEMKey }) as JsonWebKey & { kid: string };
+
+    // They should be different objects
+    expect(jwkA).not.toBe(jwkB);
+    // But have the same modulus
+    expect(jwkA.n).toBe(jwkB.n);
+    // And different prefixed kids
+    expect(jwkA.kid).toBe('local-ins_key_a');
+    expect(jwkB.kid).toBe('local-ins_key_b');
+  });
 });
 
 describe('tokens.loadClerkJWKFromRemote(options)', () => {

packages/backend/src/tokens/handshake.ts

@@ -7,7 +7,7 @@ import type { AuthenticateContext } from './authenticateContext';
 import type { SignedInState, SignedOutState } from './authStatus';
 import { AuthErrorReason, signedIn, signedOut } from './authStatus';
 import { getCookieName, getCookieValue } from './cookie';
-import { loadClerkJWKFromLocal, loadClerkJWKFromRemote } from './keys';
+import { loadClerkJwkFromPem, loadClerkJWKFromRemote } from './keys';
 import type { OrganizationMatcher } from './organizationMatcher';
 import { TokenType } from './tokenTypes';
 import type { OrganizationSyncOptions, OrganizationSyncTarget } from './types';
@@ -66,7 +66,7 @@ export async function verifyHandshakeToken(
   let key;
 
   if (jwtKey) {
-    key = loadClerkJWKFromLocal(jwtKey);
+    key = loadClerkJwkFromPem({ kid, pem: jwtKey });
   } else if (secretKey) {
     // Fetch JWKS from Backend API using the key
     key = await loadClerkJWKFromRemote({ secretKey, apiUrl, apiVersion, kid, jwksCacheTtlInMs, skipJwksCache });
@@ -78,9 +78,7 @@ export async function verifyHandshakeToken(
     });
   }
 
-  return await verifyHandshakeJwt(token, {
-    key,
-  });
+  return verifyHandshakeJwt(token, { key });
 }
 
 export class HandshakeService {

packages/backend/src/tokens/keys.ts

@@ -30,58 +30,59 @@ function getCacheValues() {
   return Object.values(cache);
 }
 
-function setInCache(jwk: JsonWebKeyWithKid, shouldExpire = true) {
-  cache[jwk.kid] = jwk;
+function setInCache(cacheKey: string, jwk: JsonWebKeyWithKid, shouldExpire = true) {
+  cache[cacheKey] = jwk;
   lastUpdatedAt = shouldExpire ? Date.now() : -1;
 }
 
-const LocalJwkKid = 'local';
 const PEM_HEADER = '-----BEGIN PUBLIC KEY-----';
 const PEM_TRAILER = '-----END PUBLIC KEY-----';
 const RSA_PREFIX = 'MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA';
 const RSA_SUFFIX = 'IDAQAB';
 
+type LoadClerkJwkFromPemOptions = {
+  kid: string;
+  pem?: string;
+};
+
 /**
- *
  * Loads a local PEM key usually from process.env and transform it to JsonWebKey format.
- * The result is also cached on the module level to avoid unnecessary computations in subsequent invocations.
- *
- * @param {string} localKey
- * @returns {JsonWebKey} key
+ * The result is cached on the module level to avoid unnecessary computations in subsequent invocations.
  */
-export function loadClerkJWKFromLocal(localKey?: string): JsonWebKey {
-  if (!getFromCache(LocalJwkKid)) {
-    if (!localKey) {
-      throw new TokenVerificationError({
-        action: TokenVerificationErrorAction.SetClerkJWTKey,
-        message: 'Missing local JWK.',
-        reason: TokenVerificationErrorReason.LocalJWKMissing,
-      });
-    }
+export function loadClerkJwkFromPem(params: LoadClerkJwkFromPemOptions): JsonWebKey {
+  const { kid, pem } = params;
+
+  // We use a cache key that includes the local prefix in order to avoid
+  // cache conflicts when loadClerkJwkFromPem and loadClerkJWKFromRemote
+  // are called with the same kid
+  const prefixedKid = `local-${kid}`;
+  const cachedJwk = getFromCache(prefixedKid);
+
+  if (cachedJwk) {
+    return cachedJwk;
+  }
 
-    const modulus = localKey
-      .replace(/\r\n|\n|\r/g, '')
-      .replace(PEM_HEADER, '')
-      .replace(PEM_TRAILER, '')
-      .replace(RSA_PREFIX, '')
-      .replace(RSA_SUFFIX, '')
-      .replace(/\+/g, '-')
-      .replace(/\//g, '_');
-
-    // JWK https://datatracker.ietf.org/doc/html/rfc7517
-    setInCache(
-      {
-        kid: 'local',
-        kty: 'RSA',
-        alg: 'RS256',
-        n: modulus,
-        e: 'AQAB',
-      },
-      false, // local key never expires in cache
-    );
+  if (!pem) {
+    throw new TokenVerificationError({
+      action: TokenVerificationErrorAction.SetClerkJWTKey,
+      message: 'Missing local JWK.',
+      reason: TokenVerificationErrorReason.LocalJWKMissing,
+    });
   }
 
-  return getFromCache(LocalJwkKid);
+  const modulus = pem
+    .replace(/\r\n|\n|\r/g, '')
+    .replace(PEM_HEADER, '')
+    .replace(PEM_TRAILER, '')
+    .replace(RSA_PREFIX, '')
+    .replace(RSA_SUFFIX, '')
+    .replace(/\+/g, '-')
+    .replace(/\//g, '_');
+
+  // https://datatracker.ietf.org/doc/html/rfc7517
+  const jwk = { kid: prefixedKid, kty: 'RSA', alg: 'RS256', n: modulus, e: 'AQAB' };
+  setInCache(prefixedKid, jwk, false); // local key never expires in cache
+  return jwk;
 }
 
 /**
@@ -127,13 +128,9 @@ export type LoadClerkJWKFromRemoteOptions = {
  * @param {string} options.alg - The algorithm of the JWT
  * @returns {JsonWebKey} key
  */
-export async function loadClerkJWKFromRemote({
-  secretKey,
-  apiUrl = API_URL,
-  apiVersion = API_VERSION,
-  kid,
-  skipJwksCache,
-}: LoadClerkJWKFromRemoteOptions): Promise<JsonWebKey> {
+export async function loadClerkJWKFromRemote(params: LoadClerkJWKFromRemoteOptions): Promise<JsonWebKey> {
+  const { secretKey, apiUrl = API_URL, apiVersion = API_VERSION, kid, skipJwksCache } = params;
+
   if (skipJwksCache || cacheHasExpired() || !getFromCache(kid)) {
     if (!secretKey) {
       throw new TokenVerificationError({
@@ -153,7 +150,7 @@ export async function loadClerkJWKFromRemote({
       });
     }
 
-    keys.forEach(key => setInCache(key));
+    keys.forEach(key => setInCache(key.kid, key));
   }
 
   const jwk = getFromCache(kid);

packages/backend/src/tokens/verify.ts

@@ -1,4 +1,5 @@
 import { isClerkAPIResponseError } from '@clerk/shared/error';
+import type { Simplify } from '@clerk/shared/types';
 import type { JwtPayload } from '@clerk/types';
 
 import type { APIKey, IdPOAuthAccessToken, M2MToken } from '../api';
@@ -14,21 +15,23 @@ import type { VerifyJwtOptions } from '../jwt';
 import type { JwtReturnType, MachineTokenReturnType } from '../jwt/types';
 import { decodeJwt, verifyJwt } from '../jwt/verifyJwt';
 import type { LoadClerkJWKFromRemoteOptions } from './keys';
-import { loadClerkJWKFromLocal, loadClerkJWKFromRemote } from './keys';
+import { loadClerkJwkFromPem, loadClerkJWKFromRemote } from './keys';
 import { API_KEY_PREFIX, M2M_TOKEN_PREFIX, OAUTH_TOKEN_PREFIX } from './machine';
 import type { MachineTokenType } from './tokenTypes';
 import { TokenType } from './tokenTypes';
 
 /**
  * @interface
  */
-export type VerifyTokenOptions = Omit<VerifyJwtOptions, 'key'> &
-  Omit<LoadClerkJWKFromRemoteOptions, 'kid'> & {
-    /**
-     * Used to verify the session token in a networkless manner. Supply the PEM public key from the **[**API keys**](https://dashboard.clerk.com/last-active?path=api-keys) page -> Show JWT public key -> PEM Public Key** section in the Clerk Dashboard. **It's recommended to use [the environment variable](https://clerk.com/docs/guides/development/clerk-environment-variables) instead.** For more information, refer to [Manual JWT verification](https://clerk.com/docs/guides/sessions/manual-jwt-verification).
-     */
-    jwtKey?: string;
-  };
+export type VerifyTokenOptions = Simplify<
+  Omit<VerifyJwtOptions, 'key'> &
+    Omit<LoadClerkJWKFromRemoteOptions, 'kid'> & {
+      /**
+       * Used to verify the session token in a networkless manner. Supply the PEM public key from the **[**API keys**](https://dashboard.clerk.com/last-active?path=api-keys) page -> Show JWT public key -> PEM Public Key** section in the Clerk Dashboard. **It's recommended to use [the environment variable](https://clerk.com/docs/guides/development/clerk-environment-variables) instead.** For more information, refer to [Manual JWT verification](https://clerk.com/docs/guides/sessions/manual-jwt-verification).
+       */
+      jwtKey?: string;
+    }
+>;
 
 /**
  * > [!WARNING]
@@ -121,10 +124,10 @@ export async function verifyToken(
   const { kid } = header;
 
   try {
-    let key;
+    let key: JsonWebKey;
 
     if (options.jwtKey) {
-      key = loadClerkJWKFromLocal(options.jwtKey);
+      key = loadClerkJwkFromPem({ kid, pem: options.jwtKey });
     } else if (options.secretKey) {
       // Fetch JWKS from Backend API using the key
       key = await loadClerkJWKFromRemote({ ...options, kid });

packages/shared/src/types/index.ts

@@ -1 +1 @@
-export type _unstable_mock_type = any;
+export type { Simplify } from './utils';

packages/shared/src/types/utils.ts

@@ -0,0 +1,7 @@
+/**
+ * Useful to flatten the type output to improve type hints shown in editors. And also to transform an interface into a type to aide with assignability.
+ * https://github.com/sindresorhus/type-fest/blob/main/source/simplify.d.ts
+ */
+export type Simplify<T> = {
+  [K in keyof T]: T[K];
+} & {};