Fix fallback tokens (#242)

Author: brendan-kellam

.env.development

@@ -61,9 +61,9 @@ SOURCEBOT_LOG_LEVEL="debug" # valid values: info, debug, warn, error
 SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
 
 # Code-host fallback tokens
-# FALLBACK_GITHUB_TOKEN=""
-# FALLBACK_GITLAB_TOKEN=""
-# FALLBACK_GITEA_TOKEN=""
+# FALLBACK_GITHUB_CLOUD_TOKEN=""
+# FALLBACK_GITLAB_CLOUD_TOKEN=""
+# FALLBACK_GITEA_CLOUD_TOKEN=""
 
 # Controls the number of concurrent indexing jobs that can run at once
 # INDEX_CONCURRENCY_MULTIPLE=

packages/backend/src/env.ts

@@ -29,9 +29,9 @@ export const env = createEnv({
 
         POSTHOG_PAPIK: z.string().optional(),
 
-        FALLBACK_GITHUB_TOKEN: z.string().optional(),
-        FALLBACK_GITLAB_TOKEN: z.string().optional(),
-        FALLBACK_GITEA_TOKEN: z.string().optional(),
+        FALLBACK_GITHUB_CLOUD_TOKEN: z.string().optional(),
+        FALLBACK_GITLAB_CLOUD_TOKEN: z.string().optional(),
+        FALLBACK_GITEA_CLOUD_TOKEN: z.string().optional(),
 
         REDIS_URL: z.string().url().default("redis://localhost:6379"),
 

packages/backend/src/gitea.ts

@@ -10,9 +10,18 @@ import * as Sentry from "@sentry/node";
 import { env } from './env.js';
 
 const logger = createLogger('Gitea');
+const GITEA_CLOUD_HOSTNAME = "gitea.com";
 
 export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => {
-    const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITEA_TOKEN;
+    const hostname = config.url ?
+        new URL(config.url).hostname :
+        GITEA_CLOUD_HOSTNAME;
+
+    const token = config.token ?
+        await getTokenFromConfig(config.token, orgId, db, logger) :
+        hostname === GITEA_CLOUD_HOSTNAME ?
+        env.FALLBACK_GITEA_CLOUD_TOKEN :
+        undefined;
 
     const api = giteaApi(config.url ?? 'https://gitea.com', {
         token: token,

packages/backend/src/github.ts

@@ -10,6 +10,7 @@ import * as Sentry from "@sentry/node";
 import { env } from "./env.js";
 
 const logger = createLogger("GitHub");
+const GITHUB_CLOUD_HOSTNAME = "github.com";
 
 export type OctokitRepository = {
     name: string,
@@ -40,7 +41,15 @@ const isHttpError = (error: unknown, status: number): boolean => {
 }
 
 export const getGitHubReposFromConfig = async (config: GithubConnectionConfig, orgId: number, db: PrismaClient, signal: AbortSignal) => {
-    const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITHUB_TOKEN;
+    const hostname = config.url ?
+        new URL(config.url).hostname :
+        GITHUB_CLOUD_HOSTNAME;
+
+    const token = config.token ?
+        await getTokenFromConfig(config.token, orgId, db, logger) :
+        hostname === GITHUB_CLOUD_HOSTNAME ?
+        env.FALLBACK_GITHUB_CLOUD_TOKEN :
+        undefined;
 
     const octokit = new Octokit({
         auth: token,

packages/backend/src/gitlab.ts

@@ -12,7 +12,15 @@ const logger = createLogger("GitLab");
 export const GITLAB_CLOUD_HOSTNAME = "gitlab.com";
 
 export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, orgId: number, db: PrismaClient) => {
-    const token = config.token ? await getTokenFromConfig(config.token, orgId, db, logger) : env.FALLBACK_GITLAB_TOKEN;
+    const hostname = config.url ?
+        new URL(config.url).hostname :
+        GITLAB_CLOUD_HOSTNAME;
+
+    const token = config.token ?
+        await getTokenFromConfig(config.token, orgId, db, logger) :
+        hostname === GITLAB_CLOUD_HOSTNAME ?
+        env.FALLBACK_GITLAB_CLOUD_TOKEN :
+        undefined;
 
     const api = new Gitlab({
         ...(token ? {
@@ -22,7 +30,6 @@ export const getGitLabReposFromConfig = async (config: GitlabConnectionConfig, o
             host: config.url,
         } : {}),
     });
-    const hostname = config.url ? new URL(config.url).hostname : GITLAB_CLOUD_HOSTNAME;
 
     let allRepos: ProjectSchema[] = [];
     let notFound: {