wip

brendan-kellam · brendan-kellam · commit 36386662487a · 2025-09-19T15:45:17.000-07:00
diff --git a/packages/backend/src/permissionSyncer.ts b/packages/backend/src/permissionSyncer.ts
@@ -1,11 +1,13 @@
-import { PrismaClient } from "@sourcebot/db";
+import * as Sentry from "@sentry/node";
+import { PrismaClient, Repo, RepoPermissionSyncStatus } from "@sourcebot/db";
 import { createLogger } from "@sourcebot/logger";
 import { BitbucketConnectionConfig } from "@sourcebot/schemas/v3/bitbucket.type";
 import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type";
 import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type";
 import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type";
 import { Job, Queue, Worker } from 'bullmq';
 import { Redis } from 'ioredis';
+import { env } from "./env.js";
 import { createOctokitFromConfig, getUserIdsWithReadAccessToRepo } from "./github.js";
 import { RepoWithConnections } from "./types.js";
 
@@ -17,6 +19,8 @@ const QUEUE_NAME = 'repoPermissionSyncQueue';
 
 const logger = createLogger('permission-syncer');
 
+const SUPPORTED_CODE_HOST_TYPES = ['github'];
+
 export class RepoPermissionSyncer {
     private queue: Queue<RepoPermissionSyncJob>;
     private worker: Worker<RepoPermissionSyncJob>;
@@ -30,48 +34,94 @@ export class RepoPermissionSyncer {
         });
         this.worker = new Worker<RepoPermissionSyncJob>(QUEUE_NAME, this.runJob.bind(this), {
             connection: redis,
+            concurrency: 1,
         });
         this.worker.on('completed', this.onJobCompleted.bind(this));
         this.worker.on('failed', this.onJobFailed.bind(this));
     }
 
-    public async scheduleJob(repoId: number) {
-        await this.queue.add(QUEUE_NAME, {
-            repoId,
-        });
-    }
-
     public startScheduler() {
         logger.debug('Starting scheduler');
 
-        // @todo: we should only sync permissions for a repository if it has been at least ~24 hours since the last sync.
         return setInterval(async () => {
+            // @todo: make this configurable
+            const thresholdDate = new Date(Date.now() - 1000 * 60 * 60 * 24);
             const repos = await this.db.repo.findMany({
+                // Repos need their permissions to be synced against the code host when...
                 where: {
-                    external_codeHostType: {
-                        in: ['github'],
-                    }
+                    // They belong to a code host that supports permissions syncing
+                    AND: [
+                        {
+                            external_codeHostType: {
+                                in: SUPPORTED_CODE_HOST_TYPES,
+                            }
+                        },
+                        // and, they either require a sync (SYNC_NEEDED) or have been in a completed state (SYNCED or FAILED)
+                        // for > some duration (default 24 hours)
+                        {
+                            OR: [
+                                {
+                                    permissionSyncStatus: RepoPermissionSyncStatus.SYNC_NEEDED
+                                },
+                                {
+                                    AND: [
+                                        {
+                                            OR: [
+                                                { permissionSyncStatus: RepoPermissionSyncStatus.SYNCED },
+                                                { permissionSyncStatus: RepoPermissionSyncStatus.FAILED },
+                                            ]
+                                        },
+                                        {
+                                            OR: [
+                                                { permissionSyncJobLastCompletedAt: null },
+                                                { permissionSyncJobLastCompletedAt: { lt: thresholdDate } }
+                                            ]
+                                        }
+                                    ]
+                                }
+                            ]
+                        },
+                    ]
                 }
             });
 
-            for (const repo of repos) {
-                await this.scheduleJob(repo.id);
-            }
-
-        // @todo: make this configurable
-        }, 1000 * 60);
+            await this.schedulePermissionSync(repos);
+        }, 1000 * 30);
     }
 
     public dispose() {
         this.worker.close();
         this.queue.close();
     }
 
+    private async schedulePermissionSync(repos: Repo[]) {
+        await this.db.$transaction(async (tx) => {
+            await tx.repo.updateMany({
+                where: { id: { in: repos.map(repo => repo.id) } },
+                data: { permissionSyncStatus: RepoPermissionSyncStatus.IN_SYNC_QUEUE },
+            });
+
+            await this.queue.addBulk(repos.map(repo => ({
+                name: 'repoPermissionSyncJob',
+                data: {
+                    repoId: repo.id,
+                },
+                opts: {
+                    removeOnComplete: env.REDIS_REMOVE_ON_COMPLETE,
+                    removeOnFail: env.REDIS_REMOVE_ON_FAIL,
+                }
+            })))
+        });
+    }
+
     private async runJob(job: Job<RepoPermissionSyncJob>) {
         const id = job.data.repoId;
-        const repo = await this.db.repo.findUnique({
+        const repo = await this.db.repo.update({
             where: {
-                id,
+                id
+            },
+            data: {
+                permissionSyncStatus: RepoPermissionSyncStatus.SYNCING,
             },
             include: {
                 connections: {
@@ -86,6 +136,8 @@ export class RepoPermissionSyncer {
             throw new Error(`Repo ${id} not found`);
         }
 
+        logger.info(`Syncing permissions for repo ${repo.displayName}...`);
+
         const connection = getFirstConnectionWithToken(repo);
         if (!connection) {
             throw new Error(`No connection with token found for repo ${id}`);
@@ -119,8 +171,6 @@ export class RepoPermissionSyncer {
             return [];
         })();
 
-        logger.info(`User IDs with read access to repo ${id}: ${userIds}`);
-
         await this.db.repo.update({
             where: {
                 id: repo.id,
@@ -141,11 +191,43 @@ export class RepoPermissionSyncer {
     }
 
     private async onJobCompleted(job: Job<RepoPermissionSyncJob>) {
-        logger.info(`Repo permission sync job completed for repo ${job.data.repoId}`);
+        const repo = await this.db.repo.update({
+            where: {
+                id: job.data.repoId,
+            },
+            data: {
+                permissionSyncStatus: RepoPermissionSyncStatus.SYNCED,
+                permissionSyncJobLastCompletedAt: new Date(),
+            },
+        });
+
+        logger.info(`Permissions synced for repo ${repo.displayName ?? repo.name}`);
     }
 
     private async onJobFailed(job: Job<RepoPermissionSyncJob> | undefined, err: Error) {
-        logger.error(`Repo permission sync job failed for repo ${job?.data.repoId}: ${err}`);
+        Sentry.captureException(err, {
+            tags: {
+                repoId: job?.data.repoId,
+                queue: QUEUE_NAME,
+            }
+        });
+
+        const errorMessage = (repoName: string) => `Repo permission sync job failed for repo ${repoName}: ${err}`;
+
+        if (job) {
+            const repo = await this.db.repo.update({
+                where: {
+                    id: job?.data.repoId,
+                },
+                data: {
+                    permissionSyncStatus: RepoPermissionSyncStatus.FAILED,
+                    permissionSyncJobLastCompletedAt: new Date(),
+                },
+            });
+            logger.error(errorMessage(repo.displayName ?? repo.name));
+        } else {
+            logger.error(errorMessage('unknown repo (id not found)'));
+        }
     }
 }
 
diff --git a/packages/db/prisma/migrations/20250918030650_add_permission_sync_tracking_to_repo_table/migration.sql b/packages/db/prisma/migrations/20250918030650_add_permission_sync_tracking_to_repo_table/migration.sql
@@ -0,0 +1,6 @@
+-- CreateEnum
+CREATE TYPE "RepoPermissionSyncStatus" AS ENUM ('SYNC_NEEDED', 'IN_SYNC_QUEUE', 'SYNCING', 'SYNCED', 'FAILED');
+
+-- AlterTable
+ALTER TABLE "Repo" ADD COLUMN     "permissionSyncJobLastCompletedAt" TIMESTAMP(3),
+ADD COLUMN     "permissionSyncStatus" "RepoPermissionSyncStatus" NOT NULL DEFAULT 'SYNC_NEEDED';
diff --git a/packages/db/prisma/schema.prisma b/packages/db/prisma/schema.prisma
@@ -30,6 +30,14 @@ enum ConnectionSyncStatus {
   FAILED
 }
 
+enum RepoPermissionSyncStatus {
+  SYNC_NEEDED
+  IN_SYNC_QUEUE
+  SYNCING
+  SYNCED
+  FAILED
+}
+
 enum StripeSubscriptionStatus {
   ACTIVE
   INACTIVE
@@ -59,6 +67,10 @@ model Repo {
   repoIndexingStatus RepoIndexingStatus     @default(NEW)
   permittedUsers     UserToRepoPermission[]
 
+  permissionSyncStatus RepoPermissionSyncStatus @default(SYNC_NEEDED)
+  /// When the repo permissions were last synced, either successfully or unsuccessfully.
+  permissionSyncJobLastCompletedAt DateTime?
+
   // The id of the repo in the external service
   external_id           String
   // The type of the external service (e.g., github, gitlab, etc.)
diff --git a/packages/web/src/app/[domain]/browse/[...path]/components/codePreviewPanel.tsx b/packages/web/src/app/[domain]/browse/[...path]/components/codePreviewPanel.tsx
@@ -10,16 +10,15 @@ interface CodePreviewPanelProps {
     path: string;
     repoName: string;
     revisionName?: string;
-    domain: string;
 }
 
-export const CodePreviewPanel = async ({ path, repoName, revisionName, domain }: CodePreviewPanelProps) => {
+export const CodePreviewPanel = async ({ path, repoName, revisionName }: CodePreviewPanelProps) => {
     const [fileSourceResponse, repoInfoResponse] = await Promise.all([
         getFileSource({
             fileName: path,
             repository: repoName,
             branch: revisionName,
-        }, domain),
+        }),
         getRepoInfoByName(repoName),
     ]);
 
diff --git a/packages/web/src/app/[domain]/browse/[...path]/page.tsx b/packages/web/src/app/[domain]/browse/[...path]/page.tsx
@@ -7,7 +7,6 @@ import { TreePreviewPanel } from "./components/treePreviewPanel";
 interface BrowsePageProps {
     params: Promise<{
         path: string[];
-        domain: string;
     }>;
 }
 
@@ -16,7 +15,6 @@ export default async function BrowsePage(props: BrowsePageProps) {
 
     const {
         path: _rawPath,
-        domain
     } = params;
 
     const rawPath = _rawPath.join('/');
@@ -35,7 +33,6 @@ export default async function BrowsePage(props: BrowsePageProps) {
                         path={path}
                         repoName={repoName}
                         revisionName={revisionName}
-                        domain={domain}
                     />
                 ) : (
                     <TreePreviewPanel
diff --git a/packages/web/src/app/[domain]/search/components/codePreviewPanel/index.tsx b/packages/web/src/app/[domain]/search/components/codePreviewPanel/index.tsx
@@ -3,7 +3,6 @@
 import { useQuery } from "@tanstack/react-query";
 import { CodePreview } from "./codePreview";
 import { SearchResultFile } from "@/features/search/types";
-import { useDomain } from "@/hooks/useDomain";
 import { SymbolIcon } from "@radix-ui/react-icons";
 import { SetStateAction, Dispatch, useMemo } from "react";
 import { getFileSource } from "@/features/search/fileSourceApi";
@@ -22,7 +21,6 @@ export const CodePreviewPanel = ({
     onClose,
     onSelectedMatchIndexChange,
 }: CodePreviewPanelProps) => {
-    const domain = useDomain();
 
     // If there are multiple branches pointing to the same revision of this file, it doesn't
     // matter which branch we use here, so use the first one.
@@ -31,13 +29,13 @@ export const CodePreviewPanel = ({
     }, [previewedFile]);
 
     const { data: file, isLoading, isPending, isError } = useQuery({
-        queryKey: ["source", previewedFile, branch, domain],
+        queryKey: ["source", previewedFile, branch],
         queryFn: () => unwrapServiceError(
             getFileSource({
                 fileName: previewedFile.fileName.text,
                 repository: previewedFile.repository,
                 branch,
-            }, domain)
+            })
         ),
         select: (data) => {
             return {
diff --git a/packages/web/src/app/api/(server)/search/route.ts b/packages/web/src/app/api/(server)/search/route.ts
@@ -5,20 +5,8 @@ import { isServiceError } from "@/lib/utils";
 import { NextRequest } from "next/server";
 import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError";
 import { searchRequestSchema } from "@/features/search/schemas";
-import { ErrorCode } from "@/lib/errorCodes";
-import { StatusCodes } from "http-status-codes";
 
 export const POST = async (request: NextRequest) => {
-    const domain = request.headers.get("X-Org-Domain");
-    const apiKey = request.headers.get("X-Sourcebot-Api-Key") ?? undefined;
-    if (!domain) {
-        return serviceErrorResponse({
-            statusCode: StatusCodes.BAD_REQUEST,
-            errorCode: ErrorCode.MISSING_ORG_DOMAIN_HEADER,
-            message: "Missing X-Org-Domain header",
-        });
-    }
-
     const body = await request.json();
     const parsed = await searchRequestSchema.safeParseAsync(body);
     if (!parsed.success) {
@@ -27,7 +15,7 @@ export const POST = async (request: NextRequest) => {
         );
     }
     
-    const response = await search(parsed.data, domain, apiKey);
+    const response = await search(parsed.data);
     if (isServiceError(response)) {
         return serviceErrorResponse(response);
     }
diff --git a/packages/web/src/app/api/(server)/source/route.ts b/packages/web/src/app/api/(server)/source/route.ts
@@ -5,20 +5,8 @@ import { schemaValidationError, serviceErrorResponse } from "@/lib/serviceError"
 import { isServiceError } from "@/lib/utils";
 import { NextRequest } from "next/server";
 import { fileSourceRequestSchema } from "@/features/search/schemas";
-import { ErrorCode } from "@/lib/errorCodes";
-import { StatusCodes } from "http-status-codes";
 
 export const POST = async (request: NextRequest) => {
-    const domain = request.headers.get("X-Org-Domain");
-    const apiKey = request.headers.get("X-Sourcebot-Api-Key") ?? undefined;
-    if (!domain) {
-        return serviceErrorResponse({
-            statusCode: StatusCodes.BAD_REQUEST,
-            errorCode: ErrorCode.MISSING_ORG_DOMAIN_HEADER,
-            message: "Missing X-Org-Domain header",
-        });
-    }
-
     const body = await request.json();
     const parsed = await fileSourceRequestSchema.safeParseAsync(body);
     if (!parsed.success) {
@@ -27,7 +15,7 @@ export const POST = async (request: NextRequest) => {
         );
     }
     
-    const response = await getFileSource(parsed.data, domain, apiKey);
+    const response = await getFileSource(parsed.data);
     if (isServiceError(response)) {
         return serviceErrorResponse(response);
     }
diff --git a/packages/web/src/features/agents/review-agent/nodes/fetchFileContent.ts b/packages/web/src/features/agents/review-agent/nodes/fetchFileContent.ts
@@ -2,7 +2,6 @@ import { sourcebot_context, sourcebot_pr_payload } from "@/features/agents/revie
 import { getFileSource } from "@/features/search/fileSourceApi";
 import { fileSourceResponseSchema } from "@/features/search/schemas";
 import { isServiceError } from "@/lib/utils";
-import { env } from "@/env.mjs";
 import { createLogger } from "@sourcebot/logger";
 
 const logger = createLogger('fetch-file-content');
@@ -17,7 +16,7 @@ export const fetchFileContent = async (pr_payload: sourcebot_pr_payload, filenam
     }
     logger.debug(JSON.stringify(fileSourceRequest, null, 2));
 
-    const response = await getFileSource(fileSourceRequest, "~", env.REVIEW_AGENT_API_KEY);
+    const response = await getFileSource(fileSourceRequest);
     if (isServiceError(response)) {
         throw new Error(`Failed to fetch file content for ${filename} from ${repoPath}: ${response.message}`);
     }
diff --git a/packages/web/src/features/chat/agent.ts b/packages/web/src/features/chat/agent.ts
diff --git a/packages/web/src/features/chat/tools.ts b/packages/web/src/features/chat/tools.ts
diff --git a/packages/web/src/features/codeNav/actions.ts b/packages/web/src/features/codeNav/actions.ts
diff --git a/packages/web/src/features/search/fileSourceApi.ts b/packages/web/src/features/search/fileSourceApi.ts

Original file line number	Diff line number	Diff line change
`@@ -2,7 +2,6 @@ import { sourcebot_context, sourcebot_pr_payload } from "@/features/agents/revie`
`2`	`2`	`import { getFileSource } from "@/features/search/fileSourceApi";`
`3`	`3`	`import { fileSourceResponseSchema } from "@/features/search/schemas";`
`4`	`4`	`import { isServiceError } from "@/lib/utils";`
`5`		`-import { env } from "@/env.mjs";`
`6`	`5`	`import { createLogger } from "@sourcebot/logger";`
`7`	`6`
`8`	`7`	`const logger = createLogger('fetch-file-content');`
`@@ -17,7 +16,7 @@ export const fetchFileContent = async (pr_payload: sourcebot_pr_payload, filenam`
`17`	`16`	`}`
`18`	`17`	`logger.debug(JSON.stringify(fileSourceRequest, null, 2));`
`19`	`18`
`20`		`- const response = await getFileSource(fileSourceRequest, "~", env.REVIEW_AGENT_API_KEY);`
	`19`	`+ const response = await getFileSource(fileSourceRequest);`
`21`	`20`	`if (isServiceError(response)) {`
`22`	`21`	throw new Error(`Failed to fetch file content for ${filename} from ${repoPath}: ${response.message}`);
`23`	`22`	`}`