experiment: Self-serve repository indexing for public GitHub repositories (#468)

Author: brendan-kellam

packages/web/src/actions.ts

@@ -3,7 +3,7 @@
 import { env } from "@/env.mjs";
 import { ErrorCode } from "@/lib/errorCodes";
 import { notAuthenticated, notFound, orgNotFound, secretAlreadyExists, ServiceError, ServiceErrorException, unexpectedError } from "@/lib/serviceError";
-import { CodeHostType, isServiceError } from "@/lib/utils";
+import { CodeHostType, isHttpError, isServiceError } from "@/lib/utils";
 import { prisma } from "@/prisma";
 import { render } from "@react-email/components";
 import * as Sentry from '@sentry/nextjs';
@@ -22,6 +22,7 @@ import { StatusCodes } from "http-status-codes";
 import { cookies, headers } from "next/headers";
 import { createTransport } from "nodemailer";
 import { auth } from "./auth";
+import { Octokit } from "octokit";
 import { getConnection } from "./data/connection";
 import { IS_BILLING_ENABLED } from "./ee/features/billing/stripe";
 import InviteUserEmail from "./emails/inviteUserEmail";
@@ -790,6 +791,144 @@ export const createConnection = async (name: string, type: CodeHostType, connect
         }, OrgRole.OWNER)
     ));
 
+export const experimental_addGithubRepositoryByUrl = async (repositoryUrl: string, domain: string): Promise<{ connectionId: number } | ServiceError> => sew(() =>
+    withAuth((userId) =>
+        withOrgMembership(userId, domain, async ({ org }) => {
+            if (env.EXPERIMENT_SELF_SERVE_REPO_INDEXING_ENABLED !== 'true') {
+                return {
+                    statusCode: StatusCodes.BAD_REQUEST,
+                    errorCode: ErrorCode.INVALID_REQUEST_BODY,
+                    message: "This feature is not enabled.",
+                } satisfies ServiceError;
+            }
+
+            // Parse repository URL to extract owner/repo
+            const repoInfo = (() => {
+                const url = repositoryUrl.trim();
+    
+                // Handle various GitHub URL formats
+                const patterns = [
+                    // https://github.com/owner/repo or https://github.com/owner/repo.git
+                    /^https?:\/\/github\.com\/([a-zA-Z0-9_.-]+)\/([a-zA-Z0-9_.-]+?)(?:\.git)?\/?$/,
+                    // github.com/owner/repo
+                    /^github\.com\/([a-zA-Z0-9_.-]+)\/([a-zA-Z0-9_.-]+?)(?:\.git)?\/?$/,
+                    // owner/repo
+                    /^([a-zA-Z0-9_.-]+)\/([a-zA-Z0-9_.-]+)$/
+                ];
+                
+                for (const pattern of patterns) {
+                    const match = url.match(pattern);
+                    if (match) {
+                        return {
+                            owner: match[1],
+                            repo: match[2]
+                        };
+                    }
+                }
+                
+                return null;
+            })();
+
+            if (!repoInfo) {
+                return {
+                    statusCode: StatusCodes.BAD_REQUEST,
+                    errorCode: ErrorCode.INVALID_REQUEST_BODY,
+                    message: "Invalid repository URL format. Please use 'owner/repo' or 'https://github.com/owner/repo' format.",
+                } satisfies ServiceError;
+            }
+
+            const { owner, repo } = repoInfo;
+            
+            // Use GitHub API to fetch repository information and get the external_id
+            const octokit = new Octokit({
+                auth: env.EXPERIMENT_SELF_SERVE_REPO_INDEXING_GITHUB_TOKEN
+            });
+
+            let githubRepo;
+            try {
+                const response = await octokit.rest.repos.get({
+                    owner,
+                    repo,
+                });
+                githubRepo = response.data;
+            } catch (error) {
+                if (isHttpError(error, 404)) {
+                    return {
+                        statusCode: StatusCodes.NOT_FOUND,
+                        errorCode: ErrorCode.INVALID_REQUEST_BODY,
+                        message: `Repository '${owner}/${repo}' not found or is private. Only public repositories can be added.`,
+                    } satisfies ServiceError;
+                }
+
+                if (isHttpError(error, 403)) {
+                    return {
+                        statusCode: StatusCodes.FORBIDDEN,
+                        errorCode: ErrorCode.INVALID_REQUEST_BODY,
+                        message: `Access to repository '${owner}/${repo}' is forbidden. Only public repositories can be added.`,
+                    } satisfies ServiceError;
+                }
+                
+                return {
+                    statusCode: StatusCodes.INTERNAL_SERVER_ERROR,
+                    errorCode: ErrorCode.INVALID_REQUEST_BODY,
+                    message: `Failed to fetch repository information: ${error instanceof Error ? error.message : 'Unknown error'}`,
+                } satisfies ServiceError;
+            }
+
+            if (githubRepo.private) {
+                return {
+                    statusCode: StatusCodes.BAD_REQUEST,
+                    errorCode: ErrorCode.INVALID_REQUEST_BODY,
+                    message: "Only public repositories can be added.",
+                } satisfies ServiceError;
+            }
+
+            // Check if this repository is already connected using the external_id
+            const existingRepo = await prisma.repo.findFirst({
+                where: {
+                    orgId: org.id,
+                    external_id: githubRepo.id.toString(),
+                    external_codeHostType: 'github',
+                    external_codeHostUrl: 'https://github.com',
+                }
+            });
+
+            if (existingRepo) {
+                return {
+                    statusCode: StatusCodes.BAD_REQUEST,
+                    errorCode: ErrorCode.CONNECTION_ALREADY_EXISTS,
+                    message: "This repository already exists.",
+                } satisfies ServiceError;
+            }
+
+            const connectionName = `${owner}-${repo}-${Date.now()}`;
+
+            // Create GitHub connection config
+            const connectionConfig: GithubConnectionConfig = {
+                type: "github" as const,
+                repos: [`${owner}/${repo}`],
+                ...(env.EXPERIMENT_SELF_SERVE_REPO_INDEXING_GITHUB_TOKEN ? {
+                    token: {
+                        env: 'EXPERIMENT_SELF_SERVE_REPO_INDEXING_GITHUB_TOKEN'
+                    }
+                } : {})
+            };
+
+            const connection = await prisma.connection.create({
+                data: {
+                    orgId: org.id,
+                    name: connectionName,
+                    config: connectionConfig as unknown as Prisma.InputJsonValue,
+                    connectionType: 'github',
+                }
+            });
+
+            return {
+                connectionId: connection.id,
+            }
+        }, OrgRole.GUEST), /* allowAnonymousAccess = */ true
+    ));
+
 export const updateConnectionDisplayName = async (connectionId: number, name: string, domain: string): Promise<{ success: boolean } | ServiceError> => sew(() =>
     withAuth((userId) =>
         withOrgMembership(userId, domain, async ({ org }) => {

packages/web/src/app/[domain]/repos/addRepoButton.tsx

@@ -9,7 +9,6 @@ import { Tooltip, TooltipContent, TooltipProvider, TooltipTrigger } from "@/comp
 import { cn, getRepoImageSrc } from "@/lib/utils"
 import { RepoIndexingStatus } from "@sourcebot/db";
 import { DropdownMenu, DropdownMenuContent, DropdownMenuItem, DropdownMenuTrigger } from "@/components/ui/dropdown-menu"
-import { AddRepoButton } from "./addRepoButton"
 
 export type RepositoryColumnInfo = {
     repoId: number
@@ -97,12 +96,7 @@ const StatusIndicator = ({ status }: { status: RepoIndexingStatus }) => {
 export const columns = (domain: string): ColumnDef<RepositoryColumnInfo>[] => [
     {
         accessorKey: "name",
-        header: () => (
-            <div className="flex items-center w-[400px]">
-                <span>Repository</span>
-                <AddRepoButton />
-            </div>
-        ),
+        header: 'Repository',
         cell: ({ row }) => {
             const repo = row.original
             const url = repo.url

packages/web/src/app/[domain]/repos/columns.tsx

@@ -0,0 +1,128 @@
+'use client';
+
+import { Button } from "@/components/ui/button";
+import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle } from "@/components/ui/dialog";
+import { Form, FormControl, FormField, FormItem, FormLabel, FormMessage } from "@/components/ui/form";
+import { Input } from "@/components/ui/input";
+import { zodResolver } from "@hookform/resolvers/zod";
+import { useForm } from "react-hook-form";
+import { z } from "zod";
+import { experimental_addGithubRepositoryByUrl } from "@/actions";
+import { useDomain } from "@/hooks/useDomain";
+import { isServiceError } from "@/lib/utils";
+import { useToast } from "@/components/hooks/use-toast";
+import { useRouter } from "next/navigation";
+
+interface AddRepositoryDialogProps {
+    isOpen: boolean;
+    onOpenChange: (open: boolean) => void;
+}
+
+// Validation schema for repository URLs
+const formSchema = z.object({
+    repositoryUrl: z.string()
+        .min(1, "Repository URL is required")
+        .refine((url) => {
+            // Allow various GitHub URL formats:
+            // - https://github.com/owner/repo
+            // - github.com/owner/repo
+            // - owner/repo
+            const patterns = [
+                /^https?:\/\/github\.com\/[a-zA-Z0-9_.-]+\/[a-zA-Z0-9_.-]+\/?$/,
+                /^github\.com\/[a-zA-Z0-9_.-]+\/[a-zA-Z0-9_.-]+\/?$/,
+                /^[a-zA-Z0-9_.-]+\/[a-zA-Z0-9_.-]+$/
+            ];
+            return patterns.some(pattern => pattern.test(url.trim()));
+        }, "Please enter a valid GitHub repository URL (e.g., owner/repo or https://github.com/owner/repo)"),
+});
+
+export const AddRepositoryDialog = ({ isOpen, onOpenChange }: AddRepositoryDialogProps) => {
+    const domain = useDomain();
+    const { toast } = useToast();
+    const router = useRouter();
+
+    const form = useForm<z.infer<typeof formSchema>>({
+        resolver: zodResolver(formSchema),
+        defaultValues: {
+            repositoryUrl: "",
+        },
+    });
+
+    const { isSubmitting } = form.formState;
+
+    const onSubmit = async (data: z.infer<typeof formSchema>) => {
+
+        const result = await experimental_addGithubRepositoryByUrl(data.repositoryUrl.trim(), domain);
+        if (isServiceError(result)) {
+            toast({
+                title: "Error adding repository",
+                description: result.message,
+                variant: "destructive",
+            });
+        } else {
+            toast({
+                title: "Repository added successfully!",
+                description: "It will be indexed shortly.",
+            });
+            form.reset();
+            onOpenChange(false);
+            router.refresh();
+        }
+    };
+
+    const handleCancel = () => {
+        form.reset();
+        onOpenChange(false);
+    };
+
+    return (
+        <Dialog open={isOpen} onOpenChange={onOpenChange}>
+            <DialogContent className="sm:max-w-md">
+                <DialogHeader>
+                    <DialogTitle>Add a public repository from GitHub</DialogTitle>
+                    <DialogDescription>
+                        Paste the repo URL - the code will be indexed and available in search.
+                    </DialogDescription>
+                </DialogHeader>
+                
+                <Form {...form}>
+                    <form onSubmit={form.handleSubmit(onSubmit)} className="space-y-4">
+                        <FormField
+                            control={form.control}
+                            name="repositoryUrl"
+                            render={({ field }) => (
+                                <FormItem>
+                                    <FormLabel>Repository URL</FormLabel>
+                                    <FormControl>
+                                        <Input
+                                            {...field}
+                                            placeholder="https://github.com/user/project"
+                                            disabled={isSubmitting}
+                                        />
+                                    </FormControl>
+                                    <FormMessage />
+                                </FormItem>
+                            )}
+                        />
+                    </form>
+                </Form>
+                
+                <DialogFooter>
+                    <Button
+                        variant="outline"
+                        onClick={handleCancel}
+                        disabled={isSubmitting}
+                    >
+                        Cancel
+                    </Button>
+                    <Button
+                        onClick={form.handleSubmit(onSubmit)}
+                        disabled={isSubmitting}
+                    >
+                        {isSubmitting ? "Adding..." : "Add Repository"}
+                    </Button>
+                </DialogFooter>
+            </DialogContent>
+        </Dialog>
+    );
+};

packages/web/src/app/[domain]/repos/components/addRepositoryDialog.tsx

@@ -2,6 +2,7 @@ import { RepositoryTable } from "./repositoryTable";
 import { getOrgFromDomain } from "@/data/org";
 import { PageNotFound } from "../components/pageNotFound";
 import { Header } from "../components/header";
+import { env } from "@/env.mjs";
 
 export default async function ReposPage({ params: { domain } }: { params: { domain: string } }) {
     const org = await getOrgFromDomain(domain);
@@ -16,7 +17,9 @@ export default async function ReposPage({ params: { domain } }: { params: { doma
             </Header>
             <div className="flex flex-col items-center">
                 <div className="w-full">
-                    <RepositoryTable />
+                    <RepositoryTable
+                        isAddReposButtonVisible={env.EXPERIMENT_SELF_SERVE_REPO_INDEXING_ENABLED === 'true'}
+                    />
                 </div>
             </div>
         </div>