add gitea support

msukkari · msukkari · commit c449409670e1 · 2025-02-09T18:17:28.000-08:00
diff --git a/packages/backend/src/connectionManager.ts b/packages/backend/src/connectionManager.ts
@@ -5,7 +5,7 @@ import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type";
 import { createLogger } from "./logger.js";
 import os from 'os';
 import { Redis } from 'ioredis';
-import { RepoData, compileGithubConfig, compileGitlabConfig } from "./repoCompileUtils.js";
+import { RepoData, compileGithubConfig, compileGitlabConfig, compileGiteaConfig } from "./repoCompileUtils.js";
 
 interface IConnectionManager {
     scheduleConnectionSync: (connection: Connection) => Promise<void>;
@@ -86,6 +86,9 @@ export class ConnectionManager implements IConnectionManager {
                 case 'gitlab': {
                     return await compileGitlabConfig(config, job.data.connectionId, orgId, this.db);
                 }
+                case 'gitea': {
+                    return await compileGiteaConfig(config, job.data.connectionId, orgId, this.db);
+                }
                 default: {
                     return [];
                 }
diff --git a/packages/backend/src/gitea.ts b/packages/backend/src/gitea.ts
@@ -1,17 +1,16 @@
 import { Api, giteaApi, HttpResponse, Repository as GiteaRepository } from 'gitea-js';
-import { GiteaConfig } from "@sourcebot/schemas/v2/index.type"
-import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from './utils.js';
-import { AppContext, GitRepository } from './types.js';
+import { GiteaConnectionConfig } from '@sourcebot/schemas/v3/gitea.type';
+import { getTokenFromConfig, measure } from './utils.js';
 import fetch from 'cross-fetch';
 import { createLogger } from './logger.js';
-import path from 'path';
 import micromatch from 'micromatch';
+import { PrismaClient } from '@sourcebot/db';
 
 const logger = createLogger('Gitea');
 
-export const getGiteaReposFromConfig = async (config: GiteaConfig, orgId: number, ctx: AppContext) => {
+export const getGiteaReposFromConfig = async (config: GiteaConnectionConfig, orgId: number, db: PrismaClient) => {
     // TODO: pass in DB here to fetch secret properly
-    const token = config.token ? await getTokenFromConfig(config.token, orgId) : undefined;
+    const token = config.token ? await getTokenFromConfig(config.token, orgId, db) : undefined;
 
     const api = giteaApi(config.url ?? 'https://gitea.com', {
         token,
@@ -34,94 +33,107 @@ export const getGiteaReposFromConfig = async (config: GiteaConfig, orgId: number
         const _repos = await getReposOwnedByUsers(config.users, api);
         allRepos = allRepos.concat(_repos);
     }
-
-    let repos: GitRepository[] = allRepos
-        .map((repo) => {
-            const hostname = config.url ? new URL(config.url).hostname : 'gitea.com';
-            const repoId = `${hostname}/${repo.full_name!}`;
-            const repoPath = path.resolve(path.join(ctx.reposPath, `${repoId}.git`));
-
-            const cloneUrl = new URL(repo.clone_url!);
-            if (token) {
-                cloneUrl.username = token;
-            }
-
-            return {
-                vcs: 'git',
-                codeHost: 'gitea',
-                name: repo.full_name!,
-                id: repoId,
-                cloneUrl: cloneUrl.toString(),
-                path: repoPath,
-                isStale: false,
-                isFork: repo.fork!,
-                isArchived: !!repo.archived,
-                gitConfigMetadata: {
-                    'zoekt.web-url-type': 'gitea',
-                    'zoekt.web-url': repo.html_url!,
-                    'zoekt.name': repoId,
-                    'zoekt.archived': marshalBool(repo.archived),
-                    'zoekt.fork': marshalBool(repo.fork!),
-                    'zoekt.public': marshalBool(repo.internal === false && repo.private === false),
-                },
-                branches: [],
-                tags: []
-            } satisfies GitRepository;
-        });
     
-    if (config.exclude) {
-        if (!!config.exclude.forks) {
-            repos = excludeForkedRepos(repos, logger);
-        }
-
-        if (!!config.exclude.archived) {
-            repos = excludeArchivedRepos(repos, logger);
-        }
-
-        if (config.exclude.repos) {
-            repos = excludeReposByName(repos, config.exclude.repos, logger);
+    allRepos = allRepos.filter(repo => repo.full_name !== undefined);
+    allRepos = allRepos.filter(repo => {
+        if (repo.full_name === undefined) {
+            logger.warn(`Repository with undefined full_name found: orgId=${orgId}, repoId=${repo.id}`);
+            return false;
         }
-    }
-
-    logger.debug(`Found ${repos.length} total repositories.`);
+        return true;
+    });
 
+    
     if (config.revisions) {
         if (config.revisions.branches) {
             const branchGlobs = config.revisions.branches;
-            repos = await Promise.all(
-                repos.map(async (repo) => {
-                    const [owner, name] = repo.name.split('/');
+            allRepos = await Promise.all(
+                allRepos.map(async (repo) => {
+                    const [owner, name] = repo.full_name!.split('/');
                     let branches = (await getBranchesForRepo(owner, name, api)).map(branch => branch.name!);
                     branches = micromatch.match(branches, branchGlobs);
-
+                    
                     return {
                         ...repo,
                         branches,
                     };
                 })
             )
         }
-
+        
         if (config.revisions.tags) {
             const tagGlobs = config.revisions.tags;
-            repos = await Promise.all(
-                repos.map(async (repo) => {
-                    const [owner, name] = repo.name.split('/');
+            allRepos = await Promise.all(
+                allRepos.map(async (allRepos) => {
+                    const [owner, name] = allRepos.name!.split('/');
                     let tags = (await getTagsForRepo(owner, name, api)).map(tag => tag.name!);
                     tags = micromatch.match(tags, tagGlobs);
-
+                    
                     return {
-                        ...repo,
+                        ...allRepos,
                         tags,
                     };
                 })
             )
         }
     }
+
+    let repos = allRepos
+        .filter((repo) => {
+            const isExcluded = shouldExcludeRepo({
+                repo,
+                exclude: config.exclude,
+            });
+
+            return !isExcluded;
+        });
     
+    logger.debug(`Found ${repos.length} total repositories.`);
     return repos;
 }
 
+const shouldExcludeRepo = ({
+    repo,
+    exclude
+} : {
+    repo: GiteaRepository,
+    exclude?: {
+        forks?: boolean,
+        archived?: boolean,
+        repos?: string[],
+    }
+}) => {
+    let reason = '';
+    const repoName = repo.full_name!;
+
+    const shouldExclude = (() => {
+        if (!!exclude?.forks && repo.fork) {
+            reason = `\`exclude.forks\` is true`;
+            return true;
+        }
+    
+        if (!!exclude?.archived && !!repo.archived) {
+            reason = `\`exclude.archived\` is true`;
+            return true;
+        }
+
+        if (exclude?.repos) {
+            if (micromatch.isMatch(repoName, exclude.repos)) {
+                reason = `\`exclude.repos\` contains ${repoName}`;
+                return true;
+            }
+        }
+
+        return false;
+    })();
+
+    if (shouldExclude) {
+        logger.debug(`Excluding repo ${repoName}. Reason: ${reason}`);
+    }
+
+    return shouldExclude;
+}
+
 const getTagsForRepo = async <T>(owner: string, repo: string, api: Api<T>) => {
     try {
         logger.debug(`Fetching tags for repo ${owner}/${repo}...`);
diff --git a/packages/backend/src/repoCompileUtils.ts b/packages/backend/src/repoCompileUtils.ts
@@ -1,10 +1,11 @@
 import { GithubConnectionConfig } from '@sourcebot/schemas/v3/github.type';
 import { getGitHubReposFromConfig } from "./github.js";
 import { getGitLabReposFromConfig } from "./gitlab.js";
+import { getGiteaReposFromConfig } from "./gitea.js";
 import { Prisma, PrismaClient } from '@sourcebot/db';
 import { WithRequired } from "./types.js"
 import { marshalBool } from "./utils.js";
-import { GitlabConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
+import { GiteaConnectionConfig, GitlabConnectionConfig } from '@sourcebot/schemas/v3/connection.type';
 
 export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;
 
@@ -102,6 +103,51 @@ export const compileGitlabConfig = async (
             },
         };
 
+        return record;
+    })
+}
+
+export const compileGiteaConfig = async (
+    config: GiteaConnectionConfig,
+    connectionId: number,
+    orgId: number,
+    db: PrismaClient) => {
+
+    const giteaRepos = await getGiteaReposFromConfig(config, orgId, db);
+    const hostUrl = config.url ?? 'https://gitea.com';
+
+    return giteaRepos.map((repo) => {
+        const repoUrl = `${hostUrl}/${repo.full_name}`;
+        const cloneUrl = new URL(repo.clone_url!);
+
+        const record: RepoData = {
+            external_id: repo.id!.toString(),
+            external_codeHostType: 'gitea',
+            external_codeHostUrl: hostUrl,
+            cloneUrl: cloneUrl.toString(),
+            name: repo.full_name!,
+            isFork: repo.fork!,
+            isArchived: !!repo.archived,
+            org: {
+                connect: {
+                    id: orgId,
+                },
+            },
+            connections: {
+                create: {
+                    connectionId: connectionId,
+                }
+            },
+            metadata: {
+                'zoekt.web-url-type': 'gitea',
+                'zoekt.web-url': repo.html_url!,
+                'zoekt.name': repo.full_name!,
+                'zoekt.archived': marshalBool(repo.archived),
+                'zoekt.fork': marshalBool(repo.fork!),
+                'zoekt.public': marshalBool(repo.internal === false && repo.private === false),
+            },
+        };
+
         return record;
     })
 }
diff --git a/packages/web/src/actions.ts b/packages/web/src/actions.ts
@@ -9,6 +9,7 @@ import { ErrorCode } from "@/lib/errorCodes";
 import { isServiceError } from "@/lib/utils";
 import { githubSchema } from "@sourcebot/schemas/v3/github.schema";
 import { gitlabSchema } from "@sourcebot/schemas/v3/gitlab.schema";
+import { giteaSchema } from "@sourcebot/schemas/v3/gitea.schema";
 import { ConnectionConfig } from "@sourcebot/schemas/v3/connection.type";
 import { encrypt } from "@sourcebot/crypto"
 import { getConnection } from "./data/connection";
@@ -279,6 +280,8 @@ const parseConnectionConfig = (connectionType: string, config: string) => {
                 return githubSchema;
             case "gitlab":
                 return gitlabSchema;
+            case 'gitea':
+                return giteaSchema;
         }
     })();
 
diff --git a/packages/web/src/app/connections/[id]/components/configSetting.tsx b/packages/web/src/app/connections/[id]/components/configSetting.tsx
@@ -11,9 +11,10 @@ import { z } from "zod";
 import { ConfigEditor, QuickAction } from "../../components/configEditor";
 import { createZodConnectionConfigValidator } from "../../utils";
 import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type";
-import { githubQuickActions, gitlabQuickActions } from "../../quickActions";
+import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type";
+import { githubQuickActions, gitlabQuickActions, giteaQuickActions } from "../../quickActions";
 import { Schema } from "ajv";
-import { GitLabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type";
+import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type";
 import { gitlabSchema } from "@sourcebot/schemas/v3/gitlab.schema";
 import { updateConnectionConfigAndScheduleSync } from "@/actions";
 import { useToast } from "@/components/hooks/use-toast";
@@ -39,13 +40,21 @@ export const ConfigSetting = (props: ConfigSettingProps) => {
     }
 
     if (type === 'gitlab') {
-        return <ConfigSettingInternal<GitLabConnectionConfig>
+        return <ConfigSettingInternal<GitlabConnectionConfig>
             {...props}
             quickActions={gitlabQuickActions}
             schema={gitlabSchema}
         />;
     }
 
+    if (type === 'gitea') {
+        return <ConfigSettingInternal<GiteaConnectionConfig>
+            {...props}
+            quickActions={giteaQuickActions}
+            schema={githubSchema}
+        />;
+    }
+
     return null;
 }
 
diff --git a/packages/web/src/app/connections/new/[type]/components/connectionCreationForm.tsx b/packages/web/src/app/connections/new/[type]/components/connectionCreationForm.tsx
@@ -18,7 +18,7 @@ import { z } from "zod";
 import { ConfigEditor, QuickActionFn } from "../../../components/configEditor";
 
 interface ConnectionCreationForm<T> {
-    type: 'github' | 'gitlab';
+    type: 'github' | 'gitlab' | 'gitea';
     defaultValues: {
         name: string;
         config: string;
diff --git a/packages/web/src/app/connections/new/[type]/page.tsx b/packages/web/src/app/connections/new/[type]/page.tsx
@@ -1,10 +1,12 @@
 'use client';
 
-import { githubQuickActions, gitlabQuickActions } from "../../quickActions";
+import { giteaQuickActions, githubQuickActions, gitlabQuickActions } from "../../quickActions";
 import ConnectionCreationForm from "./components/connectionCreationForm";
-import { GitLabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type";
+import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type";
+import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type";
 import { gitlabSchema } from "@sourcebot/schemas/v3/gitlab.schema";
 import { githubSchema } from "@sourcebot/schemas/v3/github.schema";
+import { giteaSchema } from "@sourcebot/schemas/v3/gitea.schema";
 import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type";
 import { useRouter } from "next/navigation";
 
@@ -22,16 +24,20 @@ export default function NewConnectionPage({
         return <GitLabCreationForm />;
     }
 
+    if (type === 'gitea') {
+        return <GiteaCreationForm />;
+    }
+
     router.push('/connections');
 }
 
 const GitLabCreationForm = () => {
-    const defaultConfig: GitLabConnectionConfig = {
+    const defaultConfig: GitlabConnectionConfig = {
         type: 'gitlab',
     }
 
     return (
-        <ConnectionCreationForm<GitLabConnectionConfig>
+        <ConnectionCreationForm<GitlabConnectionConfig>
             type="gitlab"
             title="Create a GitLab connection"
             defaultValues={{
@@ -61,4 +67,24 @@ const GitHubCreationForm = () => {
             quickActions={githubQuickActions}
         />
     )
+}
+
+const GiteaCreationForm = () => {
+    const defaultConfig: GiteaConnectionConfig = {
+        type: 'gitea',
+    }
+
+    return (
+        <ConnectionCreationForm<GiteaConnectionConfig>
+            type="gitea"
+            title="Create a Gitea connection"
+            defaultValues={{
+                config: JSON.stringify(defaultConfig, null, 2),
+                name: 'my-gitea-connection',
+            }}
+            schema={giteaSchema}
+            quickActions={giteaQuickActions}
+        />
+    )
+
 }
diff --git a/packages/web/src/app/connections/quickActions.ts b/packages/web/src/app/connections/quickActions.ts
