Skip to content

Commit 6aed4a9

Browse files
brendan-kellambrendan-kellam
committed
github topic filtering support
1 parent b9e6ce5 commit 6aed4a9

File tree

6 files changed

+198
-6
lines changed

6 files changed

+198
-6
lines changed

‎packages/backend/src/github.ts

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ import { GitHubConfig } from "./schemas/v2.js";
33
import { createLogger } from "./logger.js";
44
import { AppContext, GitRepository } from "./types.js";
55
import path from 'path';
6-
import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, getTokenFromConfig, marshalBool, measure } from "./utils.js";
6+
import { excludeArchivedRepos, excludeForkedRepos, excludeReposByName, excludeReposByTopic, getTokenFromConfig, includeReposByTopic, marshalBool, measure } from "./utils.js";
77
import micromatch from "micromatch";
88

99
const logger = createLogger("GitHub");
@@ -21,6 +21,7 @@ type OctokitRepository = {
2121
subscribers_count?: number,
2222
forks_count?: number,
2323
archived?: boolean,
24+
topics?: string[],
2425
}
2526

2627
export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: AbortSignal, ctx: AppContext) => {
@@ -80,6 +81,7 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
8081
isStale: false,
8182
isFork: repo.fork,
8283
isArchived: !!repo.archived,
84+
topics: repo.topics ?? [],
8385
gitConfigMetadata: {
8486
'zoekt.web-url-type': 'github',
8587
'zoekt.web-url': repo.html_url,
@@ -97,6 +99,10 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
9799
} satisfies GitRepository;
98100
});
99101

102+
if (config.topics) {
103+
repos = includeReposByTopic(repos, config.topics, logger);
104+
}
105+
100106
if (config.exclude) {
101107
if (!!config.exclude.forks) {
102108
repos = excludeForkedRepos(repos, logger);
@@ -109,6 +115,10 @@ export const getGitHubReposFromConfig = async (config: GitHubConfig, signal: Abo
109115
if (config.exclude.repos) {
110116
repos = excludeReposByName(repos, config.exclude.repos, logger);
111117
}
118+
119+
if (config.exclude.topics) {
120+
repos = excludeReposByTopic(repos, config.exclude.topics, logger);
121+
}
112122
}
113123

114124
logger.debug(`Found ${repos.length} total repositories.`);

‎packages/backend/src/schemas/v2.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,12 @@ export interface GitHubConfig {
5454
* List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'.
5555
*/
5656
repos?: string[];
57+
/**
58+
* List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported.
59+
*
60+
* @minItems 1
61+
*/
62+
topics?: [string, ...string[]];
5763
exclude?: {
5864
/**
5965
* Exclude forked repositories from syncing.
@@ -67,6 +73,7 @@ export interface GitHubConfig {
6773
* List of individual repositories to exclude from syncing. Glob patterns are supported.
6874
*/
6975
repos?: string[];
76+
topics?: string[];
7077
};
7178
revisions?: GitRevisions;
7279
}

‎packages/backend/src/types.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ interface BaseRepository {
88
isFork?: boolean;
99
isArchived?: boolean;
1010
codeHost?: string;
11+
topics?: string[];
1112
}
1213

1314
export interface GitRepository extends BaseRepository {

‎packages/backend/src/utils.test.ts

Lines changed: 133 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { expect, test } from 'vitest';
2-
import { arraysEqualShallow, isRemotePath, excludeReposByName } from './utils';
2+
import { arraysEqualShallow, isRemotePath, excludeReposByName, includeReposByTopic, excludeReposByTopic } from './utils';
33
import { Repository } from './types';
44

55
const testNames: string[] = [
@@ -125,3 +125,135 @@ test('isRemotePath should return false for non HTTP paths', () => {
125125
expect(isRemotePath('')).toBe(false);
126126
expect(isRemotePath(' ')).toBe(false);
127127
});
128+
129+
130+
test('includeReposByTopic should return repos with matching topics', () => {
131+
const repos = [
132+
{ id: '1', topics: ['javascript', 'typescript'] },
133+
{ id: '2', topics: ['python', 'django'] },
134+
{ id: '3', topics: ['typescript', 'react'] }
135+
].map(r => ({
136+
...createRepository(r.id),
137+
...r,
138+
} satisfies Repository));
139+
140+
const result = includeReposByTopic(repos, ['typescript']);
141+
expect(result.length).toBe(2);
142+
expect(result.map(r => r.id)).toEqual(['1', '3']);
143+
});
144+
145+
test('includeReposByTopic should handle glob patterns in topic matching', () => {
146+
const repos = [
147+
{ id: '1', topics: ['frontend-app', 'backend-app'] },
148+
{ id: '2', topics: ['mobile-app', 'web-app'] },
149+
{ id: '3', topics: ['desktop-app', 'cli-app'] }
150+
].map(r => ({
151+
...createRepository(r.id),
152+
...r,
153+
} satisfies Repository));
154+
155+
const result = includeReposByTopic(repos, ['*-app']);
156+
expect(result.length).toBe(3);
157+
});
158+
159+
test('includeReposByTopic should handle repos with no topics', () => {
160+
const repos = [
161+
{ id: '1', topics: ['javascript'] },
162+
{ id: '2', topics: undefined },
163+
{ id: '3', topics: [] }
164+
].map(r => ({
165+
...createRepository(r.id),
166+
...r,
167+
} satisfies Repository));
168+
169+
const result = includeReposByTopic(repos, ['javascript']);
170+
expect(result.length).toBe(1);
171+
expect(result[0].id).toBe('1');
172+
});
173+
174+
test('includeReposByTopic should return empty array when no repos match topics', () => {
175+
const repos = [
176+
{ id: '1', topics: ['frontend'] },
177+
{ id: '2', topics: ['backend'] }
178+
].map(r => ({
179+
...createRepository(r.id),
180+
...r,
181+
} satisfies Repository));
182+
183+
const result = includeReposByTopic(repos, ['mobile']);
184+
expect(result).toEqual([]);
185+
});
186+
187+
188+
test('excludeReposByTopic should exclude repos with matching topics', () => {
189+
const repos = [
190+
{ id: '1', topics: ['javascript', 'typescript'] },
191+
{ id: '2', topics: ['python', 'django'] },
192+
{ id: '3', topics: ['typescript', 'react'] }
193+
].map(r => ({
194+
...createRepository(r.id),
195+
...r,
196+
} satisfies Repository));
197+
198+
const result = excludeReposByTopic(repos, ['typescript']);
199+
expect(result.length).toBe(1);
200+
expect(result[0].id).toBe('2');
201+
});
202+
203+
test('excludeReposByTopic should handle glob patterns', () => {
204+
const repos = [
205+
{ id: '1', topics: ['test-lib', 'test-app'] },
206+
{ id: '2', topics: ['prod-lib', 'prod-app'] },
207+
{ id: '3', topics: ['dev-tool'] }
208+
].map(r => ({
209+
...createRepository(r.id),
210+
...r,
211+
} satisfies Repository));
212+
213+
const result = excludeReposByTopic(repos, ['test-*']);
214+
expect(result.length).toBe(2);
215+
expect(result.map(r => r.id)).toEqual(['2', '3']);
216+
});
217+
218+
test('excludeReposByTopic should handle multiple exclude patterns', () => {
219+
const repos = [
220+
{ id: '1', topics: ['frontend', 'react'] },
221+
{ id: '2', topics: ['backend', 'node'] },
222+
{ id: '3', topics: ['mobile', 'react-native'] }
223+
].map(r => ({
224+
...createRepository(r.id),
225+
...r,
226+
} satisfies Repository));
227+
228+
const result = excludeReposByTopic(repos, ['*end', '*native']);
229+
expect(result.length).toBe(0);
230+
});
231+
232+
test('excludeReposByTopic should not exclude repos when no topics match', () => {
233+
const repos = [
234+
{ id: '1', topics: ['frontend'] },
235+
{ id: '2', topics: ['backend'] },
236+
{ id: '3', topics: undefined }
237+
].map(r => ({
238+
...createRepository(r.id),
239+
...r,
240+
} satisfies Repository));
241+
242+
const result = excludeReposByTopic(repos, ['mobile']);
243+
expect(result.length).toBe(3);
244+
expect(result.map(r => r.id)).toEqual(['1', '2', '3']);
245+
});
246+
247+
test('excludeReposByTopic should handle empty exclude patterns array', () => {
248+
const repos = [
249+
{ id: '1', topics: ['frontend'] },
250+
{ id: '2', topics: ['backend'] }
251+
].map(r => ({
252+
...createRepository(r.id),
253+
...r,
254+
} satisfies Repository));
255+
256+
const result = excludeReposByTopic(repos, []);
257+
expect(result.length).toBe(2);
258+
expect(result).toEqual(repos);
259+
});

‎packages/backend/src/utils.ts

Lines changed: 31 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ export const marshalBool = (value?: boolean) => {
2020
export const excludeForkedRepos = <T extends Repository>(repos: T[], logger?: Logger) => {
2121
return repos.filter((repo) => {
2222
if (!!repo.isFork) {
23-
logger?.debug(`Excluding repo ${repo.id}. Reason: exclude.forks is true`);
23+
logger?.debug(`Excluding repo ${repo.id}. Reason: \`exclude.forks\` is true`);
2424
return false;
2525
}
2626
return true;
@@ -30,7 +30,7 @@ export const excludeForkedRepos = <T extends Repository>(repos: T[], logger?: Lo
3030
export const excludeArchivedRepos = <T extends Repository>(repos: T[], logger?: Logger) => {
3131
return repos.filter((repo) => {
3232
if (!!repo.isArchived) {
33-
logger?.debug(`Excluding repo ${repo.id}. Reason: exclude.archived is true`);
33+
logger?.debug(`Excluding repo ${repo.id}. Reason: \`exclude.archived\` is true`);
3434
return false;
3535
}
3636
return true;
@@ -41,7 +41,7 @@ export const excludeArchivedRepos = <T extends Repository>(repos: T[], logger?:
4141
export const excludeReposByName = <T extends Repository>(repos: T[], excludedRepoNames: string[], logger?: Logger) => {
4242
return repos.filter((repo) => {
4343
if (micromatch.isMatch(repo.name, excludedRepoNames)) {
44-
logger?.debug(`Excluding repo ${repo.id}. Reason: exclude.repos contains ${repo.name}`);
44+
logger?.debug(`Excluding repo ${repo.id}. Reason: \`exclude.repos\` contains ${repo.name}`);
4545
return false;
4646
}
4747
return true;
@@ -51,13 +51,40 @@ export const excludeReposByName = <T extends Repository>(repos: T[], excludedRep
5151
export const includeReposByName = <T extends Repository>(repos: T[], includedRepoNames: string[], logger?: Logger) => {
5252
return repos.filter((repo) => {
5353
if (micromatch.isMatch(repo.name, includedRepoNames)) {
54-
logger?.debug(`Including repo ${repo.id}. Reason: repos contain ${repo.name}`);
54+
logger?.debug(`Including repo ${repo.id}. Reason: \`repos\` contain ${repo.name}`);
5555
return true;
5656
}
5757
return false;
5858
});
5959
}
6060

61+
export const includeReposByTopic = <T extends Repository>(repos: T[], includedRepoTopics: string[], logger?: Logger) => {
62+
return repos.filter((repo) => {
63+
const topics = repo.topics ?? [];
64+
const matchingTopics = topics.filter((topic) => micromatch.isMatch(topic, includedRepoTopics));
65+
66+
if (matchingTopics.length > 0) {
67+
68+
logger?.debug(`Including repo ${repo.id}. Reason: \`topics\` matches the following topics: ${matchingTopics.join(', ')}`);
69+
return true;
70+
}
71+
return false;
72+
});
73+
}
74+
75+
export const excludeReposByTopic = <T extends Repository>(repos: T[], excludedRepoTopics: string[], logger?: Logger) => {
76+
return repos.filter((repo) => {
77+
const topics = repo.topics ?? [];
78+
const matchingTopics = topics.filter((topic) => micromatch.isMatch(topic, excludedRepoTopics));
79+
80+
if (matchingTopics.length > 0) {
81+
logger?.debug(`Excluding repo ${repo.id}. Reason: \`exclude.topics\` matches the following topics: ${matchingTopics.join(', ')}`);
82+
return false;
83+
}
84+
return true;
85+
});
86+
}
87+
6188
export const getTokenFromConfig = (token: string | { env: string }, ctx: AppContext) => {
6289
if (typeof token === 'string') {
6390
return token;

‎schemas/v2/index.json

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,14 @@
130130
},
131131
"description": "List of individual repositories to sync with. Expected to be formatted as '{orgName}/{repoName}' or '{userName}/{repoName}'."
132132
},
133+
"topics": {
134+
"type": "array",
135+
"items": {
136+
"type": "string"
137+
},
138+
"minItems": 1,
139+
"description": "List of repository topics to include when syncing. Only repositories that match at least one of the provided `topics` will be synced. If not specified, all repositories will be synced, unless explicitly defined in the `exclude` property. Glob patterns are supported."
140+
},
133141
"exclude": {
134142
"type": "object",
135143
"properties": {
@@ -150,6 +158,13 @@
150158
},
151159
"default": [],
152160
"description": "List of individual repositories to exclude from syncing. Glob patterns are supported."
161+
},
162+
"topics": {
163+
"type": "array",
164+
"items": {
165+
"type": "string"
166+
},
167+
"description": "List of repository topics to exclude when syncing. Repositories that match one of the provided `topics` will be excluded from syncing. Glob patterns are supported."
153168
}
154169
},
155170
"additionalProperties": false

0 commit comments

Comments
 (0)