Skip to content

Commit b0dc257

Browse files
committed
merge main
2 parents 4af3b53 + 03999f0 commit b0dc257

File tree

111 files changed

+2918
-5567
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

111 files changed

+2918
-5567
lines changed

.env.development

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -80,9 +80,6 @@ SOURCEBOT_TELEMETRY_DISABLED=true # Disables telemetry collection
8080
# Controls the number of concurrent indexing jobs that can run at once
8181
# INDEX_CONCURRENCY_MULTIPLE=
8282

83-
# Controls the polling interval for the web app
84-
# NEXT_PUBLIC_POLLING_INTERVAL_MS=
85-
8683
# Controls the version of the web app
8784
# NEXT_PUBLIC_SOURCEBOT_VERSION=
8885

CHANGELOG.md

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
1414

1515
### Fixed
1616
- Fixed "dubious ownership" errors when cloning / fetching repos. [#553](https://github.com/sourcebot-dev/sourcebot/pull/553)
17+
- Fixed issue with Ask Sourcebot tutorial re-appearing after restarting the browser. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
18+
- Fixed `repoIndexTimeoutMs` not being used for index job timeouts. [#567](https://github.com/sourcebot-dev/sourcebot/pull/567)
1719

1820
### Changed
19-
- Remove spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552)
2021
- Improved search performance for unbounded search queries. [#555](https://github.com/sourcebot-dev/sourcebot/pull/555)
22+
- Improved homepage performance by removing client side polling. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
23+
- Changed navbar indexing indicator to only report progress for first time indexing jobs. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
24+
- Improved repo indexing job stability and robustness. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
25+
26+
### Removed
27+
- Removed spam "login page loaded" log. [#552](https://github.com/sourcebot-dev/sourcebot/pull/552)
28+
- Removed connections management page. [#563](https://github.com/sourcebot-dev/sourcebot/pull/563)
2129

2230
### Added
2331
- Added support for passing db connection url as seperate `DATABASE_HOST`, `DATABASE_USERNAME`, `DATABASE_PASSWORD`, `DATABASE_NAME`, and `DATABASE_ARGS` env vars. [#545](https://github.com/sourcebot-dev/sourcebot/pull/545)

package.json

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
"scripts": {
77
"build": "cross-env SKIP_ENV_VALIDATION=1 yarn workspaces foreach -A run build",
88
"test": "yarn workspaces foreach -A run test",
9-
"dev": "yarn dev:prisma:migrate:dev && npm-run-all --print-label --parallel dev:zoekt dev:backend dev:web watch:mcp watch:schemas",
9+
"dev": "concurrently --kill-others --names \"zoekt,worker,web,mcp,schemas\" 'yarn dev:zoekt' 'yarn dev:backend' 'yarn dev:web' 'yarn watch:mcp' 'yarn watch:schemas'",
1010
"with-env": "cross-env PATH=\"$PWD/bin:$PATH\" dotenv -e .env.development -c --",
1111
"dev:zoekt": "yarn with-env zoekt-webserver -index .sourcebot/index -rpc",
1212
"dev:backend": "yarn with-env yarn workspace @sourcebot/backend dev:watch",
@@ -21,9 +21,9 @@
2121
"build:deps": "yarn workspaces foreach -R --from '{@sourcebot/schemas,@sourcebot/error,@sourcebot/crypto,@sourcebot/db,@sourcebot/shared}' run build"
2222
},
2323
"devDependencies": {
24+
"concurrently": "^9.2.1",
2425
"cross-env": "^7.0.3",
25-
"dotenv-cli": "^8.0.0",
26-
"npm-run-all": "^4.1.5"
26+
"dotenv-cli": "^8.0.0"
2727
},
2828
"packageManager": "[email protected]",
2929
"resolutions": {

packages/backend/package.json

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
"git-url-parse": "^16.1.0",
4646
"gitea-js": "^1.22.0",
4747
"glob": "^11.0.0",
48+
"groupmq": "^1.0.0",
4849
"ioredis": "^5.4.2",
4950
"lowdb": "^7.0.1",
5051
"micromatch": "^4.0.8",

packages/backend/src/connectionManager.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -364,12 +364,12 @@ export class ConnectionManager {
364364
}
365365
}
366366

367-
public dispose() {
367+
public async dispose() {
368368
if (this.interval) {
369369
clearInterval(this.interval);
370370
}
371-
this.worker.close();
372-
this.queue.close();
371+
await this.worker.close();
372+
await this.queue.close();
373373
}
374374
}
375375

packages/backend/src/constants.ts

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
1+
import { env } from "./env.js";
12
import { Settings } from "./types.js";
3+
import path from "path";
24

35
/**
46
* Default settings.
@@ -22,4 +24,7 @@ export const DEFAULT_SETTINGS: Settings = {
2224

2325
export const PERMISSION_SYNC_SUPPORTED_CODE_HOST_TYPES = [
2426
'github',
25-
];
27+
];
28+
29+
export const REPOS_CACHE_DIR = path.join(env.DATA_CACHE_DIR, 'repos');
30+
export const INDEX_CACHE_DIR = path.join(env.DATA_CACHE_DIR, 'index');

packages/backend/src/ee/repoPermissionSyncer.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -100,12 +100,12 @@ export class RepoPermissionSyncer {
100100
}, 1000 * 5);
101101
}
102102

103-
public dispose() {
103+
public async dispose() {
104104
if (this.interval) {
105105
clearInterval(this.interval);
106106
}
107-
this.worker.close();
108-
this.queue.close();
107+
await this.worker.close();
108+
await this.queue.close();
109109
}
110110

111111
private async schedulePermissionSync(repos: Repo[]) {

packages/backend/src/ee/userPermissionSyncer.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,12 +101,12 @@ export class UserPermissionSyncer {
101101
}, 1000 * 5);
102102
}
103103

104-
public dispose() {
104+
public async dispose() {
105105
if (this.interval) {
106106
clearInterval(this.interval);
107107
}
108-
this.worker.close();
109-
this.queue.close();
108+
await this.worker.close();
109+
await this.queue.close();
110110
}
111111

112112
private async schedulePermissionSync(users: User[]) {

packages/backend/src/env.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ export const env = createEnv({
4444
LOGTAIL_TOKEN: z.string().optional(),
4545
LOGTAIL_HOST: z.string().url().optional(),
4646
SOURCEBOT_LOG_LEVEL: z.enum(["info", "debug", "warn", "error"]).default("info"),
47+
DEBUG_ENABLE_GROUPMQ_LOGGING: booleanSchema.default('false'),
4748

4849
DATABASE_URL: z.string().url().default("postgresql://postgres:postgres@localhost:5432/postgres"),
4950
CONFIG_PATH: z.string().optional(),

packages/backend/src/git.ts

Lines changed: 96 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -1,30 +1,67 @@
11
import { CheckRepoActions, GitConfigScope, simpleGit, SimpleGitProgressEvent } from 'simple-git';
22
import { mkdir } from 'node:fs/promises';
33
import { env } from './env.js';
4+
import { dirname, resolve } from 'node:path';
5+
import { existsSync } from 'node:fs';
46

57
type onProgressFn = (event: SimpleGitProgressEvent) => void;
68

9+
/**
10+
* Creates a simple-git client that has it's working directory
11+
* set to the given path.
12+
*/
13+
const createGitClientForPath = (path: string, onProgress?: onProgressFn, signal?: AbortSignal) => {
14+
if (!existsSync(path)) {
15+
throw new Error(`Path ${path} does not exist`);
16+
}
17+
18+
const parentPath = resolve(dirname(path));
19+
20+
const git = simpleGit({
21+
progress: onProgress,
22+
abort: signal,
23+
})
24+
.env({
25+
...process.env,
26+
/**
27+
* @note on some inside-baseball on why this is necessary: The specific
28+
* issue we saw was that a `git clone` would fail without throwing, and
29+
* then a subsequent `git config` command would run, but since the clone
30+
* failed, it wouldn't be running in a git directory. Git would then walk
31+
* up the directory tree until it either found a git directory (in the case
32+
* of the development env) or it would hit a GIT_DISCOVERY_ACROSS_FILESYSTEM
33+
* error when trying to cross a filesystem boundary (in the prod case).
34+
* GIT_CEILING_DIRECTORIES ensures that this walk will be limited to the
35+
* parent directory.
36+
*/
37+
GIT_CEILING_DIRECTORIES: parentPath,
38+
})
39+
.cwd({
40+
path,
41+
});
42+
43+
return git;
44+
}
45+
746
export const cloneRepository = async (
847
{
948
cloneUrl,
1049
authHeader,
1150
path,
1251
onProgress,
52+
signal,
1353
}: {
1454
cloneUrl: string,
1555
authHeader?: string,
1656
path: string,
1757
onProgress?: onProgressFn
58+
signal?: AbortSignal
1859
}
1960
) => {
2061
try {
2162
await mkdir(path, { recursive: true });
2263

23-
const git = simpleGit({
24-
progress: onProgress,
25-
}).cwd({
26-
path,
27-
})
64+
const git = createGitClientForPath(path, onProgress, signal);
2865

2966
const cloneArgs = [
3067
"--bare",
@@ -33,7 +70,11 @@ export const cloneRepository = async (
3370

3471
await git.clone(cloneUrl, path, cloneArgs);
3572

36-
await unsetGitConfig(path, ["remote.origin.url"]);
73+
await unsetGitConfig({
74+
path,
75+
keys: ["remote.origin.url"],
76+
signal,
77+
});
3778
} catch (error: unknown) {
3879
const baseLog = `Failed to clone repository: ${path}`;
3980

@@ -54,20 +95,17 @@ export const fetchRepository = async (
5495
authHeader,
5596
path,
5697
onProgress,
98+
signal,
5799
}: {
58100
cloneUrl: string,
59101
authHeader?: string,
60102
path: string,
61-
onProgress?: onProgressFn
103+
onProgress?: onProgressFn,
104+
signal?: AbortSignal
62105
}
63106
) => {
107+
const git = createGitClientForPath(path, onProgress, signal);
64108
try {
65-
const git = simpleGit({
66-
progress: onProgress,
67-
}).cwd({
68-
path: path,
69-
})
70-
71109
if (authHeader) {
72110
await git.addConfig("http.extraHeader", authHeader);
73111
}
@@ -90,12 +128,6 @@ export const fetchRepository = async (
90128
}
91129
} finally {
92130
if (authHeader) {
93-
const git = simpleGit({
94-
progress: onProgress,
95-
}).cwd({
96-
path: path,
97-
})
98-
99131
await git.raw(["config", "--unset", "http.extraHeader", authHeader]);
100132
}
101133
}
@@ -107,10 +139,19 @@ export const fetchRepository = async (
107139
* that do not exist yet. It will _not_ remove any existing keys that are not
108140
* present in gitConfig.
109141
*/
110-
export const upsertGitConfig = async (path: string, gitConfig: Record<string, string>, onProgress?: onProgressFn) => {
111-
const git = simpleGit({
112-
progress: onProgress,
113-
}).cwd(path);
142+
export const upsertGitConfig = async (
143+
{
144+
path,
145+
gitConfig,
146+
onProgress,
147+
signal,
148+
}: {
149+
path: string,
150+
gitConfig: Record<string, string>,
151+
onProgress?: onProgressFn,
152+
signal?: AbortSignal
153+
}) => {
154+
const git = createGitClientForPath(path, onProgress, signal);
114155

115156
try {
116157
for (const [key, value] of Object.entries(gitConfig)) {
@@ -129,10 +170,19 @@ export const upsertGitConfig = async (path: string, gitConfig: Record<string, st
129170
* Unsets the specified keys in the git config for the repo at the given path.
130171
* If a key is not set, this is a no-op.
131172
*/
132-
export const unsetGitConfig = async (path: string, keys: string[], onProgress?: onProgressFn) => {
133-
const git = simpleGit({
134-
progress: onProgress,
135-
}).cwd(path);
173+
export const unsetGitConfig = async (
174+
{
175+
path,
176+
keys,
177+
onProgress,
178+
signal,
179+
}: {
180+
path: string,
181+
keys: string[],
182+
onProgress?: onProgressFn,
183+
signal?: AbortSignal
184+
}) => {
185+
const git = createGitClientForPath(path, onProgress, signal);
136186

137187
try {
138188
const configList = await git.listConfig();
@@ -155,10 +205,20 @@ export const unsetGitConfig = async (path: string, keys: string[], onProgress?:
155205
/**
156206
* Returns true if `path` is the _root_ of a git repository.
157207
*/
158-
export const isPathAValidGitRepoRoot = async (path: string, onProgress?: onProgressFn) => {
159-
const git = simpleGit({
160-
progress: onProgress,
161-
}).cwd(path);
208+
export const isPathAValidGitRepoRoot = async ({
209+
path,
210+
onProgress,
211+
signal,
212+
}: {
213+
path: string,
214+
onProgress?: onProgressFn,
215+
signal?: AbortSignal
216+
}) => {
217+
if (!existsSync(path)) {
218+
return false;
219+
}
220+
221+
const git = createGitClientForPath(path, onProgress, signal);
162222

163223
try {
164224
return git.checkIsRepo(CheckRepoActions.IS_REPO_ROOT);
@@ -184,7 +244,7 @@ export const isUrlAValidGitRepo = async (url: string) => {
184244
}
185245

186246
export const getOriginUrl = async (path: string) => {
187-
const git = simpleGit().cwd(path);
247+
const git = createGitClientForPath(path);
188248

189249
try {
190250
const remotes = await git.getConfig('remote.origin.url', GitConfigScope.local);
@@ -199,18 +259,13 @@ export const getOriginUrl = async (path: string) => {
199259
}
200260

201261
export const getBranches = async (path: string) => {
202-
const git = simpleGit();
203-
const branches = await git.cwd({
204-
path,
205-
}).branch();
206-
262+
const git = createGitClientForPath(path);
263+
const branches = await git.branch();
207264
return branches.all;
208265
}
209266

210267
export const getTags = async (path: string) => {
211-
const git = simpleGit();
212-
const tags = await git.cwd({
213-
path,
214-
}).tags();
268+
const git = createGitClientForPath(path);
269+
const tags = await git.tags();
215270
return tags.all;
216271
}

0 commit comments

Comments
 (0)