From afb86ad6efa6502f9a80cacbcd0d009e9bdf2c95 Mon Sep 17 00:00:00 2001 From: e-for-eshaan Date: Wed, 15 May 2024 17:11:49 +0530 Subject: [PATCH 1/5] adds a graphql api to fetch user and org-repos for the given PAT and search-string --- .../api/internal/[org_id]/git_provider_org.ts | 104 +--------- .../pages/api/internal/[org_id]/utils.ts | 177 ++++++++++++++++++ 2 files changed, 185 insertions(+), 96 deletions(-) create mode 100644 web-server/pages/api/internal/[org_id]/utils.ts diff --git a/web-server/pages/api/internal/[org_id]/git_provider_org.ts b/web-server/pages/api/internal/[org_id]/git_provider_org.ts index d0837a055..928f85f05 100644 --- a/web-server/pages/api/internal/[org_id]/git_provider_org.ts +++ b/web-server/pages/api/internal/[org_id]/git_provider_org.ts @@ -1,15 +1,9 @@ -import { AxiosError } from 'axios'; import * as yup from 'yup'; -import { internal } from '@/api-helpers/axios'; +import { searchGithubRepos } from '@/api/internal/[org_id]/utils'; import { Endpoint } from '@/api-helpers/global'; -import { Errors, ResponseError } from '@/constants/error'; import { Integration } from '@/constants/integrations'; -import { LoadedOrg, Repo } from '@/types/github'; -import { BaseRepo } from '@/types/resources'; import { dec } from '@/utils/auth-supplementary'; -import { getBaseRepoFromUnionRepo } from '@/utils/code'; -import { homogenize } from '@/utils/datatype'; import { db, getFirstRow } from '@/utils/db'; export type CodeSourceProvidersIntegration = @@ -30,57 +24,17 @@ const endpoint = new Endpoint(pathSchema); endpoint.handle.GET(getSchema, async (req, res) => { const { org_id, search_text } = req.payload; - let count = 0; - const repos = await getRepos(org_id, search_text); - const searchResults = [] as BaseRepo[]; - for (let raw_repo of repos) { - const repo = getBaseRepoFromUnionRepo(raw_repo); - if (count >= 5) break; - if (!search_text) { - count++; - searchResults.push(repo); - continue; - } - const repoName = homogenize(`${repo.parent}/${repo.name}`); - const searchText = homogenize(search_text); - const matchesSearch = repoName.includes(searchText); - if (matchesSearch) { - count++; - searchResults.push(repo); - } - } - return res.status(200).send(searchResults); + const token = await getGithubToken(org_id); + const repos = await searchGithubRepos(token, search_text); + + return res.status(200).send(repos); }); export default endpoint.serve(); -const providerOrgBrandingMap = { - bitbucket: 'workspaces', - github: 'orgs', - gitlab: 'groups' -}; - -const THRESHOLD = 300; - -export const getProviderOrgs = ( - org_id: ID, - provider: CodeSourceProvidersIntegration -) => - internal - .get<{ orgs: LoadedOrg[] }>( - `/orgs/${org_id}/integrations/${provider}/${providerOrgBrandingMap[provider]}` - ) - .catch((e: AxiosError) => { - if (e.response.status !== 404) throw e; - throw new ResponseError(Errors.INTEGRATION_NOT_FOUND); - }); - -async function getRepos( - org_id: ID, - searchQuery?: string -): Promise[]> { - const token = await db('Integration') +const getGithubToken = async (org_id: ID) => { + return await db('Integration') .select() .where({ org_id, @@ -89,46 +43,4 @@ async function getRepos( .returning('*') .then(getFirstRow) .then((r) => dec(r.access_token_enc_chunks)); - - const baseUrl = 'https://api.github.com/user/repos'; - const params: URLSearchParams = new URLSearchParams(); - params.set('access_token', token); - - if (searchQuery) { - params.set('q', searchQuery); - } - - let allRepos: any[] = []; - let url = `${baseUrl}?${params.toString()}`; - let response: Response; - - do { - if (allRepos.length >= THRESHOLD) { - break; - } - response = await fetch(url, { - headers: { - Authorization: `token ${token}` - } - }); - - if (!response.ok) { - throw new Error(`Failed to fetch repos: ${response.statusText}`); - } - - const data = (await response.json()) as any[]; - allRepos = allRepos.concat(data); - - const nextLink = response.headers.get('Link'); - if (nextLink) { - const nextUrl = nextLink - .split(',') - .find((link) => link.includes('rel="next"')); - url = nextUrl ? nextUrl.trim().split(';')[0].slice(1, -1) : ''; - } else { - url = ''; - } - } while (url); - - return allRepos; -} +}; diff --git a/web-server/pages/api/internal/[org_id]/utils.ts b/web-server/pages/api/internal/[org_id]/utils.ts new file mode 100644 index 000000000..8f1ecb023 --- /dev/null +++ b/web-server/pages/api/internal/[org_id]/utils.ts @@ -0,0 +1,177 @@ +import { BaseRepo } from '@/types/resources'; + +const GITHUB_API_URL = 'https://api.github.com/graphql'; + +interface Repo { + id: string; + name: string; + url: string; + isPrivate: boolean; + owner: string; +} + +async function getOrgs(pat: string): Promise { + const query = ` + query { + viewer { + organizations(first: 100) { + nodes { + login + } + } + } + } + `; + + const response = await fetch(GITHUB_API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${pat}` + }, + body: JSON.stringify({ query }) + }); + + const responseBody = (await response.json()) as any; + + if (!response.ok) { + throw new Error(`GitHub API error: ${responseBody.message}`); + } + + return responseBody.data.viewer.organizations.nodes.map( + (org: any) => org.login + ); +} + +async function getUserRepos( + pat: string, + searchString: string +): Promise { + const query = ` + query($queryString: String!) { + search(type: REPOSITORY, query: $queryString, first: 50) { + edges { + node { + ... on Repository { + name + url + databaseId + description + owner { + login + } + } + } + } + } + } + `; + + const queryString = `${searchString} in:name user:@me`; + + const response = await fetch(GITHUB_API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${pat}` + }, + body: JSON.stringify({ query, variables: { queryString } }) + }); + + const responseBody = (await response.json()) as any; + + if (!response.ok) { + throw new Error(`GitHub API error: ${responseBody.message}`); + } + + const repositories = responseBody.data.search.edges.map( + (edge: any) => edge.node + ); + + return repositories.map( + (repo: any) => + ({ + id: repo.databaseId, + name: repo.name, + desc: repo.description, + slug: repo.name, + parent: repo.owner.login, + web_url: repo.url, + language: repo.languages, + branch: '' + }) as BaseRepo + ); +} + +async function searchGithubRepos( + pat: string, + searchString: string +): Promise { + const organizations = await getOrgs(pat); + + const repoPromises = organizations.map(async (org) => { + const query = ` + query($queryString: String!) { + search(type: REPOSITORY, query: $queryString, first: 50) { + edges { + node { + ... on Repository { + name + url + description + databaseId + owner { + login + } + } + } + } + } + } + `; + + const queryString = `org:${org} ${searchString} in:name`; + + const response = await fetch(GITHUB_API_URL, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + Authorization: `Bearer ${pat}` + }, + body: JSON.stringify({ query, variables: { queryString } }) + }); + + const responseBody = (await response.json()) as any; + + if (!response.ok) { + throw new Error(`GitHub API error: ${responseBody.message}`); + } + + const repositories = responseBody.data.search.edges.map( + (edge: any) => edge.node + ); + + return repositories.map( + (repo: any) => + ({ + id: repo.databaseId, + name: repo.name, + desc: repo.description, + slug: repo.name, + parent: repo.owner.login, + web_url: repo.url, + language: repo.languages, + branch: '' + }) as BaseRepo + ); + }); + + const [userRepos, orgRepos] = await Promise.all([ + getUserRepos(pat, searchString), + await Promise.all(repoPromises) + ]); + + return [...userRepos, ...orgRepos.flat()]; +} + +export { searchGithubRepos }; From 2a3430b52fc531e9979bf0d62697c7e50f6eb41a Mon Sep 17 00:00:00 2001 From: e-for-eshaan Date: Wed, 15 May 2024 17:25:02 +0530 Subject: [PATCH 2/5] simplifies fetch by removing org-based logic --- .../pages/api/internal/[org_id]/utils.ts | 165 ++++-------------- 1 file changed, 33 insertions(+), 132 deletions(-) diff --git a/web-server/pages/api/internal/[org_id]/utils.ts b/web-server/pages/api/internal/[org_id]/utils.ts index 8f1ecb023..32cd3f470 100644 --- a/web-server/pages/api/internal/[org_id]/utils.ts +++ b/web-server/pages/api/internal/[org_id]/utils.ts @@ -10,64 +10,38 @@ interface Repo { owner: string; } -async function getOrgs(pat: string): Promise { - const query = ` - query { - viewer { - organizations(first: 100) { - nodes { - login - } - } - } - } - `; - - const response = await fetch(GITHUB_API_URL, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${pat}` - }, - body: JSON.stringify({ query }) - }); - - const responseBody = (await response.json()) as any; - - if (!response.ok) { - throw new Error(`GitHub API error: ${responseBody.message}`); - } - - return responseBody.data.viewer.organizations.nodes.map( - (org: any) => org.login - ); -} - -async function getUserRepos( +export const searchGithubRepos = async ( pat: string, searchString: string -): Promise { +): Promise => { const query = ` - query($queryString: String!) { - search(type: REPOSITORY, query: $queryString, first: 50) { - edges { - node { - ... on Repository { - name - url - databaseId - description - owner { - login - } - } - } - } - } - } - `; - - const queryString = `${searchString} in:name user:@me`; + query($queryString: String!) { + search(type: REPOSITORY, query: $queryString, first: 50) { + edges { + node { + ... on Repository { + name + url + defaultBranchRef { + name + } + databaseId + description + primaryLanguage { + name + } + + owner { + login + } + } + } + } + } + } + `; + + const queryString = `${searchString} in:name`; const response = await fetch(GITHUB_API_URL, { method: 'POST', @@ -94,84 +68,11 @@ async function getUserRepos( id: repo.databaseId, name: repo.name, desc: repo.description, - slug: repo.name, + slug: `${repo.owner.login}/${repo.name}`, parent: repo.owner.login, web_url: repo.url, - language: repo.languages, - branch: '' + language: repo.primaryLanguage?.name, + branch: repo.defaultBranchRef?.name }) as BaseRepo ); -} - -async function searchGithubRepos( - pat: string, - searchString: string -): Promise { - const organizations = await getOrgs(pat); - - const repoPromises = organizations.map(async (org) => { - const query = ` - query($queryString: String!) { - search(type: REPOSITORY, query: $queryString, first: 50) { - edges { - node { - ... on Repository { - name - url - description - databaseId - owner { - login - } - } - } - } - } - } - `; - - const queryString = `org:${org} ${searchString} in:name`; - - const response = await fetch(GITHUB_API_URL, { - method: 'POST', - headers: { - 'Content-Type': 'application/json', - Authorization: `Bearer ${pat}` - }, - body: JSON.stringify({ query, variables: { queryString } }) - }); - - const responseBody = (await response.json()) as any; - - if (!response.ok) { - throw new Error(`GitHub API error: ${responseBody.message}`); - } - - const repositories = responseBody.data.search.edges.map( - (edge: any) => edge.node - ); - - return repositories.map( - (repo: any) => - ({ - id: repo.databaseId, - name: repo.name, - desc: repo.description, - slug: repo.name, - parent: repo.owner.login, - web_url: repo.url, - language: repo.languages, - branch: '' - }) as BaseRepo - ); - }); - - const [userRepos, orgRepos] = await Promise.all([ - getUserRepos(pat, searchString), - await Promise.all(repoPromises) - ]); - - return [...userRepos, ...orgRepos.flat()]; -} - -export { searchGithubRepos }; +}; From cab1e1e0e0b74219fbe7c707c3618da48adb7d5b Mon Sep 17 00:00:00 2001 From: e-for-eshaan Date: Wed, 15 May 2024 17:28:52 +0530 Subject: [PATCH 3/5] fixes slug name for the repo --- web-server/pages/api/internal/[org_id]/utils.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/web-server/pages/api/internal/[org_id]/utils.ts b/web-server/pages/api/internal/[org_id]/utils.ts index 32cd3f470..cfd2f96e0 100644 --- a/web-server/pages/api/internal/[org_id]/utils.ts +++ b/web-server/pages/api/internal/[org_id]/utils.ts @@ -68,7 +68,7 @@ export const searchGithubRepos = async ( id: repo.databaseId, name: repo.name, desc: repo.description, - slug: `${repo.owner.login}/${repo.name}`, + slug: repo.name, parent: repo.owner.login, web_url: repo.url, language: repo.primaryLanguage?.name, From acc66a883116372c6ac454ef86b0134d2ea7a0e1 Mon Sep 17 00:00:00 2001 From: e-for-eshaan Date: Wed, 15 May 2024 17:43:55 +0530 Subject: [PATCH 4/5] declares types for the responses associated with GH's GraphQL API --- .../pages/api/internal/[org_id]/utils.ts | 39 +++++++++++++------ 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/web-server/pages/api/internal/[org_id]/utils.ts b/web-server/pages/api/internal/[org_id]/utils.ts index cfd2f96e0..599831231 100644 --- a/web-server/pages/api/internal/[org_id]/utils.ts +++ b/web-server/pages/api/internal/[org_id]/utils.ts @@ -2,18 +2,37 @@ import { BaseRepo } from '@/types/resources'; const GITHUB_API_URL = 'https://api.github.com/graphql'; -interface Repo { - id: string; +type GithubRepo = { name: string; url: string; - isPrivate: boolean; - owner: string; -} + defaultBranchRef?: { + name: string; + }; + databaseId: string; + description?: string; + primaryLanguage?: { + name: string; + }; + owner: { + login: string; + }; +}; + +type RepoReponse = { + data: { + search: { + edges: { + node: GithubRepo; + }[]; + }; + }; + message?: string; +}; export const searchGithubRepos = async ( pat: string, searchString: string -): Promise => { +): Promise => { const query = ` query($queryString: String!) { search(type: REPOSITORY, query: $queryString, first: 50) { @@ -52,18 +71,16 @@ export const searchGithubRepos = async ( body: JSON.stringify({ query, variables: { queryString } }) }); - const responseBody = (await response.json()) as any; + const responseBody = (await response.json()) as RepoReponse; if (!response.ok) { throw new Error(`GitHub API error: ${responseBody.message}`); } - const repositories = responseBody.data.search.edges.map( - (edge: any) => edge.node - ); + const repositories = responseBody.data.search.edges.map((edge) => edge.node); return repositories.map( - (repo: any) => + (repo) => ({ id: repo.databaseId, name: repo.name, From b9d9f15017f8665cb7f55c468a29d64e3b38a99c Mon Sep 17 00:00:00 2001 From: e-for-eshaan Date: Wed, 15 May 2024 18:09:55 +0530 Subject: [PATCH 5/5] updates the default-date selection for the dora-page --- web-server/src/components/DateRangePicker/utils.ts | 11 +++++++---- web-server/src/slices/app.ts | 8 ++++---- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/web-server/src/components/DateRangePicker/utils.ts b/web-server/src/components/DateRangePicker/utils.ts index 76bc387b5..90b962ae4 100644 --- a/web-server/src/components/DateRangePicker/utils.ts +++ b/web-server/src/components/DateRangePicker/utils.ts @@ -187,9 +187,12 @@ export const presetOptions: { } ]; -export const defaultRange = - process.env.NEXT_PUBLIC_APP_ENVIRONMENT === 'development' - ? DateRangeLogic.oneWeek() - : DateRangeLogic.oneMonth(); +export const defaultDate = { + preset: 'twoWeeks', + range: DateRangeLogic.twoWeeks() +} as { + preset: QuickRangeOptions; + range: DateRange; +}; export const DATE_RANGE_MAX_DIFF = 95; diff --git a/web-server/src/slices/app.ts b/web-server/src/slices/app.ts index 25960db50..7f789f269 100644 --- a/web-server/src/slices/app.ts +++ b/web-server/src/slices/app.ts @@ -4,8 +4,8 @@ import { head, uniq } from 'ramda'; import { handleApi } from '@/api-helpers/axios-api-instance'; import { - defaultRange, - QuickRangeOptions + QuickRangeOptions, + defaultDate } from '@/components/DateRangePicker/utils'; import { Team } from '@/types/api/teams'; import { StateFetchConfig } from '@/types/redux'; @@ -72,10 +72,10 @@ const initialState: State = { errors: {}, singleTeam: [], allTeams: [], - dateRange: defaultRange.map((date) => + dateRange: defaultDate.range.map((date) => date.toISOString() ) as SerializableDateRange, - dateMode: 'oneMonth', + dateMode: defaultDate.preset, branchMode: ActiveBranchMode.ALL, branchNames: '', teamsProdBranchMap: {},