From 1bb8ef24e8d76685a599914f463b3d3a868abf9b Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Thu, 13 Apr 2023 01:16:40 +0000 Subject: [PATCH 01/24] [tests] unique ContextURL for commit only --- test/tests/workspace/git_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/tests/workspace/git_test.go b/test/tests/workspace/git_test.go index 94c4c3a805bff1..f1c1de184cd2e6 100644 --- a/test/tests/workspace/git_test.go +++ b/test/tests/workspace/git_test.go @@ -35,7 +35,7 @@ func TestGitActions(t *testing.T) { tests := []GitTest{ { Name: "create, add and commit", - ContextURL: "github.com/gitpod-io/gitpod-test-repo/tree/integration-test/commit-and-push", + ContextURL: "github.com/gitpod-io/gitpod-test-repo/tree/integration-test/commit", WorkspaceRoot: "/workspace/gitpod-test-repo", Action: func(rsa *integration.RpcClient, git integration.GitClient, workspaceRoot string) (err error) { var resp agent.ExecResponse From 173a21015236639e6a2416849b710fdb759c499d Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Thu, 13 Apr 2023 01:17:22 +0000 Subject: [PATCH 02/24] [tests] include std output in the error message for commit Sometimes commit fails with status 1 w/o error output...so it's helpful to look at StdOut --- test/pkg/integration/git-client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pkg/integration/git-client.go b/test/pkg/integration/git-client.go index 25ff5faf2cd9a1..3199ea91409ce3 100644 --- a/test/pkg/integration/git-client.go +++ b/test/pkg/integration/git-client.go @@ -123,7 +123,7 @@ func (g GitClient) Commit(dir string, message string, all bool) error { return err } if resp.ExitCode != 0 { - return fmt.Errorf("commit returned rc: %d err: %v", resp.ExitCode, resp.Stderr) + return fmt.Errorf("commit returned rc: %d, out: %v, err: %v", resp.ExitCode, resp.Stdout, resp.Stderr) } return nil } From 4ee8ed9c8d3566ac66f2ee2b012500d75a119d4f Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Thu, 13 Apr 2023 01:19:04 +0000 Subject: [PATCH 03/24] [tests] Fix TestGitActions by using the test case context Prior to this, createWorkspace was working, but getWorkspace could not find the created workspace. --- test/README.md | 5 ++++- test/pkg/integration/workspace.go | 25 +++++++++++++++++++------ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/test/README.md b/test/README.md index c8bd3d474baea4..ef60c2ae036f57 100644 --- a/test/README.md +++ b/test/README.md @@ -65,7 +65,10 @@ If you want to run an entire test suite, the easiest is to use `./test/run.sh`: ``` If you're iterating on a single test, the easiest is to use `go test` directly. -If your integration tests depends on having having a user token available, then you'll have to set `USER_TOKEN` manually (see `test/run.sh` on how to fetch the credentials that are used during our build) + +If your integration tests depends on having having a user token available, then you'll have to set `USER_NAME` and `USER_TOKEN`. This can be done a couple ways: +1. Get credentials persisted as secrets (either in Github Actions, or GCP Secret Manager via the `core-dev` project), which vary by job that trigger tests. Refer to `run.sh` for details. +2. In your Gitpod (preview) environment, create a `USER_TOKEN` via your Gitpod user's User Settings, first setup any Integrations needed for the VCS, and then make an Access Token with Full Access permissions. By default the workspace tests run against `ws-manager-mk2`, set `WS_MANAGER_MK2=false` to run against mk1. diff --git a/test/pkg/integration/workspace.go b/test/pkg/integration/workspace.go index 0a3940f6145533..0dd95b6b2854a9 100644 --- a/test/pkg/integration/workspace.go +++ b/test/pkg/integration/workspace.go @@ -348,6 +348,14 @@ func LaunchWorkspaceWithOptions(t *testing.T, ctx context.Context, opts *LaunchW defaultServerOpts = []GitpodServerOpt{WithGitpodUser(username)} } + // set IDESettings defaults, just incase the consumer hasn't + if opts.IDESettings == nil { + opts.IDESettings = &protocol.IDESettings{ + DefaultIde: "code", + UseLatestVersion: false, + } + } + parallelLimiter <- struct{}{} defer func() { if err != nil && stopWs == nil { @@ -365,7 +373,8 @@ func LaunchWorkspaceWithOptions(t *testing.T, ctx context.Context, opts *LaunchW var resp *protocol.WorkspaceCreationResult for i := 0; i < 3; i++ { - t.Logf("attemp to create the workspace: %s", opts.ContextURL) + t.Logf("attempt to create the workspace: %s, with defaultIde: %s", opts.ContextURL, opts.IDESettings.DefaultIde) + resp, err = server.CreateWorkspace(cctx, &protocol.CreateWorkspaceOptions{ ContextURL: opts.ContextURL, IgnoreRunningPrebuild: true, @@ -396,18 +405,22 @@ func LaunchWorkspaceWithOptions(t *testing.T, ctx context.Context, opts *LaunchW break } - t.Logf("attemp to get the workspace information: %s", resp.CreatedWorkspaceID) - + t.Logf("attempt to get the workspace information: %s", resp.CreatedWorkspaceID) + launchStart := time.Now() var wi *protocol.WorkspaceInfo for i := 0; i < 3; i++ { - wi, err = server.GetWorkspace(ctx, resp.CreatedWorkspaceID) + launchDuration := time.Since(launchStart) + wi, err = server.GetWorkspace(cctx, resp.CreatedWorkspaceID) if err != nil || wi.LatestInstance == nil { time.Sleep(2 * time.Second) + t.Logf("error or nil instance since %s", launchDuration) continue } if wi.LatestInstance.Status.Phase != "preparing" { + t.Logf("not preparing") break } + t.Logf("sleeping") time.Sleep(5 * time.Second) } if wi == nil || wi.LatestInstance == nil { @@ -423,7 +436,7 @@ func LaunchWorkspaceWithOptions(t *testing.T, ctx context.Context, opts *LaunchW if wi.LatestInstance.Status.Conditions.NeededImageBuild { for ctx.Err() == nil { - wi, err = server.GetWorkspace(ctx, resp.CreatedWorkspaceID) + wi, err = server.GetWorkspace(cctx, resp.CreatedWorkspaceID) if err != nil { return nil, nil, xerrors.Errorf("cannot get workspace: %w", err) } @@ -442,7 +455,7 @@ func LaunchWorkspaceWithOptions(t *testing.T, ctx context.Context, opts *LaunchW }() t.Log("wait for workspace to be fully up and running") - wsState, err := WaitForWorkspaceStart(t, ctx, wi.LatestInstance.ID, resp.CreatedWorkspaceID, api) + wsState, err := WaitForWorkspaceStart(t, cctx, wi.LatestInstance.ID, resp.CreatedWorkspaceID, api) if err != nil { return nil, nil, xerrors.Errorf("failed to wait for the workspace to start up: %w", err) } From 99aa47cff94caa0b881a8726f5405f7bd8e44aac Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Fri, 14 Apr 2023 20:42:48 +0000 Subject: [PATCH 04/24] [test] bypass exit code 1 on `git commit` with `--allow-empty` --- test/pkg/integration/git-client.go | 15 ++++++++++++--- test/tests/workspace/git_test.go | 21 +++++++++++++++------ 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/test/pkg/integration/git-client.go b/test/pkg/integration/git-client.go index 3199ea91409ce3..a1a3117e3c77f5 100644 --- a/test/pkg/integration/git-client.go +++ b/test/pkg/integration/git-client.go @@ -74,8 +74,14 @@ func (g GitClient) ConfigSafeDirectory() error { return nil } -func (g GitClient) ConfigUserName(dir string) error { - args := []string{"config", "--local", "user.name", "integration-test"} +func (g GitClient) ConfigUserName(dir string, username string) error { + var userName string + if username == "" { + userName = "integration-test" + } else { + userName = username + } + args := []string{"config", "--local", "user.name", userName} var resp agent.ExecResponse err := g.rpcClient.Call("WorkspaceAgent.Exec", &agent.ExecRequest{ Dir: dir, @@ -108,11 +114,14 @@ func (g GitClient) ConfigUserEmail(dir string, files ...string) error { return nil } -func (g GitClient) Commit(dir string, message string, all bool) error { +func (g GitClient) Commit(dir string, message string, all bool, moreArgs ...string) error { args := []string{"commit", "-m", message} if all { args = append(args, "--all") } + if moreArgs != nil { + args = append(args, moreArgs...) + } var resp agent.ExecResponse err := g.rpcClient.Call("WorkspaceAgent.Exec", &agent.ExecRequest{ Dir: dir, diff --git a/test/tests/workspace/git_test.go b/test/tests/workspace/git_test.go index f1c1de184cd2e6..5743c26657a662 100644 --- a/test/tests/workspace/git_test.go +++ b/test/tests/workspace/git_test.go @@ -6,6 +6,7 @@ package workspace import ( "context" + "fmt" "os" "testing" "time" @@ -44,17 +45,20 @@ func TestGitActions(t *testing.T) { Command: "bash", Args: []string{ "-c", - "echo \"another test run...\" >> file_to_commit.txt", + "touch file_to_commit.txt", }, }, &resp) if err != nil { return err } + if resp.ExitCode != 0 { + return fmt.Errorf("file create returned rc: %d, out: %v, err: %v", resp.ExitCode, resp.Stdout, resp.Stderr) + } err = git.ConfigSafeDirectory() if err != nil { return err } - err = git.ConfigUserName(workspaceRoot) + err = git.ConfigUserName(workspaceRoot, username) if err != nil { return err } @@ -66,7 +70,7 @@ func TestGitActions(t *testing.T) { if err != nil { return err } - err = git.Commit(workspaceRoot, "automatic test commit", false) + err = git.Commit(workspaceRoot, "automatic test commit", false, "--allow-empty") if err != nil { return err } @@ -74,6 +78,8 @@ func TestGitActions(t *testing.T) { }, }, { + // as of Apr 14, 2023, test fails with: + // fatal: could not read Username for 'https://github.com': No such device or address Skip: true, Name: "create, add and commit and PUSH", ContextURL: "github.com/gitpod-io/gitpod-test-repo/tree/integration-test/commit-and-push", @@ -85,17 +91,20 @@ func TestGitActions(t *testing.T) { Command: "bash", Args: []string{ "-c", - "echo \"another test run...\" >> file_to_commit.txt", + "touch file_to_commit.txt", }, }, &resp) if err != nil { return err } + if resp.ExitCode != 0 { + return fmt.Errorf("file create returned rc: %d, out: %v, err: %v", resp.ExitCode, resp.Stdout, resp.Stderr) + } err = git.ConfigSafeDirectory() if err != nil { return err } - err = git.ConfigUserName(workspaceRoot) + err = git.ConfigUserName(workspaceRoot, username) if err != nil { return err } @@ -107,7 +116,7 @@ func TestGitActions(t *testing.T) { if err != nil { return err } - err = git.Commit(workspaceRoot, "automatic test commit", false) + err = git.Commit(workspaceRoot, "automatic test commit", false, "--allow-empty") if err != nil { return err } From a3b493762a2527ec19064a64d32b1ad4439cfdda Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Sat, 15 Apr 2023 02:44:13 +0000 Subject: [PATCH 05/24] [test] Fix gcloud auth, do setup before auth --- .github/workflows/workspace-integration-tests.yml | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/.github/workflows/workspace-integration-tests.yml b/.github/workflows/workspace-integration-tests.yml index 83f8f8e9b5582a..c1242bbd2470fc 100644 --- a/.github/workflows/workspace-integration-tests.yml +++ b/.github/workflows/workspace-integration-tests.yml @@ -38,21 +38,14 @@ jobs: steps: # sometimes auth fails with: # google-github-actions/setup-gcloud failed with: EACCES: permission denied, mkdir '/__t/gcloud' + - name: Set up Cloud SDK + uses: google-github-actions/setup-gcloud@v1 - id: auth uses: google-github-actions/auth@v1 continue-on-error: true with: token_format: access_token credentials_json: "${{ secrets.GCP_CREDENTIALS }}" - # so we retry on failure - - id: auth-retry - uses: google-github-actions/auth@v1 - if: steps.auth.outcome == 'failure' - with: - token_format: access_token - credentials_json: "${{ secrets.GCP_CREDENTIALS }}" - - name: Set up Cloud SDK - uses: google-github-actions/setup-gcloud@v1 # do this step as early as possible, so that Slack Notify failure has the secret - name: Get Secrets from GCP id: "secrets" From ca75e37a030204ed602ca91ec097d9668119ca54 Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Wed, 19 Apr 2023 20:34:54 +0000 Subject: [PATCH 06/24] Show all output Might remove later... --- .github/workflows/workspace-integration-tests.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/workspace-integration-tests.yml b/.github/workflows/workspace-integration-tests.yml index c1242bbd2470fc..de3faad86ce8d8 100644 --- a/.github/workflows/workspace-integration-tests.yml +++ b/.github/workflows/workspace-integration-tests.yml @@ -233,6 +233,7 @@ jobs: uses: test-summary/action@v2 with: paths: "test/tests/**/TEST-*.xml" + show: "all" if: always() - name: Slack Notification uses: rtCamp/action-slack-notify@v2 From 7d60c2852ef0ec1038f58dae302517306c19d8d9 Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Sun, 23 Apr 2023 18:46:08 +0000 Subject: [PATCH 07/24] [test] avoid using deleted user, identity, and token --- test/pkg/integration/apis.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/pkg/integration/apis.go b/test/pkg/integration/apis.go index 68d143ad865268..afbf33d17b4a99 100644 --- a/test/pkg/integration/apis.go +++ b/test/pkg/integration/apis.go @@ -459,7 +459,7 @@ func (c *ComponentAPI) CreateUser(username string, token string) (string, error) } var userId string - err = db.QueryRow(`SELECT id FROM d_b_user WHERE name = ?`, username).Scan(&userId) + err = db.QueryRow(`SELECT id FROM d_b_user WHERE name = ? and markedDeleted != 1 and blocked != 1`, username).Scan(&userId) if err != nil && !errors.Is(err, sql.ErrNoRows) { return "", err } @@ -485,7 +485,7 @@ func (c *ComponentAPI) CreateUser(username string, token string) (string, error) } var authId string - err = db.QueryRow(`SELECT authId FROM d_b_identity WHERE userId = ?`, userId).Scan(&authId) + err = db.QueryRow(`SELECT authId FROM d_b_identity WHERE userId = ? and deleted != 1`, userId).Scan(&authId) if err != nil && !errors.Is(err, sql.ErrNoRows) { return "", err } @@ -504,7 +504,7 @@ func (c *ComponentAPI) CreateUser(username string, token string) (string, error) } var cnt int - err = db.QueryRow(`SELECT COUNT(1) AS cnt FROM d_b_token_entry WHERE authId = ?`, authId).Scan(&cnt) + err = db.QueryRow(`SELECT COUNT(1) AS cnt FROM d_b_token_entry WHERE authId = ? and deleted != 1`, authId).Scan(&cnt) if err != nil && !errors.Is(err, sql.ErrNoRows) { return "", err } From c216ff6c392bd0903009a40c00311d0d2451df0d Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Sun, 23 Apr 2023 18:47:57 +0000 Subject: [PATCH 08/24] [test] add organizationId to CreateWorkspaceOptions It's expected on the Typescript side: https://github.com/gitpod-io/gitpod/blob/90b7d658589fd6e9fb239ff239591b9f1218fc83/components/server/src/workspace/gitpod-server-impl.ts#L1227-L1235 --- components/gitpod-protocol/go/gitpod-service.go | 6 +++++- test/pkg/integration/workspace.go | 13 ++++--------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/components/gitpod-protocol/go/gitpod-service.go b/components/gitpod-protocol/go/gitpod-service.go index 58c55b0b91c8d5..8f5fef9a87f995 100644 --- a/components/gitpod-protocol/go/gitpod-service.go +++ b/components/gitpod-protocol/go/gitpod-service.go @@ -2052,7 +2052,11 @@ type UpdateOwnAuthProviderParams struct { // CreateWorkspaceOptions is the CreateWorkspaceOptions message type type CreateWorkspaceOptions struct { StartWorkspaceOptions - ContextURL string `json:"contextUrl,omitempty"` + ContextURL string `json:"contextUrl,omitempty"` + // references: + // https://github.com/gitpod-io/gitpod/pull/17040 + // https://github.com/gitpod-io/gitpod/blob/90b7d658589fd6e9fb239ff239591b9f1218fc83/components/server/src/workspace/gitpod-server-impl.ts#L1227-L1235 + OrganizationId string `json:"organizationId,omitempty"` IgnoreRunningWorkspaceOnSameCommit bool `json:"ignoreRunningWorkspaceOnSameCommit,omitemopty"` IgnoreRunningPrebuild bool `json:"ignoreRunningPrebuild,omitemopty"` AllowUsingPreviousPrebuilds bool `json:"allowUsingPreviousPrebuilds,omitemopty"` diff --git a/test/pkg/integration/workspace.go b/test/pkg/integration/workspace.go index 0dd95b6b2854a9..e365e27b85b579 100644 --- a/test/pkg/integration/workspace.go +++ b/test/pkg/integration/workspace.go @@ -348,14 +348,6 @@ func LaunchWorkspaceWithOptions(t *testing.T, ctx context.Context, opts *LaunchW defaultServerOpts = []GitpodServerOpt{WithGitpodUser(username)} } - // set IDESettings defaults, just incase the consumer hasn't - if opts.IDESettings == nil { - opts.IDESettings = &protocol.IDESettings{ - DefaultIde: "code", - UseLatestVersion: false, - } - } - parallelLimiter <- struct{}{} defer func() { if err != nil && stopWs == nil { @@ -373,10 +365,13 @@ func LaunchWorkspaceWithOptions(t *testing.T, ctx context.Context, opts *LaunchW var resp *protocol.WorkspaceCreationResult for i := 0; i < 3; i++ { - t.Logf("attempt to create the workspace: %s, with defaultIde: %s", opts.ContextURL, opts.IDESettings.DefaultIde) + u, _ := api.GetUserId(username) + t.Logf("attempt to create the workspace as user %v, with context %v\n", u, opts.ContextURL) + teams, _ := server.GetTeams(cctx) resp, err = server.CreateWorkspace(cctx, &protocol.CreateWorkspaceOptions{ ContextURL: opts.ContextURL, + OrganizationId: teams[0].ID, IgnoreRunningPrebuild: true, IgnoreRunningWorkspaceOnSameCommit: true, StartWorkspaceOptions: protocol.StartWorkspaceOptions{ From 4f66addf9264394689e020e030a3a48eb60de008 Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Mon, 24 Apr 2023 01:37:34 +0000 Subject: [PATCH 09/24] [test] orgId is required on createWorkspace But sometimes there's no team :shrug: --- test/pkg/integration/workspace.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/pkg/integration/workspace.go b/test/pkg/integration/workspace.go index e365e27b85b579..0564052f2cc75d 100644 --- a/test/pkg/integration/workspace.go +++ b/test/pkg/integration/workspace.go @@ -369,9 +369,17 @@ func LaunchWorkspaceWithOptions(t *testing.T, ctx context.Context, opts *LaunchW t.Logf("attempt to create the workspace as user %v, with context %v\n", u, opts.ContextURL) teams, _ := server.GetTeams(cctx) + var orgId string + if len(teams) == 0 { + // hack: there might be a better value to use here + orgId = u + } else { + orgId = teams[0].ID + } + resp, err = server.CreateWorkspace(cctx, &protocol.CreateWorkspaceOptions{ ContextURL: opts.ContextURL, - OrganizationId: teams[0].ID, + OrganizationId: orgId, IgnoreRunningPrebuild: true, IgnoreRunningWorkspaceOnSameCommit: true, StartWorkspaceOptions: protocol.StartWorkspaceOptions{ From 0bd754ed4ee0f20bc6360116ffc099167b63759e Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Mon, 24 Apr 2023 19:27:53 +0000 Subject: [PATCH 10/24] [test] fix git context tests We use UBP now, there is no more unleashed. Also, remove the "ff" feature flag code (which was for PVC). It was mutating the username, resulting in Code 460 errors on createWorkspace --- test/tests/workspace/contexts_test.go | 131 +++++++++++--------------- 1 file changed, 53 insertions(+), 78 deletions(-) diff --git a/test/tests/workspace/contexts_test.go b/test/tests/workspace/contexts_test.go index 7010a4572af171..c965b5d831d8ef 100644 --- a/test/tests/workspace/contexts_test.go +++ b/test/tests/workspace/contexts_test.go @@ -111,102 +111,77 @@ func runContextTests(t *testing.T, tests []ContextTest) { f := features.New("context"). WithLabel("component", "server"). Assess("should run context tests", func(ctx context.Context, t *testing.T, cfg *envconf.Config) context.Context { - ffs := []struct { - Name string - FF string - }{ - {Name: "classic"}, - // {Name: "pvc", FF: "persistent_volume_claim"}, - } - for _, ff := range ffs { - func() { - sctx, scancel := context.WithTimeout(context.Background(), 5*time.Minute) - defer scancel() + sctx, scancel := context.WithTimeout(context.Background(), time.Duration(5*len(tests))*time.Minute) + defer scancel() - api := integration.NewComponentAPI(sctx, cfg.Namespace(), kubeconfig, cfg.Client()) - defer api.Done(t) + api := integration.NewComponentAPI(sctx, cfg.Namespace(), kubeconfig, cfg.Client()) + defer api.Done(t) - username := username + ff.Name - userId, err := api.CreateUser(username, userToken) - if err != nil { - t.Fatal(err) + for _, test := range tests { + test := test + t.Run(test.ContextURL, func(t *testing.T) { + report.SetupReport(t, report.FeatureContentInit, fmt.Sprintf("Test to open %v", test.ContextURL)) + if test.Skip { + t.SkipNow() } - if err := api.UpdateUserFeatureFlag(userId, ff.FF); err != nil { - t.Fatal(err) - } - }() - } + t.Parallel() - for _, ff := range ffs { - for _, test := range tests { - test := test - t.Run(test.ContextURL+"_"+ff.Name, func(t *testing.T) { - report.SetupReport(t, report.FeatureContentInit, fmt.Sprintf("Test to open %v", test.ContextURL)) - if test.Skip { - t.SkipNow() - } + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*len(tests))*time.Minute) + defer cancel() - t.Parallel() + api := integration.NewComponentAPI(ctx, cfg.Namespace(), kubeconfig, cfg.Client()) + defer api.Done(t) - username := username + ff.Name + nfo, stopWs, err := integration.LaunchWorkspaceFromContextURL(t, ctx, test.ContextURL, username, api) + if err != nil { + t.Fatal(err) + } - ctx, cancel := context.WithTimeout(context.Background(), time.Duration(5*len(tests)*len(ffs))*time.Minute) - defer cancel() + t.Cleanup(func() { + sctx, scancel := context.WithTimeout(context.Background(), 10*time.Minute) + defer scancel() - api := integration.NewComponentAPI(ctx, cfg.Namespace(), kubeconfig, cfg.Client()) - defer api.Done(t) + sapi := integration.NewComponentAPI(sctx, cfg.Namespace(), kubeconfig, cfg.Client()) + defer sapi.Done(t) - nfo, stopWs, err := integration.LaunchWorkspaceFromContextURL(t, ctx, test.ContextURL, username, api) + _, err := stopWs(true, sapi) if err != nil { t.Fatal(err) } + }) - t.Cleanup(func() { - sctx, scancel := context.WithTimeout(context.Background(), 10*time.Minute) - defer scancel() - - sapi := integration.NewComponentAPI(sctx, cfg.Namespace(), kubeconfig, cfg.Client()) - defer sapi.Done(t) - - _, err := stopWs(true, sapi) - if err != nil { - t.Fatal(err) - } - }) - - rsa, closer, err := integration.Instrument(integration.ComponentWorkspace, "workspace", cfg.Namespace(), kubeconfig, cfg.Client(), integration.WithInstanceID(nfo.LatestInstance.ID)) - if err != nil { - t.Fatal(err) - } - defer rsa.Close() - integration.DeferCloser(t, closer) + rsa, closer, err := integration.Instrument(integration.ComponentWorkspace, "workspace", cfg.Namespace(), kubeconfig, cfg.Client(), integration.WithInstanceID(nfo.LatestInstance.ID)) + if err != nil { + t.Fatal(err) + } + defer rsa.Close() + integration.DeferCloser(t, closer) - if test.ExpectedBranch == "" && test.ExpectedBranchFunc == nil { - return - } + if test.ExpectedBranch == "" && test.ExpectedBranchFunc == nil { + return + } - // get actual from workspace - git := integration.Git(rsa) - err = git.ConfigSafeDirectory() - if err != nil { - t.Fatal(err) - } - actBranch, err := git.GetBranch(test.WorkspaceRoot, test.IgnoreError) - if err != nil { - t.Fatal(err) - } + // get actual from workspace + git := integration.Git(rsa) + err = git.ConfigSafeDirectory() + if err != nil { + t.Fatal(err) + } + actBranch, err := git.GetBranch(test.WorkspaceRoot, test.IgnoreError) + if err != nil { + t.Fatal(err) + } - expectedBranch := test.ExpectedBranch - if test.ExpectedBranchFunc != nil { - expectedBranch = test.ExpectedBranchFunc(username) - } - if actBranch != expectedBranch { - t.Fatalf("expected branch '%s', got '%s'!", expectedBranch, actBranch) - } - }) - } + expectedBranch := test.ExpectedBranch + if test.ExpectedBranchFunc != nil { + expectedBranch = test.ExpectedBranchFunc(username) + } + if actBranch != expectedBranch { + t.Fatalf("expected branch '%s', got '%s'!", expectedBranch, actBranch) + } + }) } return ctx }). From 1257b4c28be8a0b7867095b29c3d6be7c642ecc5 Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Mon, 24 Apr 2023 19:30:59 +0000 Subject: [PATCH 11/24] [test] Use example test as the example --- test/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/README.md b/test/README.md index ef60c2ae036f57..9514db2c44b176 100644 --- a/test/README.md +++ b/test/README.md @@ -90,7 +90,7 @@ cd test go test -v ./... \ -kubeconfig=/home/gitpod/.kube/config \ -namespace=default \ - -run TestAdminBlockUser + -run TestWorkspaceInstrumentation ``` # Tips From 5f59a1a177769d904e21649c91106b51f2420ba5 Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Mon, 24 Apr 2023 20:49:02 +0000 Subject: [PATCH 12/24] [test] fix context tests when run as gitpod-integration-test user --- test/tests/workspace/contexts_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/tests/workspace/contexts_test.go b/test/tests/workspace/contexts_test.go index c965b5d831d8ef..1879cfb0cb1c46 100644 --- a/test/tests/workspace/contexts_test.go +++ b/test/tests/workspace/contexts_test.go @@ -134,6 +134,11 @@ func runContextTests(t *testing.T, tests []ContextTest) { api := integration.NewComponentAPI(ctx, cfg.Namespace(), kubeconfig, cfg.Client()) defer api.Done(t) + _, err := api.CreateUser(username, userToken) + if err != nil { + t.Fatal(err) + } + nfo, stopWs, err := integration.LaunchWorkspaceFromContextURL(t, ctx, test.ContextURL, username, api) if err != nil { t.Fatal(err) From 05a9bb38be95e4c4bc7601e2b47af01af50e6397 Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Mon, 24 Apr 2023 20:49:17 +0000 Subject: [PATCH 13/24] [test] clean-up --- test/pkg/integration/apis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/pkg/integration/apis.go b/test/pkg/integration/apis.go index afbf33d17b4a99..eeb349b92b7a10 100644 --- a/test/pkg/integration/apis.go +++ b/test/pkg/integration/apis.go @@ -416,7 +416,7 @@ func (c *ComponentAPI) GetUserId(user string) (userId string, err error) { if user == "" { row = db.QueryRow(`SELECT id FROM d_b_user WHERE NOT id = "` + gitpodBuiltinUserID + `" AND blocked = FALSE AND markedDeleted = FALSE`) } else { - row = db.QueryRow("SELECT id FROM d_b_user WHERE name = ?", user) + row = db.QueryRow("SELECT id FROM d_b_user WHERE name = ? AND blocked != 1 and markedDeleted != 1", user) } var id string From 242093eaf43d1831f848636fbb653c77c84067cb Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Mon, 24 Apr 2023 21:12:50 +0000 Subject: [PATCH 14/24] [test] wait for workspaces to stop Tests intermittently fail with to avoid intermittent failures --- test/pkg/integration/workspace.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/pkg/integration/workspace.go b/test/pkg/integration/workspace.go index 0564052f2cc75d..2c6deb494d9005 100644 --- a/test/pkg/integration/workspace.go +++ b/test/pkg/integration/workspace.go @@ -297,7 +297,7 @@ func LaunchWorkspaceDirectly(t *testing.T, ctx context.Context, api *ComponentAP stopWs = stopWsF(t, req.Id, req.Metadata.MetaId, api, req.Type == wsmanapi.WorkspaceType_PREBUILD) defer func() { if err != nil { - _, _ = stopWs(false, api) + _, _ = stopWs(true, api) } }() @@ -453,7 +453,7 @@ func LaunchWorkspaceWithOptions(t *testing.T, ctx context.Context, opts *LaunchW stopWs = stopWsF(t, wi.LatestInstance.ID, resp.CreatedWorkspaceID, api, false) defer func() { if err != nil { - _, _ = stopWs(false, api) + _, _ = stopWs(true, api) } }() From 9b01eb48afeb2c7689987b911f41fd6ec6436512 Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Mon, 24 Apr 2023 21:24:39 +0000 Subject: [PATCH 15/24] [test] add code owners This way, we can assert tests are passing for all teams prior to merging --- .github/CODEOWNERS | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 9c9605dd27a45a..9fa421f99000f4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -128,3 +128,18 @@ /CHANGELOG.md /components/ide/jetbrains/backend-plugin/gradle-latest.properties /components/ide/jetbrains/gateway-plugin/gradle-latest.properties + +# +# Add so that teams assert we're not breaking each other's integration tests +/test/pkg/agent @gitpod-io/engineering-workspace +/test/pkg/integration @gitpod-io/engineering-ide @gitpod-io/engineering-workspace +/test/pkg/report @gitpod-io/engineering-workspace +/test/tests/workspace @gitpod-io/engineering-workspace +/test/tests/smoke-test @gitpod-io/engineering-ide @gitpod-io/engineering-workspace +/test/tests/ide @gitpod-io/engineering-ide +/test/tests/components/content-service @gitpod-io/engineering-workspace +/test/tests/components/database @gitpod-io/engineering-webapp +/test/tests/components/image-builder @gitpod-io/engineering-workspace +/test/tests/components/server @gitpod-io/engineering-webapp +/test/tests/components/ws-daemon @gitpod-io/engineering-workspace +/test/tests/components/ws-manager @gitpod-io/engineering-workspace From fe3c251d6411d225ac45d45968bdf7bebb260759 Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Wed, 26 Apr 2023 02:19:25 +0000 Subject: [PATCH 16/24] [test] limit # of tests that can run in parallel --- .github/workflows/workspace-integration-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/workspace-integration-tests.yml b/.github/workflows/workspace-integration-tests.yml index de3faad86ce8d8..81f731bda08491 100644 --- a/.github/workflows/workspace-integration-tests.yml +++ b/.github/workflows/workspace-integration-tests.yml @@ -218,7 +218,7 @@ jobs: fi set +e - go test -p 2 -v ./... "${args[@]}" -run '.*[^.SerialOnly]$' 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "TEST-${TEST_NAME}-PARALLEL.xml" -iocopy + go test -p 2 --parallel 2 -v ./... "${args[@]}" -run '.*[^.SerialOnly]$' 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "TEST-${TEST_NAME}-PARALLEL.xml" -iocopy RC=${PIPESTATUS[0]} set -e From 4be112368b6083d23f6a5ea9c901d04771e718cc Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Wed, 26 Apr 2023 03:54:29 +0000 Subject: [PATCH 17/24] [test] no parallel tests Test to see if flakeyness goes away... ...and bump the timeout because we reduced parallel runs --- .github/workflows/workspace-integration-tests.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/workspace-integration-tests.yml b/.github/workflows/workspace-integration-tests.yml index 81f731bda08491..20b4714ab1bc28 100644 --- a/.github/workflows/workspace-integration-tests.yml +++ b/.github/workflows/workspace-integration-tests.yml @@ -186,7 +186,7 @@ jobs: args+=( "-kubeconfig=/home/gitpod/.kube/config" ) args+=( "-namespace=default" ) [[ "$USERNAME" != "" ]] && args+=( "-username=$USERNAME" ) - args+=( "-timeout=60m" ) + args+=( "-timeout=90m" ) BASE_TESTS_DIR="$GITHUB_WORKSPACE/test/tests" CONTENT_SERVICE_TESTS="$BASE_TESTS_DIR/components/content-service" @@ -218,7 +218,8 @@ jobs: fi set +e - go test -p 2 --parallel 2 -v ./... "${args[@]}" -run '.*[^.SerialOnly]$' 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "TEST-${TEST_NAME}-PARALLEL.xml" -iocopy + # running tests in parallel saves time, but is flakey. + go test -p 1 --parallel 1 -v ./... "${args[@]}" -run '.*[^.SerialOnly]$' 2>&1 | go-junit-report -subtest-mode=exclude-parents -set-exit-code -out "TEST-${TEST_NAME}-PARALLEL.xml" -iocopy RC=${PIPESTATUS[0]} set -e From 15aa2c808d677c7ed5b8a1ea34e758215158f3f0 Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Wed, 26 Apr 2023 16:54:47 +0000 Subject: [PATCH 18/24] [preview] update the VM image to have parity with production This: 1. updates from K3s 1.23 to 1.26 2. requires that we remove PodSecurityPolicy changes (as it's no longer supported) 3. resolves intermittent disk pressure issues --- dev/preview/BUILD.yaml | 2 +- dev/preview/infrastructure/modules/gce/variables.tf | 2 +- dev/preview/infrastructure/modules/harvester/variables.tf | 2 +- dev/preview/infrastructure/variables.tf | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dev/preview/BUILD.yaml b/dev/preview/BUILD.yaml index 57e3e98be41bb6..d7c48073d4f213 100644 --- a/dev/preview/BUILD.yaml +++ b/dev/preview/BUILD.yaml @@ -33,7 +33,7 @@ scripts: export TF_VAR_preview_name="${TF_VAR_preview_name:-$(previewctl get name)}" export TF_VAR_vm_cpu="${TF_VAR_vm_cpu:-6}" export TF_VAR_vm_memory="${TF_VAR_vm_memory:-12Gi}" - export TF_VAR_vm_storage_class="${TF_VAR_vm_storage_class:-longhorn-gitpod-k3s-202209251218-onereplica}" + export TF_VAR_vm_storage_class="${TF_VAR_vm_storage_class:-longhorn-gitpod-k3s-202304191605-onereplica}" ./workflow/preview/deploy-harvester.sh - name: delete-preview diff --git a/dev/preview/infrastructure/modules/gce/variables.tf b/dev/preview/infrastructure/modules/gce/variables.tf index e9723589f7f674..e8fb8866365e62 100644 --- a/dev/preview/infrastructure/modules/gce/variables.tf +++ b/dev/preview/infrastructure/modules/gce/variables.tf @@ -44,7 +44,7 @@ variable "harvester_ingress_ip" { variable "vm_image" { type = string description = "The VM image" - default = "gitpod-k3s-202302132222" + default = "gitpod-k3s-202304191605" } variable "cert_issuer" { diff --git a/dev/preview/infrastructure/modules/harvester/variables.tf b/dev/preview/infrastructure/modules/harvester/variables.tf index 11416d9bde21dc..eb210d6435ad0a 100644 --- a/dev/preview/infrastructure/modules/harvester/variables.tf +++ b/dev/preview/infrastructure/modules/harvester/variables.tf @@ -33,7 +33,7 @@ variable "ssh_key" { variable "vm_image" { type = string description = "The VM image" - default = "gitpod-k3s-202209251218" + default = "gitpod-k3s-202304191605" } variable "harvester_ingress_ip" { diff --git a/dev/preview/infrastructure/variables.tf b/dev/preview/infrastructure/variables.tf index 738c73807ae886..79c0d32cd69ec8 100644 --- a/dev/preview/infrastructure/variables.tf +++ b/dev/preview/infrastructure/variables.tf @@ -35,7 +35,7 @@ variable "vm_type" { variable "vm_image" { type = string description = "The VM image" - default = "gitpod-k3s-202209251218" + default = "gitpod-k3s-202304191605" } variable "cert_issuer" { From 96b3e8487b6ef26da0ad054726445a29a058401e Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Wed, 26 Apr 2023 17:56:54 +0000 Subject: [PATCH 19/24] [preview] no PSP in support of VM image update * We were getting PSP from rook/ceph, which I think was for PVC * We were getting PSP from the monitoring-satellite --- .werft/vm/manifests/rook-ceph/common.yaml | 403 +++++++----------- .../preview/deploy-monitoring-satellite.sh | 3 + 2 files changed, 160 insertions(+), 246 deletions(-) diff --git a/.werft/vm/manifests/rook-ceph/common.yaml b/.werft/vm/manifests/rook-ceph/common.yaml index ce88bac708ef22..f09eb72d46c350 100644 --- a/.werft/vm/manifests/rook-ceph/common.yaml +++ b/.werft/vm/manifests/rook-ceph/common.yaml @@ -80,24 +80,24 @@ rules: resources: ["volumesnapshots/status"] verbs: ["update", "patch"] --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: 'psp:rook' - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/part-of: rook-ceph-operator -rules: - - apiGroups: - - policy - resources: - - podsecuritypolicies - resourceNames: - - 00-rook-privileged - verbs: - - use ---- +# apiVersion: rbac.authorization.k8s.io/v1 +# kind: ClusterRole +# metadata: +# name: 'psp:rook' +# labels: +# operator: rook +# storage-backend: ceph +# app.kubernetes.io/part-of: rook-ceph-operator +# rules: +# - apiGroups: +# - policy +# resources: +# - podsecuritypolicies +# resourceNames: +# - 00-rook-privileged +# verbs: +# - use +# --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: @@ -700,76 +700,75 @@ subjects: - kind: ServiceAccount name: rook-ceph-system namespace: rook-ceph # namespace:operator ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-ceph-system-psp - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/part-of: rook-ceph-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-ceph-system - namespace: rook-ceph # namespace:operator ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-cephfs-plugin-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-plugin-sa - namespace: rook-ceph # namespace:operator ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-cephfs-provisioner-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-cephfs-provisioner-sa - namespace: rook-ceph # namespace:operator ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-rbd-plugin-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-plugin-sa - namespace: rook-ceph # namespace:operator ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: rook-csi-rbd-provisioner-sa-psp -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: 'psp:rook' -subjects: - - kind: ServiceAccount - name: rook-csi-rbd-provisioner-sa - namespace: rook-ceph # namespace:operator ---- +# apiVersion: rbac.authorization.k8s.io/v1 +# kind: ClusterRoleBinding +# metadata: +# name: rook-ceph-system-psp +# labels: +# operator: rook +# storage-backend: ceph +# app.kubernetes.io/part-of: rook-ceph-operator +# roleRef: +# apiGroup: rbac.authorization.k8s.io +# kind: ClusterRole +# name: 'psp:rook' +# subjects: +# - kind: ServiceAccount +# name: rook-ceph-system +# namespace: rook-ceph # namespace:operator +# --- +# apiVersion: rbac.authorization.k8s.io/v1 +# kind: ClusterRoleBinding +# metadata: +# name: rook-csi-cephfs-plugin-sa-psp +# roleRef: +# apiGroup: rbac.authorization.k8s.io +# kind: ClusterRole +# name: 'psp:rook' +# subjects: +# - kind: ServiceAccount +# name: rook-csi-cephfs-plugin-sa +# namespace: rook-ceph # namespace:operator +# --- +# apiVersion: rbac.authorization.k8s.io/v1 +# kind: ClusterRoleBinding +# metadata: +# name: rook-csi-cephfs-provisioner-sa-psp +# roleRef: +# apiGroup: rbac.authorization.k8s.io +# kind: ClusterRole +# name: 'psp:rook' +# subjects: +# - kind: ServiceAccount +# name: rook-csi-cephfs-provisioner-sa +# namespace: rook-ceph # namespace:operator +# --- +# apiVersion: rbac.authorization.k8s.io/v1 +# kind: ClusterRoleBinding +# metadata: +# name: rook-csi-rbd-plugin-sa-psp +# roleRef: +# apiGroup: rbac.authorization.k8s.io +# kind: ClusterRole +# name: 'psp:rook' +# subjects: +# - kind: ServiceAccount +# name: rook-csi-rbd-plugin-sa +# namespace: rook-ceph # namespace:operator +# --- +# apiVersion: rbac.authorization.k8s.io/v1 +# kind: ClusterRoleBinding +# metadata: +# name: rook-csi-rbd-provisioner-sa-psp +# roleRef: +# apiGroup: rbac.authorization.k8s.io +# kind: ClusterRole +# name: 'psp:rook' +# subjects: +# - kind: ServiceAccount +# name: rook-csi-rbd-provisioner-sa +# namespace: rook-ceph # namespace:operator +# --- # We expect most Kubernetes teams to follow the Kubernetes docs and have these PSPs. # * privileged (for kube-system namespace) # * restricted (for all logged in users) @@ -780,76 +779,76 @@ subjects: # environments with other `00`-prefixed PSPs. # # More on PSP ordering: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#policy-order -apiVersion: policy/v1beta1 -kind: PodSecurityPolicy -metadata: - name: 00-rook-privileged - annotations: - seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' - seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' -spec: - privileged: true - allowedCapabilities: - # required by CSI - - SYS_ADMIN - - MKNOD - fsGroup: - rule: RunAsAny - # runAsUser, supplementalGroups - Rook needs to run some pods as root - # Ceph pods could be run as the Ceph user, but that user isn't always known ahead of time - runAsUser: - rule: RunAsAny - supplementalGroups: - rule: RunAsAny - # seLinux - seLinux context is unknown ahead of time; set if this is well-known - seLinux: - rule: RunAsAny - volumes: - # recommended minimum set - - configMap - - downwardAPI - - emptyDir - - persistentVolumeClaim - - secret - - projected - # required for Rook - - hostPath - # allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known - # allowedHostPaths: - # - pathPrefix: "/run/udev" # for OSD prep - # readOnly: false - # - pathPrefix: "/dev" # for OSD prep - # readOnly: false - # - pathPrefix: "/var/lib/rook" # or whatever the dataDirHostPath value is set to - # readOnly: false - # Ceph requires host IPC for setting up encrypted devices - hostIPC: true - # Ceph OSDs need to share the same PID namespace - hostPID: true - # hostNetwork can be set to 'false' if host networking isn't used - hostNetwork: true - hostPorts: - # Ceph messenger protocol v1 - - min: 6789 - max: 6790 # <- support old default port - # Ceph messenger protocol v2 - - min: 3300 - max: 3300 - # Ceph RADOS ports for OSDs, MDSes - - min: 6800 - max: 7300 - # # Ceph dashboard port HTTP (not recommended) - # - min: 7000 - # max: 7000 - # Ceph dashboard port HTTPS - - min: 8443 - max: 8443 - # Ceph mgr Prometheus Metrics - - min: 9283 - max: 9283 - # port for CSIAddons - - min: 9070 - max: 9070 +# apiVersion: policy/v1beta1 +# kind: PodSecurityPolicy +# metadata: +# name: 00-rook-privileged +# annotations: +# seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' +# seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' +# spec: +# privileged: true +# allowedCapabilities: +# # required by CSI +# - SYS_ADMIN +# - MKNOD +# fsGroup: +# rule: RunAsAny +# # runAsUser, supplementalGroups - Rook needs to run some pods as root +# # Ceph pods could be run as the Ceph user, but that user isn't always known ahead of time +# runAsUser: +# rule: RunAsAny +# supplementalGroups: +# rule: RunAsAny +# # seLinux - seLinux context is unknown ahead of time; set if this is well-known +# seLinux: +# rule: RunAsAny +# volumes: +# # recommended minimum set +# - configMap +# - downwardAPI +# - emptyDir +# - persistentVolumeClaim +# - secret +# - projected +# # required for Rook +# - hostPath +# # allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known +# # allowedHostPaths: +# # - pathPrefix: "/run/udev" # for OSD prep +# # readOnly: false +# # - pathPrefix: "/dev" # for OSD prep +# # readOnly: false +# # - pathPrefix: "/var/lib/rook" # or whatever the dataDirHostPath value is set to +# # readOnly: false +# # Ceph requires host IPC for setting up encrypted devices +# hostIPC: true +# # Ceph OSDs need to share the same PID namespace +# hostPID: true +# # hostNetwork can be set to 'false' if host networking isn't used +# hostNetwork: true +# hostPorts: +# # Ceph messenger protocol v1 +# - min: 6789 +# max: 6790 # <- support old default port +# # Ceph messenger protocol v2 +# - min: 3300 +# max: 3300 +# # Ceph RADOS ports for OSDs, MDSes +# - min: 6800 +# max: 7300 +# # # Ceph dashboard port HTTP (not recommended) +# # - min: 7000 +# # max: 7000 +# # Ceph dashboard port HTTPS +# - min: 8443 +# max: 8443 +# # Ceph mgr Prometheus Metrics +# - min: 9283 +# max: 9283 +# # port for CSIAddons +# - min: 9070 +# max: 9070 --- kind: Role apiVersion: rbac.authorization.k8s.io/v1 @@ -1147,38 +1146,6 @@ subjects: name: rook-ceph-cmd-reporter namespace: rook-ceph # namespace:cluster --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-cmd-reporter-psp - namespace: rook-ceph # namespace:cluster -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: - - kind: ServiceAccount - name: rook-ceph-cmd-reporter - namespace: rook-ceph # namespace:cluster ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-default-psp - namespace: rook-ceph # namespace:cluster - labels: - operator: rook - storage-backend: ceph - app.kubernetes.io/part-of: rook-ceph-operator -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: - - kind: ServiceAccount - name: default - namespace: rook-ceph # namespace:cluster ---- # Allow the ceph mgr to access resources scoped to the CephCluster namespace necessary for mgr modules kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -1194,20 +1161,6 @@ subjects: name: rook-ceph-mgr namespace: rook-ceph # namespace:cluster --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-mgr-psp - namespace: rook-ceph # namespace:cluster -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: - - kind: ServiceAccount - name: rook-ceph-mgr - namespace: rook-ceph # namespace:cluster ---- # Allow the ceph mgr to access resources in the Rook operator namespace necessary for mgr modules kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -1238,20 +1191,6 @@ subjects: name: rook-ceph-osd namespace: rook-ceph # namespace:cluster --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-osd-psp - namespace: rook-ceph # namespace:cluster -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: - - kind: ServiceAccount - name: rook-ceph-osd - namespace: rook-ceph # namespace:cluster ---- # Allow the osd purge job to run in this namespace kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -1267,20 +1206,6 @@ subjects: name: rook-ceph-purge-osd namespace: rook-ceph # namespace:cluster --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-purge-osd-psp - namespace: rook-ceph # namespace:cluster -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: - - kind: ServiceAccount - name: rook-ceph-purge-osd - namespace: rook-ceph # namespace:cluster ---- # Allow the rgw pods in this namespace to work with configmaps kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 @@ -1296,20 +1221,6 @@ subjects: name: rook-ceph-rgw namespace: rook-ceph # namespace:cluster --- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: rook-ceph-rgw-psp - namespace: rook-ceph # namespace:cluster -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: psp:rook -subjects: - - kind: ServiceAccount - name: rook-ceph-rgw - namespace: rook-ceph # namespace:cluster ---- # Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1 diff --git a/dev/preview/workflow/preview/deploy-monitoring-satellite.sh b/dev/preview/workflow/preview/deploy-monitoring-satellite.sh index 6606f436db7f4f..feffe9b6ddc71f 100755 --- a/dev/preview/workflow/preview/deploy-monitoring-satellite.sh +++ b/dev/preview/workflow/preview/deploy-monitoring-satellite.sh @@ -74,6 +74,9 @@ envsubst <"${ROOT}/dev/preview/workflow/config/monitoring-satellite.yaml" \ echo "Setting default namespace to monitoring-satellite" kubens monitoring-satellite +# remove PodSecurityPolicy rendered objects, we're using K3s 1.26, which doesn't support PSP anymore +find "${manifests_dir}" -type f -iname "*PodSecurityPolicy*" -exec rm {} \; + # we have to apply the CRDs first and wait until they are available before we can apply the rest find "${manifests_dir}" -name "*CustomResourceDefinition*" -exec kubectl --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" --context "${PREVIEW_K3S_KUBE_CONTEXT}" apply -f {} --server-side \; From 8af791ea457a4a57ef52bae6772eb1ec585fe6f4 Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Wed, 26 Apr 2023 23:23:57 +0000 Subject: [PATCH 20/24] [test] don't wait for workspace stop with git_test.go, we're testing git actions. Why? We miss state transitions, it's not guaranteed each one will be returned, and there are other tests waiting. For example, in the below log, we miss INITIALIZING, RUNNING, and STOPPING. workspace.go:369: attempt to create the workspace as user 0565bb3c-e724-4da9-84fb-22e2a7b23b8c, with context github.com/gitpod-io/gitpod-test-repo/tree/integration-test/commit workspace.go:411: attempt to get the workspace information: gitpodio-gitpodtestrepo-nscsowy1njb workspace.go:423: not preparing workspace.go:432: got the workspace information: gitpodio-gitpodtestrepo-nscsowy1njb workspace.go:460: wait for workspace to be fully up and running workspace.go:569: prepare for a connection with ws-manager workspace.go:590: established for a connection with ws-manager workspace.go:598: check if the status of workspace is in the running phase: 462f1325-3019-4547-8666-508e8353335e workspace.go:631: status: 462f1325-3019-4547-8666-508e8353335e, PENDING workspace.go:598: check if the status of workspace is in the running phase: 462f1325-3019-4547-8666-508e8353335e workspace.go:631: status: 462f1325-3019-4547-8666-508e8353335e, PENDING workspace.go:598: check if the status of workspace is in the running phase: 462f1325-3019-4547-8666-508e8353335e workspace.go:631: status: 462f1325-3019-4547-8666-508e8353335e, CREATING workspace.go:598: check if the status of workspace is in the running phase: 462f1325-3019-4547-8666-508e8353335e workspace.go:631: status: 462f1325-3019-4547-8666-508e8353335e, CREATING workspace.go:598: check if the status of workspace is in the running phase: 462f1325-3019-4547-8666-508e8353335e workspace.go:631: status: 462f1325-3019-4547-8666-508e8353335e, CREATING workspace.go:598: check if the status of workspace is in the running phase: 462f1325-3019-4547-8666-508e8353335e workspace.go:504: waiting for stopping the workspace: 462f1325-3019-4547-8666-508e8353335e workspace.go:514: attemp to delete the workspace: 462f1325-3019-4547-8666-508e8353335e workspace.go:797: confirmed the worksapce is stopped: 462f1325-3019-4547-8666-508e8353335e, STOPPED workspace.go:538: successfully terminated workspace git_test.go:172: failed to wait for the workspace to start up: cannot wait for workspace: context deadline exceeded --- test/tests/workspace/git_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/tests/workspace/git_test.go b/test/tests/workspace/git_test.go index 5743c26657a662..d4bea11c6f86ce 100644 --- a/test/tests/workspace/git_test.go +++ b/test/tests/workspace/git_test.go @@ -178,7 +178,7 @@ func TestGitActions(t *testing.T) { sapi := integration.NewComponentAPI(sctx, cfg.Namespace(), kubeconfig, cfg.Client()) defer sapi.Done(t) - _, err := stopWs(true, sapi) + _, err := stopWs(false, sapi) if err != nil { t.Fatal(err) } @@ -196,6 +196,7 @@ func TestGitActions(t *testing.T) { if err != nil { t.Fatal(err) } + t.Log("test finished successfully") }) } } From d1c504a243ef41c3976ca5b0c5cae79a1017a67f Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Fri, 28 Apr 2023 17:38:08 +0000 Subject: [PATCH 21/24] [preview] retry installing trust-manager And use trust-manager from the packer image --- dev/preview/infrastructure/modules/gce/vm.tf | 9 ++++++++- .../scripts/install-trustmanager.sh | 20 +++++++++++++++++++ dev/preview/workflow/preview/deploy-gitpod.sh | 18 ----------------- 3 files changed, 28 insertions(+), 19 deletions(-) create mode 100755 dev/preview/infrastructure/scripts/install-trustmanager.sh diff --git a/dev/preview/infrastructure/modules/gce/vm.tf b/dev/preview/infrastructure/modules/gce/vm.tf index 04293cca9539ba..8c0e849c29dc42 100644 --- a/dev/preview/infrastructure/modules/gce/vm.tf +++ b/dev/preview/infrastructure/modules/gce/vm.tf @@ -69,10 +69,17 @@ data "kubernetes_secret" "harvester-k3s-dockerhub-pull-account" { } locals { - startup_script = templatefile("${path.module}/../../scripts/bootstrap-k3s.sh", { + bootstrap_script = templatefile("${path.module}/../../scripts/bootstrap-k3s.sh", { vm_name = var.preview_name }) + trustmanager_script = file("${path.module}/../../scripts/install-trustmanager.sh") + + startup_script = <<-EOT + ${local.bootstrap_script} + ${local.trustmanager_script} + EOT + cloudinit_user_data = templatefile("${path.module}/cloudinit.yaml", { dockerhub_user = data.kubernetes_secret.harvester-k3s-dockerhub-pull-account.data["username"] dockerhub_passwd = data.kubernetes_secret.harvester-k3s-dockerhub-pull-account.data["password"] diff --git a/dev/preview/infrastructure/scripts/install-trustmanager.sh b/dev/preview/infrastructure/scripts/install-trustmanager.sh new file mode 100755 index 00000000000000..85d248ba6a15f6 --- /dev/null +++ b/dev/preview/infrastructure/scripts/install-trustmanager.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -eo pipefail + +logger -t install-trustmanager "Starting to install trust manager" + +# shellcheck disable=SC2016 +timeout 5m bash -c ' +while [[ -z $(kubectl get certificate trust-manager -n cert-manager --ignore-not-found=true) ]] +do + logger -t install-trustmanager "Sleeping..." + sleep 5 + kubectl apply -f /var/lib/gitpod/manifests/trust-manager.yaml --wait=false || true + logger -t install-trustmanager "Trust manager applied" +done +' + +kubectl wait --for=condition=Available --timeout=300s deployment -n cert-manager trust-manager + +logger -t install-trustmanager "Finishing installing trust manager" diff --git a/dev/preview/workflow/preview/deploy-gitpod.sh b/dev/preview/workflow/preview/deploy-gitpod.sh index a29236f4e5c803..58d7092ffa6391 100755 --- a/dev/preview/workflow/preview/deploy-gitpod.sh +++ b/dev/preview/workflow/preview/deploy-gitpod.sh @@ -179,23 +179,6 @@ function installFluentBit { upgrade --install fluent-bit fluent/fluent-bit --version 0.21.6 -n "${PREVIEW_NAMESPACE}" -f "$ROOT/.werft/vm/charts/fluentbit/values.yaml" } -function installTrustManager { - helm3 \ - --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" \ - --kube-context "${PREVIEW_K3S_KUBE_CONTEXT}" \ - repo add jetstack https://charts.jetstack.io - - helm3 \ - --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" \ - --kube-context "${PREVIEW_K3S_KUBE_CONTEXT}" \ - repo update - - helm3 \ - --kubeconfig "${PREVIEW_K3S_KUBE_PATH}" \ - --kube-context "${PREVIEW_K3S_KUBE_CONTEXT}" \ - upgrade --install --namespace cert-manager trust-manager jetstack/trust-manager --wait -} - # ==================================== # Prerequisites # ==================================== @@ -218,7 +201,6 @@ done copyImagePullSecret installRookCeph installFluentBit -installTrustManager # ======== # Init From 16b64f5ba71ccb65cc45df3fc7289f221eaf3c03 Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Fri, 28 Apr 2023 21:08:15 +0000 Subject: [PATCH 22/24] [test] clarify USER_TOKEN value for preview environments --- test/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/README.md b/test/README.md index 9514db2c44b176..c32a4bc05afb83 100644 --- a/test/README.md +++ b/test/README.md @@ -66,9 +66,9 @@ If you want to run an entire test suite, the easiest is to use `./test/run.sh`: If you're iterating on a single test, the easiest is to use `go test` directly. -If your integration tests depends on having having a user token available, then you'll have to set `USER_NAME` and `USER_TOKEN`. This can be done a couple ways: +If your integration tests depends on having having a user token available, then you'll have to set `USER_NAME` and `USER_TOKEN` environment variables. This can be done a couple ways: 1. Get credentials persisted as secrets (either in Github Actions, or GCP Secret Manager via the `core-dev` project), which vary by job that trigger tests. Refer to `run.sh` for details. -2. In your Gitpod (preview) environment, create a `USER_TOKEN` via your Gitpod user's User Settings, first setup any Integrations needed for the VCS, and then make an Access Token with Full Access permissions. +2. In your Gitpod (preview) environment, log into the preview environment, set `USER_NAME` to the user you logged in with, and set `USER_TOKEN` to any (does not have to be valid). By default the workspace tests run against `ws-manager-mk2`, set `WS_MANAGER_MK2=false` to run against mk1. From 4655ceea57d157f645fc1aedcfe325eadf28aeec Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Mon, 1 May 2023 13:03:21 +0000 Subject: [PATCH 23/24] Cleanup --- .werft/vm/manifests/rook-ceph/common.yaml | 18 ------------------ .../gitpod-protocol/go/gitpod-service.go | 5 +---- 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/.werft/vm/manifests/rook-ceph/common.yaml b/.werft/vm/manifests/rook-ceph/common.yaml index f09eb72d46c350..cb45d35a99c5e7 100644 --- a/.werft/vm/manifests/rook-ceph/common.yaml +++ b/.werft/vm/manifests/rook-ceph/common.yaml @@ -80,24 +80,6 @@ rules: resources: ["volumesnapshots/status"] verbs: ["update", "patch"] --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: ClusterRole -# metadata: -# name: 'psp:rook' -# labels: -# operator: rook -# storage-backend: ceph -# app.kubernetes.io/part-of: rook-ceph-operator -# rules: -# - apiGroups: -# - policy -# resources: -# - podsecuritypolicies -# resourceNames: -# - 00-rook-privileged -# verbs: -# - use -# --- kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: diff --git a/components/gitpod-protocol/go/gitpod-service.go b/components/gitpod-protocol/go/gitpod-service.go index 8f5fef9a87f995..6a8e5f134b43a8 100644 --- a/components/gitpod-protocol/go/gitpod-service.go +++ b/components/gitpod-protocol/go/gitpod-service.go @@ -2052,10 +2052,7 @@ type UpdateOwnAuthProviderParams struct { // CreateWorkspaceOptions is the CreateWorkspaceOptions message type type CreateWorkspaceOptions struct { StartWorkspaceOptions - ContextURL string `json:"contextUrl,omitempty"` - // references: - // https://github.com/gitpod-io/gitpod/pull/17040 - // https://github.com/gitpod-io/gitpod/blob/90b7d658589fd6e9fb239ff239591b9f1218fc83/components/server/src/workspace/gitpod-server-impl.ts#L1227-L1235 + ContextURL string `json:"contextUrl,omitempty"` OrganizationId string `json:"organizationId,omitempty"` IgnoreRunningWorkspaceOnSameCommit bool `json:"ignoreRunningWorkspaceOnSameCommit,omitemopty"` IgnoreRunningPrebuild bool `json:"ignoreRunningPrebuild,omitemopty"` From a10d7d571fd6cdc39132524e84429bf3e96f2085 Mon Sep 17 00:00:00 2001 From: Kyle Brennan Date: Mon, 1 May 2023 13:32:36 +0000 Subject: [PATCH 24/24] [preview] remove commented out yaml related to PodSecurityPolicy --- .werft/vm/manifests/rook-ceph/common.yaml | 149 ---------------------- 1 file changed, 149 deletions(-) diff --git a/.werft/vm/manifests/rook-ceph/common.yaml b/.werft/vm/manifests/rook-ceph/common.yaml index cb45d35a99c5e7..4a4b8245343344 100644 --- a/.werft/vm/manifests/rook-ceph/common.yaml +++ b/.werft/vm/manifests/rook-ceph/common.yaml @@ -682,155 +682,6 @@ subjects: - kind: ServiceAccount name: rook-ceph-system namespace: rook-ceph # namespace:operator -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: ClusterRoleBinding -# metadata: -# name: rook-ceph-system-psp -# labels: -# operator: rook -# storage-backend: ceph -# app.kubernetes.io/part-of: rook-ceph-operator -# roleRef: -# apiGroup: rbac.authorization.k8s.io -# kind: ClusterRole -# name: 'psp:rook' -# subjects: -# - kind: ServiceAccount -# name: rook-ceph-system -# namespace: rook-ceph # namespace:operator -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: ClusterRoleBinding -# metadata: -# name: rook-csi-cephfs-plugin-sa-psp -# roleRef: -# apiGroup: rbac.authorization.k8s.io -# kind: ClusterRole -# name: 'psp:rook' -# subjects: -# - kind: ServiceAccount -# name: rook-csi-cephfs-plugin-sa -# namespace: rook-ceph # namespace:operator -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: ClusterRoleBinding -# metadata: -# name: rook-csi-cephfs-provisioner-sa-psp -# roleRef: -# apiGroup: rbac.authorization.k8s.io -# kind: ClusterRole -# name: 'psp:rook' -# subjects: -# - kind: ServiceAccount -# name: rook-csi-cephfs-provisioner-sa -# namespace: rook-ceph # namespace:operator -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: ClusterRoleBinding -# metadata: -# name: rook-csi-rbd-plugin-sa-psp -# roleRef: -# apiGroup: rbac.authorization.k8s.io -# kind: ClusterRole -# name: 'psp:rook' -# subjects: -# - kind: ServiceAccount -# name: rook-csi-rbd-plugin-sa -# namespace: rook-ceph # namespace:operator -# --- -# apiVersion: rbac.authorization.k8s.io/v1 -# kind: ClusterRoleBinding -# metadata: -# name: rook-csi-rbd-provisioner-sa-psp -# roleRef: -# apiGroup: rbac.authorization.k8s.io -# kind: ClusterRole -# name: 'psp:rook' -# subjects: -# - kind: ServiceAccount -# name: rook-csi-rbd-provisioner-sa -# namespace: rook-ceph # namespace:operator -# --- -# We expect most Kubernetes teams to follow the Kubernetes docs and have these PSPs. -# * privileged (for kube-system namespace) -# * restricted (for all logged in users) -# -# PSPs are applied based on the first match alphabetically. `rook-ceph-operator` comes after -# `restricted` alphabetically, so we name this `00-rook-privileged`, so it stays somewhere -# close to the top and so `rook-system` gets the intended PSP. This may need to be renamed in -# environments with other `00`-prefixed PSPs. -# -# More on PSP ordering: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#policy-order -# apiVersion: policy/v1beta1 -# kind: PodSecurityPolicy -# metadata: -# name: 00-rook-privileged -# annotations: -# seccomp.security.alpha.kubernetes.io/allowedProfileNames: 'runtime/default' -# seccomp.security.alpha.kubernetes.io/defaultProfileName: 'runtime/default' -# spec: -# privileged: true -# allowedCapabilities: -# # required by CSI -# - SYS_ADMIN -# - MKNOD -# fsGroup: -# rule: RunAsAny -# # runAsUser, supplementalGroups - Rook needs to run some pods as root -# # Ceph pods could be run as the Ceph user, but that user isn't always known ahead of time -# runAsUser: -# rule: RunAsAny -# supplementalGroups: -# rule: RunAsAny -# # seLinux - seLinux context is unknown ahead of time; set if this is well-known -# seLinux: -# rule: RunAsAny -# volumes: -# # recommended minimum set -# - configMap -# - downwardAPI -# - emptyDir -# - persistentVolumeClaim -# - secret -# - projected -# # required for Rook -# - hostPath -# # allowedHostPaths can be set to Rook's known host volume mount points when they are fully-known -# # allowedHostPaths: -# # - pathPrefix: "/run/udev" # for OSD prep -# # readOnly: false -# # - pathPrefix: "/dev" # for OSD prep -# # readOnly: false -# # - pathPrefix: "/var/lib/rook" # or whatever the dataDirHostPath value is set to -# # readOnly: false -# # Ceph requires host IPC for setting up encrypted devices -# hostIPC: true -# # Ceph OSDs need to share the same PID namespace -# hostPID: true -# # hostNetwork can be set to 'false' if host networking isn't used -# hostNetwork: true -# hostPorts: -# # Ceph messenger protocol v1 -# - min: 6789 -# max: 6790 # <- support old default port -# # Ceph messenger protocol v2 -# - min: 3300 -# max: 3300 -# # Ceph RADOS ports for OSDs, MDSes -# - min: 6800 -# max: 7300 -# # # Ceph dashboard port HTTP (not recommended) -# # - min: 7000 -# # max: 7000 -# # Ceph dashboard port HTTPS -# - min: 8443 -# max: 8443 -# # Ceph mgr Prometheus Metrics -# - min: 9283 -# max: 9283 -# # port for CSIAddons -# - min: 9070 -# max: 9070 --- kind: Role apiVersion: rbac.authorization.k8s.io/v1