diff --git a/cmd/nerdctl/compose_run_linux_test.go b/cmd/nerdctl/compose_run_linux_test.go index ea075f4e9d9..74874d4d298 100644 --- a/cmd/nerdctl/compose_run_linux_test.go +++ b/cmd/nerdctl/compose_run_linux_test.go @@ -432,12 +432,13 @@ func TestComposePushAndPullWithCosignVerify(t *testing.T) { keyPair := newCosignKeyPair(t, "cosign-key-pair") defer keyPair.cleanup() - reg := testregistry.NewPlainHTTP(base, 5000) - defer reg.Cleanup() + reg := testregistry.NewWithNoAuth(base, 0, false) + defer reg.Cleanup(nil) + localhostIP := "127.0.0.1" t.Logf("localhost IP=%q", localhostIP) testImageRefPrefix := fmt.Sprintf("%s:%d/", - localhostIP, reg.ListenPort) + localhostIP, reg.Port) t.Logf("testImageRefPrefix=%q", testImageRefPrefix) var ( diff --git a/cmd/nerdctl/container_run_verify_linux_test.go b/cmd/nerdctl/container_run_verify_linux_test.go index e5c500d59da..306403c9cd9 100644 --- a/cmd/nerdctl/container_run_verify_linux_test.go +++ b/cmd/nerdctl/container_run_verify_linux_test.go @@ -34,12 +34,13 @@ func TestRunVerifyCosign(t *testing.T) { base := testutil.NewBase(t) defer base.Cmd("builder", "prune").Run() tID := testutil.Identifier(t) - reg := testregistry.NewPlainHTTP(base, 5000) - defer reg.Cleanup() + reg := testregistry.NewWithNoAuth(base, 0, false) + defer reg.Cleanup(nil) + localhostIP := "127.0.0.1" t.Logf("localhost IP=%q", localhostIP) testImageRef := fmt.Sprintf("%s:%d/%s", - localhostIP, reg.ListenPort, tID) + localhostIP, reg.Port, tID) t.Logf("testImageRef=%q", testImageRef) dockerfile := fmt.Sprintf(`FROM %s diff --git a/cmd/nerdctl/image_convert_linux_test.go b/cmd/nerdctl/image_convert_linux_test.go index c6fb457d962..a8b53401d72 100644 --- a/cmd/nerdctl/image_convert_linux_test.go +++ b/cmd/nerdctl/image_convert_linux_test.go @@ -33,7 +33,10 @@ func TestImageConvertNydus(t *testing.T) { } testutil.RequireExecutable(t, "nydus-image") testutil.DockerIncompatible(t) + base := testutil.NewBase(t) + t.Parallel() + convertedImage := testutil.Identifier(t) + ":nydus" base.Cmd("rmi", convertedImage).Run() base.Cmd("pull", testutil.CommonImage).AssertOK() @@ -48,18 +51,20 @@ func TestImageConvertNydus(t *testing.T) { t.Skip("Nydusify check is not supported rootless mode.") } - // skip if nydusify is not installed + // skip if nydusify and nydusd are not installed testutil.RequireExecutable(t, "nydusify") + testutil.RequireExecutable(t, "nydusd") // setup local docker registry - registryPort := 15000 - registry := testregistry.NewPlainHTTP(base, registryPort) - defer registry.Cleanup() + registry := testregistry.NewWithNoAuth(base, 0, false) + remoteImage := fmt.Sprintf("%s:%d/nydusd-image:test", "localhost", registry.Port) + t.Cleanup(func() { + base.Cmd("rmi", remoteImage).Run() + registry.Cleanup(nil) + }) - remoteImage := fmt.Sprintf("%s:%d/nydusd-image:test", registry.IP.String(), registryPort) base.Cmd("tag", convertedImage, remoteImage).AssertOK() - defer base.Cmd("rmi", remoteImage).Run() - base.Cmd("push", "--insecure-registry", remoteImage).AssertOK() + base.Cmd("push", remoteImage).AssertOK() nydusifyCmd := testutil.Cmd{ Cmd: icmd.Command( "nydusify", @@ -73,5 +78,8 @@ func TestImageConvertNydus(t *testing.T) { ), Base: base, } + + // nydus is creating temporary files - make sure we are in a proper location for that + nydusifyCmd.Cmd.Dir = base.T.TempDir() nydusifyCmd.AssertOK() } diff --git a/cmd/nerdctl/image_encrypt_linux_test.go b/cmd/nerdctl/image_encrypt_linux_test.go index d20daa60b9a..f6ba49ab5f1 100644 --- a/cmd/nerdctl/image_encrypt_linux_test.go +++ b/cmd/nerdctl/image_encrypt_linux_test.go @@ -113,10 +113,10 @@ func TestImageEncryptJWE(t *testing.T) { defer keyPair.cleanup() base := testutil.NewBase(t) tID := testutil.Identifier(t) - reg := testregistry.NewPlainHTTP(base, 5000) - defer reg.Cleanup() + reg := testregistry.NewWithNoAuth(base, 0, false) + defer reg.Cleanup(nil) base.Cmd("pull", testutil.CommonImage).AssertOK() - encryptImageRef := fmt.Sprintf("127.0.0.1:%d/%s:encrypted", reg.ListenPort, tID) + encryptImageRef := fmt.Sprintf("127.0.0.1:%d/%s:encrypted", reg.Port, tID) defer base.Cmd("rmi", encryptImageRef).Run() base.Cmd("image", "encrypt", "--recipient=jwe:"+keyPair.pub, testutil.CommonImage, encryptImageRef).AssertOK() base.Cmd("image", "inspect", "--mode=native", "--format={{len .Index.Manifests}}", encryptImageRef).AssertOutExactly("1\n") diff --git a/cmd/nerdctl/image_pull_linux_test.go b/cmd/nerdctl/image_pull_linux_test.go index 931d7905737..37153c3d7af 100644 --- a/cmd/nerdctl/image_pull_linux_test.go +++ b/cmd/nerdctl/image_pull_linux_test.go @@ -67,12 +67,12 @@ func TestImageVerifyWithCosign(t *testing.T) { base := testutil.NewBase(t) defer base.Cmd("builder", "prune").Run() tID := testutil.Identifier(t) - reg := testregistry.NewPlainHTTP(base, 5000) - defer reg.Cleanup() + reg := testregistry.NewWithNoAuth(base, 0, false) + defer reg.Cleanup(nil) localhostIP := "127.0.0.1" t.Logf("localhost IP=%q", localhostIP) testImageRef := fmt.Sprintf("%s:%d/%s", - localhostIP, reg.ListenPort, tID) + localhostIP, reg.Port, tID) t.Logf("testImageRef=%q", testImageRef) dockerfile := fmt.Sprintf(`FROM %s @@ -91,8 +91,8 @@ func TestImagePullPlainHttpWithDefaultPort(t *testing.T) { testutil.RequiresBuild(t) base := testutil.NewBase(t) defer base.Cmd("builder", "prune").Run() - reg := testregistry.NewPlainHTTP(base, 80) - defer reg.Cleanup() + reg := testregistry.NewWithNoAuth(base, 80, false) + defer reg.Cleanup(nil) testImageRef := fmt.Sprintf("%s/%s:%s", reg.IP.String(), testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) t.Logf("testImageRef=%q", testImageRef) @@ -117,12 +117,12 @@ func TestImageVerifyWithCosignShouldFailWhenKeyIsNotCorrect(t *testing.T) { base := testutil.NewBase(t) defer base.Cmd("builder", "prune").Run() tID := testutil.Identifier(t) - reg := testregistry.NewPlainHTTP(base, 5000) - defer reg.Cleanup() + reg := testregistry.NewWithNoAuth(base, 0, false) + defer reg.Cleanup(nil) localhostIP := "127.0.0.1" t.Logf("localhost IP=%q", localhostIP) testImageRef := fmt.Sprintf("%s:%d/%s", - localhostIP, reg.ListenPort, tID) + localhostIP, reg.Port, tID) t.Logf("testImageRef=%q", testImageRef) dockerfile := fmt.Sprintf(`FROM %s diff --git a/cmd/nerdctl/image_push_linux_test.go b/cmd/nerdctl/image_push_linux_test.go index 0a65980e19c..79fa63f52cb 100644 --- a/cmd/nerdctl/image_push_linux_test.go +++ b/cmd/nerdctl/image_push_linux_test.go @@ -30,12 +30,12 @@ import ( func TestPushPlainHTTPFails(t *testing.T) { testutil.RequiresBuild(t) base := testutil.NewBase(t) - reg := testregistry.NewPlainHTTP(base, 5000) - defer reg.Cleanup() + reg := testregistry.NewWithNoAuth(base, 0, false) + defer reg.Cleanup(nil) base.Cmd("pull", testutil.CommonImage).AssertOK() testImageRef := fmt.Sprintf("%s:%d/%s:%s", - reg.IP.String(), reg.ListenPort, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) + reg.IP.String(), reg.Port, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) t.Logf("testImageRef=%q", testImageRef) base.Cmd("tag", testutil.CommonImage, testImageRef).AssertOK() @@ -49,14 +49,14 @@ func TestPushPlainHTTPFails(t *testing.T) { func TestPushPlainHTTPLocalhost(t *testing.T) { testutil.RequiresBuild(t) base := testutil.NewBase(t) - reg := testregistry.NewPlainHTTP(base, 5000) - defer reg.Cleanup() + reg := testregistry.NewWithNoAuth(base, 0, false) + defer reg.Cleanup(nil) localhostIP := "127.0.0.1" t.Logf("localhost IP=%q", localhostIP) base.Cmd("pull", testutil.CommonImage).AssertOK() testImageRef := fmt.Sprintf("%s:%d/%s:%s", - localhostIP, reg.ListenPort, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) + localhostIP, reg.Port, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) t.Logf("testImageRef=%q", testImageRef) base.Cmd("tag", testutil.CommonImage, testImageRef).AssertOK() @@ -69,12 +69,12 @@ func TestPushPlainHTTPInsecure(t *testing.T) { testutil.DockerIncompatible(t) base := testutil.NewBase(t) - reg := testregistry.NewPlainHTTP(base, 5000) - defer reg.Cleanup() + reg := testregistry.NewWithNoAuth(base, 0, false) + defer reg.Cleanup(nil) base.Cmd("pull", testutil.CommonImage).AssertOK() testImageRef := fmt.Sprintf("%s:%d/%s:%s", - reg.IP.String(), reg.ListenPort, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) + reg.IP.String(), reg.Port, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) t.Logf("testImageRef=%q", testImageRef) base.Cmd("tag", testutil.CommonImage, testImageRef).AssertOK() @@ -87,8 +87,8 @@ func TestPushPlainHttpInsecureWithDefaultPort(t *testing.T) { testutil.DockerIncompatible(t) base := testutil.NewBase(t) - reg := testregistry.NewPlainHTTP(base, 80) - defer reg.Cleanup() + reg := testregistry.NewWithNoAuth(base, 80, false) + defer reg.Cleanup(nil) base.Cmd("pull", testutil.CommonImage).AssertOK() testImageRef := fmt.Sprintf("%s/%s:%s", @@ -105,14 +105,14 @@ func TestPushInsecureWithLogin(t *testing.T) { testutil.DockerIncompatible(t) base := testutil.NewBase(t) - reg := testregistry.NewHTTPS(base, "admin", "badmin") - defer reg.Cleanup() + reg := testregistry.NewWithTokenAuth(base, "admin", "badmin", 0, true) + defer reg.Cleanup(nil) base.Cmd("--insecure-registry", "login", "-u", "admin", "-p", "badmin", - fmt.Sprintf("%s:%d", reg.IP.String(), reg.ListenPort)).AssertOK() + fmt.Sprintf("%s:%d", reg.IP.String(), reg.Port)).AssertOK() base.Cmd("pull", testutil.CommonImage).AssertOK() testImageRef := fmt.Sprintf("%s:%d/%s:%s", - reg.IP.String(), reg.ListenPort, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) + reg.IP.String(), reg.Port, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) t.Logf("testImageRef=%q", testImageRef) base.Cmd("tag", testutil.CommonImage, testImageRef).AssertOK() @@ -126,14 +126,14 @@ func TestPushWithHostsDir(t *testing.T) { testutil.DockerIncompatible(t) base := testutil.NewBase(t) - reg := testregistry.NewHTTPS(base, "admin", "badmin") - defer reg.Cleanup() + reg := testregistry.NewWithTokenAuth(base, "admin", "badmin", 0, true) + defer reg.Cleanup(nil) - base.Cmd("--hosts-dir", reg.HostsDir, "login", "-u", "admin", "-p", "badmin", fmt.Sprintf("%s:%d", reg.IP.String(), reg.ListenPort)).AssertOK() + base.Cmd("--hosts-dir", reg.HostsDir, "login", "-u", "admin", "-p", "badmin", fmt.Sprintf("%s:%d", reg.IP.String(), reg.Port)).AssertOK() base.Cmd("pull", testutil.CommonImage).AssertOK() testImageRef := fmt.Sprintf("%s:%d/%s:%s", - reg.IP.String(), reg.ListenPort, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) + reg.IP.String(), reg.Port, testutil.Identifier(t), strings.Split(testutil.CommonImage, ":")[1]) t.Logf("testImageRef=%q", testImageRef) base.Cmd("tag", testutil.CommonImage, testImageRef).AssertOK() @@ -147,18 +147,18 @@ func TestPushNonDistributableArtifacts(t *testing.T) { testutil.DockerIncompatible(t) base := testutil.NewBase(t) - reg := testregistry.NewPlainHTTP(base, 5000) - defer reg.Cleanup() + reg := testregistry.NewWithNoAuth(base, 0, false) + defer reg.Cleanup(nil) base.Cmd("pull", testutil.NonDistBlobImage).AssertOK() testImgRef := fmt.Sprintf("%s:%d/%s:%s", - reg.IP.String(), reg.ListenPort, testutil.Identifier(t), strings.Split(testutil.NonDistBlobImage, ":")[1]) + reg.IP.String(), reg.Port, testutil.Identifier(t), strings.Split(testutil.NonDistBlobImage, ":")[1]) base.Cmd("tag", testutil.NonDistBlobImage, testImgRef).AssertOK() base.Cmd("--debug", "--insecure-registry", "push", testImgRef).AssertOK() - blobURL := fmt.Sprintf("http://%s:%d/v2/%s/blobs/%s", reg.IP.String(), reg.ListenPort, testutil.Identifier(t), testutil.NonDistBlobDigest) + blobURL := fmt.Sprintf("http://%s:%d/v2/%s/blobs/%s", reg.IP.String(), reg.Port, testutil.Identifier(t), testutil.NonDistBlobDigest) resp, err := http.Get(blobURL) assert.Assert(t, err, "error making http request") if resp.Body != nil { @@ -179,12 +179,12 @@ func TestPushSoci(t *testing.T) { testutil.DockerIncompatible(t) base := testutil.NewBase(t) requiresSoci(base) - reg := testregistry.NewPlainHTTP(base, 5000) - defer reg.Cleanup() + reg := testregistry.NewWithNoAuth(base, 0, false) + defer reg.Cleanup(nil) base.Cmd("pull", testutil.UbuntuImage).AssertOK() testImageRef := fmt.Sprintf("%s:%d/%s:%s", - reg.IP.String(), reg.ListenPort, testutil.Identifier(t), strings.Split(testutil.UbuntuImage, ":")[1]) + reg.IP.String(), reg.Port, testutil.Identifier(t), strings.Split(testutil.UbuntuImage, ":")[1]) t.Logf("testImageRef=%q", testImageRef) base.Cmd("tag", testutil.UbuntuImage, testImageRef).AssertOK() diff --git a/cmd/nerdctl/login_linux_test.go b/cmd/nerdctl/login_linux_test.go index ce145e530ee..4429b4480ea 100644 --- a/cmd/nerdctl/login_linux_test.go +++ b/cmd/nerdctl/login_linux_test.go @@ -17,174 +17,310 @@ package main import ( + "crypto/rand" + "encoding/base64" "fmt" "net" - "path" + "os" "strconv" "testing" "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/testca" "github.com/containerd/nerdctl/v2/pkg/testutil/testregistry" + "gotest.tools/v3/icmd" ) -func TestLogin(t *testing.T) { - // Skip docker, because Docker doesn't have `--hosts-dir` option, and we don't want to contaminate the global /etc/docker/certs.d during this test - testutil.DockerIncompatible(t) +func safeRandomString(n int) string { + b := make([]byte, n) + _, _ = rand.Read(b) + // XXX WARNING there is something in the registry (or more likely in the way we generate htpasswd files) + // that is broken and does not resist truly random strings + // return string(b) + return base64.URLEncoding.EncodeToString(b) +} - base := testutil.NewBase(t) - reg := testregistry.NewHTTPS(base, "admin", "validTestPassword") - defer reg.Cleanup() +type Client struct { + args []string + configPath string +} + +func (ag *Client) WithInsecure(value bool) *Client { + ag.args = append(ag.args, "--insecure-registry="+strconv.FormatBool(value)) + return ag +} - regHost := net.JoinHostPort(reg.IP.String(), strconv.Itoa(reg.ListenPort)) +func (ag *Client) WithHostsDir(hostDirs string) *Client { + ag.args = append(ag.args, "--hosts-dir", hostDirs) + return ag +} - t.Logf("Good password") - base.Cmd("--debug-full", "--hosts-dir", reg.HostsDir, "login", "-u", "admin", "-p", "validTestPassword", regHost).AssertOK() +func (ag *Client) WithCredentials(username, password string) *Client { + ag.args = append(ag.args, "--username", username, "--password", password) + return ag +} - t.Logf("Bad password") - base.Cmd("--debug-full", "--hosts-dir", reg.HostsDir, "login", "-u", "admin", "-p", "invalidTestPassword", regHost).AssertFail() +func (ag *Client) Run(base *testutil.Base, host string) *testutil.Cmd { + if ag.configPath == "" { + ag.configPath, _ = os.MkdirTemp(base.T.TempDir(), "docker-config") + } + args := append([]string{"--debug-full", "login"}, ag.args...) + icmdCmd := icmd.Command(base.Binary, append(base.Args, append(args, host)...)...) + icmdCmd.Env = append(base.Env, "HOME="+os.Getenv("HOME"), "DOCKER_CONFIG="+ag.configPath) + return &testutil.Cmd{ + Cmd: icmdCmd, + Base: base, + } } -func TestLoginWithSpecificRegHosts(t *testing.T) { - // Skip docker, because Docker doesn't have `--hosts-dir` option, and we don't want to contaminate the global /etc/docker/certs.d during this test +func TestLogin(t *testing.T) { + // Skip docker, because Docker doesn't have `--hosts-dir` nor `insecure-registry` option testutil.DockerIncompatible(t) base := testutil.NewBase(t) - reg := testregistry.NewHTTPS(base, "admin", "validTestPassword") - defer reg.Cleanup() + t.Parallel() - regHost := net.JoinHostPort(reg.IP.String(), strconv.Itoa(reg.ListenPort)) + testregistry.EnsureImages(base) - t.Logf("Prepare regHost URL with path and Scheme") - - type testCase struct { - url string - log string - } - testCases := []testCase{ + testCases := []struct { + port int + tls bool + auth string + insecure bool + }{ { - url: "https://" + path.Join(regHost, "test"), - log: "Login with repository containing path and scheme in the URL", + 80, + false, + "basic", + true, }, { - url: path.Join(regHost, "test"), - log: "Login with repository containing path and without scheme in the URL", + 443, + false, + "basic", + true, + }, + { + 0, + false, + "basic", + true, + }, + { + 80, + true, + "basic", + false, + }, + { + 443, + true, + "basic", + false, + }, + { + 0, + true, + "basic", + false, }, - } - for _, tc := range testCases { - t.Logf(tc.log) - base.Cmd("--debug-full", "--hosts-dir", reg.HostsDir, "login", "-u", "admin", "-p", "validTestPassword", tc.url).AssertOK() - } - -} - -func TestLoginWithPlainHttp(t *testing.T) { - testutil.DockerIncompatible(t) - base := testutil.NewBase(t) - reg5000 := testregistry.NewAuthWithHTTP(base, "admin", "validTestPassword", 5000, 5001) - reg80 := testregistry.NewAuthWithHTTP(base, "admin", "validTestPassword", 80, 5002) - defer reg5000.Cleanup() - defer reg80.Cleanup() - testCasesForPort5000 := []struct { - regHost string - regPort int - useRegPort bool - username string - password string - shouldSuccess bool - registry *testregistry.TestRegistry - shouldUseInSecure bool - }{ { - regHost: "127.0.0.1", - regPort: 5000, - useRegPort: true, - username: "admin", - password: "validTestPassword", - shouldSuccess: true, - registry: reg5000, - shouldUseInSecure: true, + 80, + false, + "token", + true, }, { - regHost: "127.0.0.1", - regPort: 5000, - useRegPort: true, - username: "admin", - password: "invalidTestPassword", - shouldSuccess: false, - registry: reg5000, - shouldUseInSecure: true, + 443, + false, + "token", + true, }, { - regHost: "127.0.0.1", - regPort: 5000, - useRegPort: true, - username: "admin", - password: "validTestPassword", - // Following the merging of the below, any localhost/loopback registries will - // get automatically downgraded to HTTP so this will still succceed: - // https://github.com/containerd/containerd/pull/7393 - shouldSuccess: true, - registry: reg5000, - shouldUseInSecure: false, + 0, + false, + "token", + true, }, { - regHost: "127.0.0.1", - regPort: 80, - useRegPort: false, - username: "admin", - password: "validTestPassword", - shouldSuccess: true, - registry: reg80, - shouldUseInSecure: true, + 80, + true, + "token", + false, }, { - regHost: "127.0.0.1", - regPort: 80, - useRegPort: false, - username: "admin", - password: "invalidTestPassword", - shouldSuccess: false, - registry: reg80, - shouldUseInSecure: true, + 443, + true, + "token", + false, }, { - regHost: "127.0.0.1", - regPort: 80, - useRegPort: false, - username: "admin", - password: "validTestPassword", - // Following the merging of the below, any localhost/loopback registries will - // get automatically downgraded to HTTP so this will still succceed: - // https://github.com/containerd/containerd/pull/7393 - shouldSuccess: true, - registry: reg80, - shouldUseInSecure: false, + 0, + true, + "token", + false, }, } - for _, tc := range testCasesForPort5000 { - tcName := fmt.Sprintf("%+v", tc) - t.Run(tcName, func(t *testing.T) { - regHost := tc.regHost - if tc.useRegPort { - regHost = fmt.Sprintf("%s:%d", regHost, tc.regPort) + + for _, tc := range testCases { + // Since we have a lock mechanism for acquiring ports, we can just parallelize everything + t.Run(fmt.Sprintf("Login against registry with tls: %t port: %d auth: %s", tc.tls, tc.port, tc.auth), func(t *testing.T) { + // Tests with fixed ports should not be parallelized (although the port locking mechanism will prevent conflicts) + // as their children are, and this might deadlock given how Parallel works + if tc.port == 0 { + t.Parallel() + } + + // Generate credentials so that we never cross hit another test registry (spiced up with unicode) + // Note that the grammar for basic auth does not allow colons in usernames, while token auth allows it + username := safeRandomString(30) + "∞" + password := safeRandomString(30) + ":∞" + + // Get a CA if we want TLS + var ca *testca.CA + if tc.tls { + ca = testca.New(base.T) } - if tc.shouldSuccess { - t.Logf("Good password") - } else { - t.Logf("Bad password") + + // Add the requested authentication + var auth testregistry.Auth + auth = &testregistry.NoAuth{} + var dependentCleanup func(error) + if tc.auth == "basic" { + auth = &testregistry.BasicAuth{ + Username: username, + Password: password, + } + } else if tc.auth == "token" { + authCa := ca + // We could be on !tls - still need a ca to sign jwt + if authCa == nil { + authCa = testca.New(base.T) + } + as := testregistry.NewAuthServer(base, authCa, 0, username, password, tc.tls) + auth = &testregistry.TokenAuth{ + Address: as.Scheme + "://" + net.JoinHostPort(as.IP.String(), strconv.Itoa(as.Port)), + CertPath: as.CertPath, + } + dependentCleanup = as.Cleanup } - var args []string - if tc.shouldUseInSecure { - args = append(args, "--insecure-registry") + + // Start the registry + reg := testregistry.NewRegistry(base, ca, tc.port, auth, dependentCleanup) + + // Attach our cleanup function + t.Cleanup(func() { + reg.Cleanup(nil) + }) + + regHosts := []string{ + net.JoinHostPort(reg.IP.String(), strconv.Itoa(reg.Port)), } - args = append(args, []string{ - "--debug-full", "--hosts-dir", tc.registry.HostsDir, "login", "-u", tc.username, "-p", tc.password, regHost, - }...) - cmd := base.Cmd(args...) - if tc.shouldSuccess { - cmd.AssertOK() - } else { - cmd.AssertFail() + + // XXX seems like omitting ports is broken on main currently + // (plus the hosts.toml resolution is not good either) + // XXX we should also add hostname here (maybe use the container name?) + // Obviously also need to add localhost to the mix once we fix behavior + /* + if reg.Port == 443 || reg.Port == 80 { + regHosts = append(regHosts, reg.IP.String()) + } + */ + + for _, value := range regHosts { + regHost := value + t.Run(regHost, func(t *testing.T) { + t.Parallel() + + t.Run("Valid credentials (no certs) ", func(t *testing.T) { + t.Parallel() + c := (&Client{}). + WithCredentials(username, password) + + // Fail without insecure + c.Run(base, regHost).AssertFail() + + // Succeed with insecure + c.WithInsecure(true). + Run(base, regHost).AssertOK() + }) + + t.Run("Valid credentials (with certs)", func(t *testing.T) { + t.Parallel() + c := (&Client{}). + WithCredentials(username, password). + WithHostsDir(reg.HostsDir) + + if tc.insecure { + c.Run(base, regHost).AssertFail() + } else { + c.Run(base, regHost).AssertOK() + } + + c.WithInsecure(true). + Run(base, regHost).AssertOK() + }) + + t.Run("Valid credentials (with certs), any variant", func(t *testing.T) { + t.Parallel() + c := (&Client{}). + WithCredentials(username, password). + WithHostsDir(reg.HostsDir). + // Just use insecure here for all servers - it does not matter for what we are testing here + WithInsecure(true) + + c.Run(base, "http://"+regHost).AssertOK() + c.Run(base, "https://"+regHost).AssertOK() + c.Run(base, "http://"+regHost+"/whatever?foo=bar;foo:bar#foo=bar").AssertOK() + c.Run(base, "https://"+regHost+"/whatever?foo=bar&bar=foo;foo=foo+bar:bar#foo=bar").AssertOK() + }) + + t.Run("Wrong pass (no certs)", func(t *testing.T) { + t.Parallel() + c := (&Client{}). + WithCredentials(username, "invalid") + + c.Run(base, regHost).AssertFail() + + c.WithInsecure(true). + Run(base, regHost).AssertFail() + }) + + t.Run("Wrong user (no certs)", func(t *testing.T) { + t.Parallel() + c := (&Client{}). + WithCredentials("invalid", password) + + c.Run(base, regHost).AssertFail() + + c.WithInsecure(true). + Run(base, regHost).AssertFail() + }) + + t.Run("Wrong pass (with certs)", func(t *testing.T) { + t.Parallel() + c := (&Client{}). + WithCredentials(username, "invalid"). + WithHostsDir(reg.HostsDir) + + c.Run(base, regHost).AssertFail() + + c.WithInsecure(true). + Run(base, regHost).AssertFail() + }) + + t.Run("Wrong user (with certs)", func(t *testing.T) { + t.Parallel() + c := (&Client{}). + WithCredentials("invalid", password). + WithHostsDir(reg.HostsDir) + + c.Run(base, regHost).AssertFail() + + c.WithInsecure(true). + Run(base, regHost).AssertFail() + }) + }) } }) } diff --git a/cmd/nerdctl/multi_platform_linux_test.go b/cmd/nerdctl/multi_platform_linux_test.go index f46210b98ff..cf9b130dd59 100644 --- a/cmd/nerdctl/multi_platform_linux_test.go +++ b/cmd/nerdctl/multi_platform_linux_test.go @@ -57,10 +57,10 @@ func TestMultiPlatformBuildPush(t *testing.T) { base := testutil.NewBase(t) defer base.Cmd("builder", "prune").Run() tID := testutil.Identifier(t) - reg := testregistry.NewPlainHTTP(base, 5000) - defer reg.Cleanup() + reg := testregistry.NewWithNoAuth(base, 0, false) + defer reg.Cleanup(nil) - imageName := fmt.Sprintf("localhost:%d/%s:latest", reg.ListenPort, tID) + imageName := fmt.Sprintf("localhost:%d/%s:latest", reg.Port, tID) defer base.Cmd("rmi", imageName).Run() dockerfile := fmt.Sprintf(`FROM %s @@ -84,10 +84,10 @@ func TestMultiPlatformBuildPushNoRun(t *testing.T) { base := testutil.NewBase(t) defer base.Cmd("builder", "prune").Run() tID := testutil.Identifier(t) - reg := testregistry.NewPlainHTTP(base, 5000) - defer reg.Cleanup() + reg := testregistry.NewWithNoAuth(base, 0, false) + defer reg.Cleanup(nil) - imageName := fmt.Sprintf("localhost:%d/%s:latest", reg.ListenPort, tID) + imageName := fmt.Sprintf("localhost:%d/%s:latest", reg.Port, tID) defer base.Cmd("rmi", imageName).Run() dockerfile := fmt.Sprintf(`FROM %s @@ -105,10 +105,10 @@ func TestMultiPlatformPullPushAllPlatforms(t *testing.T) { testutil.DockerIncompatible(t) base := testutil.NewBase(t) tID := testutil.Identifier(t) - reg := testregistry.NewPlainHTTP(base, 5000) - defer reg.Cleanup() + reg := testregistry.NewWithNoAuth(base, 0, false) + defer reg.Cleanup(nil) - pushImageName := fmt.Sprintf("localhost:%d/%s:latest", reg.ListenPort, tID) + pushImageName := fmt.Sprintf("localhost:%d/%s:latest", reg.Port, tID) defer base.Cmd("rmi", pushImageName).Run() base.Cmd("pull", "--all-platforms", testutil.AlpineImage).AssertOK() diff --git a/pkg/testutil/portlock/portlock.go b/pkg/testutil/portlock/portlock.go new file mode 100644 index 00000000000..5e1b5bcacee --- /dev/null +++ b/pkg/testutil/portlock/portlock.go @@ -0,0 +1,64 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +// portlock provides a mechanism for containers to acquire and release ports they plan to expose, and a wait mechanism +// This allows tests dependent on running containers to always parallelize without having to worry about port collision +// with any other test +// Note that this does NOT protect against trying to use a port that is already used by an unrelated third-party service or container +// Also note that *generally* finding a free port is not easy: +// - to just "listen" and see if it works won't work for containerized services that are DNAT-ed (plus, that would be racy) +// - inspecting iptables instead (or in addition to) may work for containers, but this depends on how networking has been set (and yes, it is also racy) +// Our approach here is optimistic: tests are responsible for calling Acquire and Release +package portlock + +import ( + "fmt" + "sync" + "time" +) + +var mut = &sync.Mutex{} //nolint:gochecknoglobals +var portList = map[int]bool{} + +func Acquire(port int) (int, error) { + flexible := false + if port == 0 { + port = 5000 + flexible = true + } + for { + mut.Lock() + if _, ok := portList[port]; !ok { + portList[port] = true + mut.Unlock() + return port, nil + } + mut.Unlock() + if flexible { + port++ + continue + } + fmt.Println("Waiting for port to become available...", port) + time.Sleep(1 * time.Second) + } +} + +func Release(port int) error { + mut.Lock() + delete(portList, port) + mut.Unlock() + return nil +} diff --git a/pkg/testutil/testca/testca.go b/pkg/testutil/testca/testca.go index 3ae92ccc923..53a0eddd37c 100644 --- a/pkg/testutil/testca/testca.go +++ b/pkg/testutil/testca/testca.go @@ -95,12 +95,14 @@ func (c *Cert) Close() error { return c.closeF() } -func (ca *CA) NewCert(host string) *Cert { +func (ca *CA) NewCert(host string, additional ...string) *Cert { t := ca.t key, err := rsa.GenerateKey(rand.Reader, keyLength) assert.NilError(t, err) + additional = append([]string{host}, additional...) + cert := &x509.Certificate{ SerialNumber: serialNumber(t), Subject: pkix.Name{ @@ -111,10 +113,12 @@ func (ca *CA) NewCert(host string) *Cert { NotAfter: time.Now().Add(24 * time.Hour), KeyUsage: x509.KeyUsageCRLSign, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - DNSNames: []string{host}, + DNSNames: additional, } - if ip := net.ParseIP(host); ip != nil { - cert.IPAddresses = append(cert.IPAddresses, ip) + for _, h := range additional { + if ip := net.ParseIP(h); ip != nil { + cert.IPAddresses = append(cert.IPAddresses, ip) + } } dir, err := os.MkdirTemp(t.TempDir(), "cert") diff --git a/pkg/testutil/testregistry/certsd.go b/pkg/testutil/testregistry/certsd.go new file mode 100644 index 00000000000..955bc4ba12f --- /dev/null +++ b/pkg/testutil/testregistry/certsd.go @@ -0,0 +1,47 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package testregistry + +import ( + "fmt" + "net" + "os" + "path/filepath" + "strconv" +) + +func generateCertsd(dir string, certPath string, hostIP string, port int) error { + joined := hostIP + if port != 0 { + joined = net.JoinHostPort(hostIP, strconv.Itoa(port)) + } + + hostsSubDir := filepath.Join(dir, joined) + err := os.MkdirAll(hostsSubDir, 0700) + if err != nil { + return err + } + + hostsTOMLPath := filepath.Join(hostsSubDir, "hosts.toml") + // See https://github.com/containerd/containerd/blob/main/docs/hosts.md + hostsTOML := fmt.Sprintf(` +server = "https://%s" +[host."https://%s"] + ca = %q + `, joined, joined, certPath) + return os.WriteFile(hostsTOMLPath, []byte(hostsTOML), 0700) +} diff --git a/pkg/testutil/testregistry/testregistry_linux.go b/pkg/testutil/testregistry/testregistry_linux.go index 4b15df56e68..b5c05525846 100644 --- a/pkg/testutil/testregistry/testregistry_linux.go +++ b/pkg/testutil/testregistry/testregistry_linux.go @@ -25,175 +25,70 @@ import ( "github.com/containerd/nerdctl/v2/pkg/testutil" "github.com/containerd/nerdctl/v2/pkg/testutil/nettestutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/portlock" "github.com/containerd/nerdctl/v2/pkg/testutil/testca" - "golang.org/x/crypto/bcrypt" "gotest.tools/v3/assert" ) -type TestRegistry struct { - IP net.IP - ListenIP net.IP - ListenPort int - HostsDir string // contains ":/hosts.toml" - Cleanup func() - Logs func() +type RegistryServer struct { + IP net.IP + Port int + Scheme string + ListenIP net.IP + Cleanup func(err error) + Logs func() + HostsDir string // contains ":/hosts.toml" } -func NewPlainHTTP(base *testutil.Base, port int) *TestRegistry { - hostIP, err := nettestutil.NonLoopbackIPv4() - assert.NilError(base.T, err) - // listen on 0.0.0.0 to enable 127.0.0.1 - listenIP := net.ParseIP("0.0.0.0") - listenPort := port - base.T.Logf("hostIP=%q, listenIP=%q, listenPort=%d", hostIP, listenIP, listenPort) +type TokenAuthServer struct { + IP net.IP + Port int + Scheme string + ListenIP net.IP + Cleanup func(err error) + Logs func() + Auth Auth + CertPath string +} - registryContainerName := "reg-" + testutil.Identifier(base.T) - cmd := base.Cmd("run", - "-d", - "-p", fmt.Sprintf("%s:%d:5000", listenIP, listenPort), - "--name", registryContainerName, - testutil.RegistryImage) - cmd.AssertOK() - if _, err = nettestutil.HTTPGet(fmt.Sprintf("http://%s:%d/v2", hostIP.String(), listenPort), 30, false); err != nil { - base.Cmd("rm", "-f", registryContainerName).Run() - base.T.Fatal(err) - } - return &TestRegistry{ - IP: hostIP, - ListenIP: listenIP, - ListenPort: listenPort, - Cleanup: func() { base.Cmd("rm", "-f", registryContainerName).AssertOK() }, - } +func EnsureImages(base *testutil.Base) { + base.Cmd("pull", testutil.RegistryImage).AssertOK() + base.Cmd("pull", testutil.DockerAuthImage).AssertOK() } -func NewAuthWithHTTP(base *testutil.Base, user, pass string, listenPort int, authPort int) *TestRegistry { +func NewAuthServer(base *testutil.Base, ca *testca.CA, port int, user, pass string, tls bool) *TokenAuthServer { name := testutil.Identifier(base.T) - hostIP, err := nettestutil.NonLoopbackIPv4() - assert.NilError(base.T, err) // listen on 0.0.0.0 to enable 127.0.0.1 listenIP := net.ParseIP("0.0.0.0") - base.T.Logf("hostIP=%q, listenIP=%q, listenPort=%d, authPort=%d", hostIP, listenIP, listenPort, authPort) - - ca := testca.New(base.T) - registryCert := ca.NewCert(hostIP.String()) - authCert := ca.NewCert(hostIP.String()) - + hostIP, err := nettestutil.NonLoopbackIPv4() + assert.NilError(base.T, err, fmt.Errorf("failed finding ipv4 non loopback interface: %w", err)) // Prepare configuration file for authentication server // Details: https://github.com/cesanta/docker_auth/blob/1.7.1/examples/simple.yml - authConfigFile, err := os.CreateTemp("", "authconfig") - assert.NilError(base.T, err) + configFile, err := os.CreateTemp("", "authconfig") + assert.NilError(base.T, err, fmt.Errorf("failed creating temporary directory for config file: %w", err)) bpass, err := bcrypt.GenerateFromPassword([]byte(pass), bcrypt.DefaultCost) - assert.NilError(base.T, err) - authConfigFileName := authConfigFile.Name() - _, err = authConfigFile.Write([]byte(fmt.Sprintf(` + assert.NilError(base.T, err, fmt.Errorf("failed bcrypt encrypting password: %w", err)) + configFileName := configFile.Name() + scheme := "http" + configContent := fmt.Sprintf(` server: addr: ":5100" - certificate: "/auth/domain.crt" - key: "/auth/domain.key" token: issuer: "Acme auth server" expiration: 900 + certificate: "/auth/domain.crt" + key: "/auth/domain.key" users: "%s": password: "%s" acl: - match: {account: "%s"} actions: ["*"] -`, user, string(bpass), user))) - assert.NilError(base.T, err) - - // Run authentication server - authContainerName := fmt.Sprintf("auth-%s-%d", name, authPort) - cmd := base.Cmd("run", - "-d", - "-p", fmt.Sprintf("%s:%d:5100", listenIP, authPort), - "--name", authContainerName, - "-v", authCert.CertPath+":/auth/domain.crt", - "-v", authCert.KeyPath+":/auth/domain.key", - "-v", authConfigFileName+":/config/auth_config.yml", - testutil.DockerAuthImage, - "/config/auth_config.yml") - cmd.AssertOK() - - // Run docker_auth-enabled registry - // Details: https://github.com/cesanta/docker_auth/blob/1.7.1/examples/simple.yml - registryContainerName := fmt.Sprintf("%s-%s-%d", "reg", name, listenPort) - cmd = base.Cmd("run", - "-d", - "-p", fmt.Sprintf("%s:%d:5000", listenIP, listenPort), - "--name", registryContainerName, - "--env", "REGISTRY_AUTH=token", - "--env", "REGISTRY_AUTH_TOKEN_REALM="+fmt.Sprintf("https://%s:%d/auth", hostIP.String(), authPort), - "--env", "REGISTRY_AUTH_TOKEN_SERVICE=Docker registry", - "--env", "REGISTRY_AUTH_TOKEN_ISSUER=Acme auth server", - "--env", "REGISTRY_AUTH_TOKEN_ROOTCERTBUNDLE=/auth/domain.crt", - // rootcertbundle is not CA cert: https://github.com/distribution/distribution/issues/1143 - "-v", authCert.CertPath+":/auth/domain.crt", - testutil.RegistryImage) - cmd.AssertOK() - joined := net.JoinHostPort(hostIP.String(), strconv.Itoa(listenPort)) - if _, err = nettestutil.HTTPGet(fmt.Sprintf("http://%s/v2", joined), 30, true); err != nil { - base.Cmd("rm", "-f", registryContainerName).Run() - base.T.Fatal(err) - } - hostsDir, err := os.MkdirTemp(base.T.TempDir(), "certs.d") - assert.NilError(base.T, err) - hostsSubDir := filepath.Join(hostsDir, joined) - err = os.MkdirAll(hostsSubDir, 0700) - assert.NilError(base.T, err) - hostsTOMLPath := filepath.Join(hostsSubDir, "hosts.toml") - // See https://github.com/containerd/containerd/blob/main/docs/hosts.md - hostsTOML := fmt.Sprintf(` -server = "https://%s" -[host."https://%s"] - ca = %q - `, joined, joined, ca.CertPath) - base.T.Logf("Writing %q: %q", hostsTOMLPath, hostsTOML) - err = os.WriteFile(hostsTOMLPath, []byte(hostsTOML), 0700) - assert.NilError(base.T, err) - return &TestRegistry{ - IP: hostIP, - ListenIP: listenIP, - ListenPort: listenPort, - HostsDir: hostsDir, - Cleanup: func() { - base.Cmd("rm", "-f", registryContainerName).AssertOK() - base.Cmd("rm", "-f", authContainerName).AssertOK() - assert.NilError(base.T, registryCert.Close()) - assert.NilError(base.T, authCert.Close()) - assert.NilError(base.T, authConfigFile.Close()) - os.Remove(authConfigFileName) - }, - Logs: func() { - base.T.Logf("%s: %q", registryContainerName, base.Cmd("logs", registryContainerName).Run().String()) - base.T.Logf("%s: %q", authContainerName, base.Cmd("logs", authContainerName).Run().String()) - }, - } -} - -func NewHTTPS(base *testutil.Base, user, pass string) *TestRegistry { - name := testutil.Identifier(base.T) - hostIP, err := nettestutil.NonLoopbackIPv4() - assert.NilError(base.T, err) - // listen on 0.0.0.0 to enable 127.0.0.1 - listenIP := net.ParseIP("0.0.0.0") - const listenPort = 5000 // TODO: choose random empty port - const authPort = 5100 // TODO: choose random empty port - base.T.Logf("hostIP=%q, listenIP=%q, listenPort=%d, authPort=%d", hostIP, listenIP, listenPort, authPort) - - ca := testca.New(base.T) - registryCert := ca.NewCert(hostIP.String()) - authCert := ca.NewCert(hostIP.String()) - - // Prepare configuration file for authentication server - // Details: https://github.com/cesanta/docker_auth/blob/1.7.1/examples/simple.yml - authConfigFile, err := os.CreateTemp("", "authconfig") - assert.NilError(base.T, err) - bpass, err := bcrypt.GenerateFromPassword([]byte(pass), bcrypt.DefaultCost) - assert.NilError(base.T, err) - authConfigFileName := authConfigFile.Name() - _, err = authConfigFile.Write([]byte(fmt.Sprintf(` +`, user, string(bpass), user) + if tls { + scheme = "https" + configContent = fmt.Sprintf(` server: addr: ":5100" certificate: "/auth/domain.crt" @@ -207,78 +102,278 @@ users: acl: - match: {account: "%s"} actions: ["*"] -`, user, string(bpass), user))) - assert.NilError(base.T, err) +`, user, string(bpass), user) + } + _, err = configFile.Write([]byte(configContent)) + assert.NilError(base.T, err, fmt.Errorf("failed writing configuration: %w", err)) - // Run authentication server - authContainerName := "auth-" + name - cmd := base.Cmd("run", - "-d", - "-p", fmt.Sprintf("%s:%d:5100", listenIP, authPort), - "--name", authContainerName, - "-v", authCert.CertPath+":/auth/domain.crt", - "-v", authCert.KeyPath+":/auth/domain.key", - "-v", authConfigFileName+":/config/auth_config.yml", - testutil.DockerAuthImage, - "/config/auth_config.yml") - cmd.AssertOK() - - // Run docker_auth-enabled registry - // Details: https://github.com/cesanta/docker_auth/blob/1.7.1/examples/simple.yml - registryContainerName := "reg-" + name - cmd = base.Cmd("run", - "-d", - "-p", fmt.Sprintf("%s:%d:5000", listenIP, listenPort), - "--name", registryContainerName, + cert := ca.NewCert(hostIP.String()) + + port, err = portlock.Acquire(port) + assert.NilError(base.T, err, fmt.Errorf("failed acquiring port: %w", err)) + containerName := fmt.Sprintf("auth-%s-%d", name, port) + + cleanup := func(err error) { + result := base.Cmd("rm", "-f", containerName).Run() + errPortRelease := portlock.Release(port) + errCertClose := cert.Close() + errConfigClose := configFile.Close() + errConfigRemove := os.Remove(configFileName) + if err == nil { + assert.NilError(base.T, result.Error, fmt.Errorf("failed stopping container: %w", err)) + assert.NilError(base.T, errPortRelease, fmt.Errorf("failed releasing port: %w", err)) + assert.NilError(base.T, errCertClose, fmt.Errorf("failed cleaning certs: %w", err)) + assert.NilError(base.T, errConfigClose, fmt.Errorf("failed closing config file: %w", err)) + assert.NilError(base.T, errConfigRemove, fmt.Errorf("failed removing config file: %w", err)) + } + } + + err = func() error { + // Run authentication server + cmd := base.Cmd( + "run", + "--pull=never", + "-d", + "-p", fmt.Sprintf("%s:%d:5100", listenIP, port), + "--name", containerName, + "-v", cert.CertPath+":/auth/domain.crt", + "-v", cert.KeyPath+":/auth/domain.key", + "-v", configFileName+":/config/auth_config.yml", + testutil.DockerAuthImage, + "/config/auth_config.yml").Run() + if cmd.Error != nil { + base.T.Logf("%s:\n%s\n%s\n-------\n%s", containerName, cmd.Cmd, cmd.Stdout(), cmd.Stderr()) + return cmd.Error + } + joined := net.JoinHostPort(hostIP.String(), strconv.Itoa(port)) + _, err = nettestutil.HTTPGet(fmt.Sprintf("%s://%s/auth", scheme, joined), 30, true) + return err + }() + + if err != nil { + cl := base.Cmd("logs", containerName).Run() + base.T.Logf("%s:\n%s\n%s\n=========================\n%s", containerName, cl.Cmd, cl.Stdout(), cl.Stderr()) + cleanup(err) + } + assert.NilError(base.T, err, fmt.Errorf("failed starting auth container in a timely manner: %w", err)) + + return &TokenAuthServer{ + IP: hostIP, + Port: port, + Scheme: scheme, + ListenIP: listenIP, + CertPath: cert.CertPath, + Auth: &TokenAuth{ + Address: scheme + "://" + net.JoinHostPort(hostIP.String(), strconv.Itoa(port)), + CertPath: cert.CertPath, + }, + Cleanup: cleanup, + Logs: func() { + base.T.Logf("%s: %q", containerName, base.Cmd("logs", containerName).Run().String()) + }, + } + +} + +// Auth is an interface to pass to the test registry for configuring authentication +type Auth interface { + Params(*testutil.Base) []string +} + +type NoAuth struct { +} + +func (na *NoAuth) Params(base *testutil.Base) []string { + return []string{} +} + +type TokenAuth struct { + Address string + CertPath string +} + +func (ta *TokenAuth) Params(base *testutil.Base) []string { + return []string{ "--env", "REGISTRY_AUTH=token", - "--env", "REGISTRY_AUTH_TOKEN_REALM="+fmt.Sprintf("https://%s:%d/auth", hostIP.String(), authPort), + "--env", "REGISTRY_AUTH_TOKEN_REALM=" + ta.Address + "/auth", "--env", "REGISTRY_AUTH_TOKEN_SERVICE=Docker registry", "--env", "REGISTRY_AUTH_TOKEN_ISSUER=Acme auth server", "--env", "REGISTRY_AUTH_TOKEN_ROOTCERTBUNDLE=/auth/domain.crt", - "--env", "REGISTRY_HTTP_TLS_CERTIFICATE=/registry/domain.crt", - "--env", "REGISTRY_HTTP_TLS_KEY=/registry/domain.key", - // rootcertbundle is not CA cert: https://github.com/distribution/distribution/issues/1143 - "-v", authCert.CertPath+":/auth/domain.crt", - "-v", registryCert.CertPath+":/registry/domain.crt", - "-v", registryCert.KeyPath+":/registry/domain.key", - testutil.RegistryImage) - cmd.AssertOK() - joined := net.JoinHostPort(hostIP.String(), strconv.Itoa(listenPort)) - if _, err = nettestutil.HTTPGet(fmt.Sprintf("https://%s/v2", joined), 30, true); err != nil { - base.Cmd("rm", "-f", registryContainerName).Run() - base.T.Fatal(err) + "-v", ta.CertPath + ":/auth/domain.crt", } - hostsDir, err := os.MkdirTemp(base.T.TempDir(), "certs.d") - assert.NilError(base.T, err) - hostsSubDir := filepath.Join(hostsDir, joined) - err = os.MkdirAll(hostsSubDir, 0700) - assert.NilError(base.T, err) - hostsTOMLPath := filepath.Join(hostsSubDir, "hosts.toml") - // See https://github.com/containerd/containerd/blob/main/docs/hosts.md - hostsTOML := fmt.Sprintf(` -server = "https://%s" -[host."https://%s"] - ca = %q - `, joined, joined, ca.CertPath) - base.T.Logf("Writing %q: %q", hostsTOMLPath, hostsTOML) - err = os.WriteFile(hostsTOMLPath, []byte(hostsTOML), 0700) - assert.NilError(base.T, err) - return &TestRegistry{ - IP: hostIP, - ListenIP: listenIP, - ListenPort: listenPort, - HostsDir: hostsDir, - Cleanup: func() { - base.Cmd("rm", "-f", registryContainerName).AssertOK() - base.Cmd("rm", "-f", authContainerName).AssertOK() - assert.NilError(base.T, registryCert.Close()) - assert.NilError(base.T, authCert.Close()) - assert.NilError(base.T, authConfigFile.Close()) - os.Remove(authConfigFileName) - }, +} + +type BasicAuth struct { + Realm string + HtFile string + Username string + Password string +} + +func (ba *BasicAuth) Params(base *testutil.Base) []string { + if ba.Realm == "" { + ba.Realm = "Basic Realm" + } + if ba.HtFile == "" && ba.Username != "" && ba.Password != "" { + pass := ba.Password + encryptedPass, _ := bcrypt.GenerateFromPassword([]byte(pass), bcrypt.DefaultCost) + tmpDir, _ := os.MkdirTemp(base.T.TempDir(), "htpasswd") + ba.HtFile = filepath.Join(tmpDir, "htpasswd") + _ = os.WriteFile(ba.HtFile, []byte(fmt.Sprintf(`%s:%s`, ba.Username, string(encryptedPass[:]))), 0600) + } + ret := []string{ + "--env", "REGISTRY_AUTH=htpasswd", + "--env", "REGISTRY_AUTH_HTPASSWD_REALM=" + ba.Realm, + "--env", "REGISTRY_AUTH_HTPASSWD_PATH=/htpasswd", + } + if ba.HtFile != "" { + ret = append(ret, "-v", ba.HtFile+":/htpasswd") + } + return ret +} + +func NewRegistry(base *testutil.Base, ca *testca.CA, port int, auth Auth, boundCleanup func(error)) *RegistryServer { + name := testutil.Identifier(base.T) + // listen on 0.0.0.0 to enable 127.0.0.1 + listenIP := net.ParseIP("0.0.0.0") + hostIP, err := nettestutil.NonLoopbackIPv4() + assert.NilError(base.T, err, fmt.Errorf("failed finding ipv4 non loopback interface: %w", err)) + port, err = portlock.Acquire(port) + assert.NilError(base.T, err, fmt.Errorf("failed acquiring port: %w", err)) + + containerName := fmt.Sprintf("registry-%s-%d", name, port) + args := []string{ + "run", + "--pull=never", + "-d", + "-p", fmt.Sprintf("%s:%d:5000", listenIP, port), + "--name", containerName, + } + scheme := "http" + var cert *testca.Cert + if ca != nil { + scheme = "https" + cert = ca.NewCert(hostIP.String(), "127.0.0.1") + args = append(args, + "--env", "REGISTRY_HTTP_TLS_CERTIFICATE=/registry/domain.crt", + "--env", "REGISTRY_HTTP_TLS_KEY=/registry/domain.key", + "-v", cert.CertPath+":/registry/domain.crt", + "-v", cert.KeyPath+":/registry/domain.key", + ) + } + + args = append(args, auth.Params(base)...) + args = append(args, testutil.RegistryImage) + + cleanup := func(err error) { + result := base.Cmd("rm", "-f", containerName).Run() + errPortRelease := portlock.Release(port) + var errCertClose error + if cert != nil { + errCertClose = cert.Close() + } + if boundCleanup != nil { + boundCleanup(err) + } + if cert != nil && err == nil { + assert.NilError(base.T, errCertClose, fmt.Errorf("failed cleaning certificates: %w", err)) + } + if err == nil { + assert.NilError(base.T, result.Error, fmt.Errorf("failed removing container: %w", err)) + assert.NilError(base.T, errPortRelease, fmt.Errorf("failed releasing port: %w", err)) + } + } + + hostsDir, err := func() (string, error) { + hDir, err := os.MkdirTemp(base.T.TempDir(), "certs.d") + if err != nil { + return "", err + } + + if ca != nil { + err = generateCertsd(hDir, ca.CertPath, hostIP.String(), port) + if err != nil { + return "", err + } + err = generateCertsd(hDir, ca.CertPath, "127.0.0.1", port) + if err != nil { + return "", err + } + if port == 443 { + err = generateCertsd(hDir, ca.CertPath, hostIP.String(), 0) + if err != nil { + return "", err + } + err = generateCertsd(hDir, ca.CertPath, "127.0.0.1", 0) + if err != nil { + return "", err + } + } + } + + cmd := base.Cmd(args...).Run() + if cmd.Error != nil { + base.T.Logf("%s:\n%s\n%s\n-------\n%s", containerName, cmd.Cmd, cmd.Stdout(), cmd.Stderr()) + return "", cmd.Error + } + + if _, err = nettestutil.HTTPGet(fmt.Sprintf("%s://%s:%s/v2", scheme, hostIP.String(), strconv.Itoa(port)), 30, true); err != nil { + return "", err + } + + return hDir, nil + }() + + if err != nil { + // cs := base.Cmd("inspect", containerName).Run() + // base.T.Logf("%s:\n%s\n%s\n=========================\n%s", containerName, cs.Cmd, cs.Stdout(), cs.Stderr()) + cl := base.Cmd("logs", containerName).Run() + base.T.Logf("%s:\n%s\n%s\n=========================\n%s", containerName, cl.Cmd, cl.Stdout(), cl.Stderr()) + cleanup(err) + } + assert.NilError(base.T, err, fmt.Errorf("failed starting registry container in a timely manner: %w", err)) + + return &RegistryServer{ + IP: hostIP, + Port: port, + Scheme: scheme, + ListenIP: listenIP, + Cleanup: cleanup, Logs: func() { - base.T.Logf("%s: %q", registryContainerName, base.Cmd("logs", registryContainerName).Run().String()) - base.T.Logf("%s: %q", authContainerName, base.Cmd("logs", authContainerName).Run().String()) + base.T.Logf("%s: %q", containerName, base.Cmd("logs", containerName).Run().String()) }, + HostsDir: hostsDir, + } +} + +func NewWithTokenAuth(base *testutil.Base, user, pass string, port int, tls bool) *RegistryServer { + ca := testca.New(base.T) + as := NewAuthServer(base, ca, 0, user, pass, tls) + auth := &TokenAuth{ + Address: as.Scheme + "://" + net.JoinHostPort(as.IP.String(), strconv.Itoa(as.Port)), + CertPath: as.CertPath, + } + return NewRegistry(base, ca, port, auth, as.Cleanup) +} + +func NewWithNoAuth(base *testutil.Base, port int, tls bool) *RegistryServer { + EnsureImages(base) + + var ca *testca.CA + if tls { + ca = testca.New(base.T) + } + return NewRegistry(base, ca, port, &NoAuth{}, nil) +} + +func NewWithBasicAuth(base *testutil.Base, user, pass string, port int, tls bool) *RegistryServer { + auth := &BasicAuth{ + Username: user, + Password: pass, + } + var ca *testca.CA + if tls { + ca = testca.New(base.T) } + return NewRegistry(base, ca, port, auth, nil) } diff --git a/pkg/testutil/testutil.go b/pkg/testutil/testutil.go index e64271bca57..accc5228d72 100644 --- a/pkg/testutil/testutil.go +++ b/pkg/testutil/testutil.go @@ -687,7 +687,7 @@ func newBase(t *testing.T, ns string, ipv6Compatible bool) *Base { IPv6Compatible: ipv6Compatible, } if base.EnableIPv6 && !base.IPv6Compatible { - t.Skip("runner skips non-IPv6 complatible tests in the IPv6 environment") + t.Skip("runner skips non-IPv6 compatible tests in the IPv6 environment") } else if !base.EnableIPv6 && base.IPv6Compatible { t.Skip("runner skips IPv6 compatible tests in the non-IPv6 environment") }