From 7d768be794517e919bf1e9aed9ec683b4024ebc1 Mon Sep 17 00:00:00 2001 From: Hayato Kiwata Date: Wed, 25 Jun 2025 09:33:58 +0000 Subject: [PATCH 1/2] fix: allow containers to start using a large number of ports Suppose we have a compose.yaml that allocates a large numbers of ports as follows. ``` > cat compose.yaml services: svc0: image: alpine command: "sleep infinity" ports: - '32000-32060:32000-32060' ``` When we run `nerdctl compose up -d` using this compose.yaml, we will get the following error. ``` FATA[0000] create container failed validation: containers.Labels: label key and value length (4711 bytes) greater than maximum size (4096 bytes), key: nerdctl/ports: invalid argument FATA[0000] error while creating container haytok-svc0-1: error while creating container haytok-svc0-1: exit status 1 ``` This issue is reported in the following issue. - https://github.com/containerd/nerdctl/issues/4027 This issue is considered to be the same as the one with errors when trying to perform many port mappings, such as `nerdctl run -p 80:80 -p 81:81 ~ -p 1000:1000 ...` The current implementation is processing to create a container with the information specified in -p to the label. And as can be seen from the error message, as the number of ports to be port mapped increases, the creation of the container fails because it violates the limit of the maximum number of bytes on the containerd side that can be allocated for a label. Therefore, this PR modifies the container creation process so that containers can be launched without having to assign the information specified in the -p option to the labels. Specifically, port mapping information is stored in the following path, and when port mapping information is required, it is retrieved from this file. ``` //containers///network-config.json ``` Signed-off-by: Hayato Kiwata --- cmd/nerdctl/compose/compose_port.go | 7 ++ .../compose/compose_port_linux_test.go | 43 +++++++ cmd/nerdctl/compose/compose_ps.go | 48 +++++--- cmd/nerdctl/container/container_port.go | 16 ++- .../container_run_network_linux_test.go | 9 +- pkg/cmd/container/create.go | 16 +-- pkg/cmd/container/inspect.go | 26 ++++- pkg/cmd/container/kill.go | 15 ++- pkg/cmd/container/list.go | 16 ++- pkg/cmd/container/remove.go | 13 +++ pkg/composer/port.go | 14 ++- .../container_network_manager.go | 6 - pkg/containerutil/containerutil.go | 13 +-- pkg/formatter/formatter.go | 10 +- pkg/inspecttypes/dockercompat/dockercompat.go | 21 ++-- .../dockercompat/dockercompat_test.go | 13 ++- pkg/inspecttypes/native/container.go | 2 + pkg/labels/labels.go | 1 + pkg/netutil/networkstore/networkstore.go | 110 ++++++++++++++++++ pkg/ocihook/ocihook.go | 9 +- pkg/portutil/portutil.go | 34 ++++-- pkg/portutil/portutil_test.go | 76 ------------ 22 files changed, 350 insertions(+), 168 deletions(-) create mode 100644 pkg/netutil/networkstore/networkstore.go diff --git a/cmd/nerdctl/compose/compose_port.go b/cmd/nerdctl/compose/compose_port.go index f08b5e9eed7..b4f7b5453d7 100644 --- a/cmd/nerdctl/compose/compose_port.go +++ b/cmd/nerdctl/compose/compose_port.go @@ -88,11 +88,18 @@ func portAction(cmd *cobra.Command, args []string) error { return err } + dataStore, err := clientutil.DataStore(globalOptions.DataRoot, globalOptions.Address) + if err != nil { + return err + } + po := composer.PortOptions{ ServiceName: args[0], Index: index, Port: port, Protocol: protocol, + DataStore: dataStore, + Namespace: globalOptions.Namespace, } return c.Port(ctx, cmd.OutOrStdout(), po) diff --git a/cmd/nerdctl/compose/compose_port_linux_test.go b/cmd/nerdctl/compose/compose_port_linux_test.go index e066a873401..514740be130 100644 --- a/cmd/nerdctl/compose/compose_port_linux_test.go +++ b/cmd/nerdctl/compose/compose_port_linux_test.go @@ -20,7 +20,11 @@ import ( "fmt" "testing" + "github.com/containerd/nerdctl/mod/tigron/expect" + "github.com/containerd/nerdctl/mod/tigron/test" + "github.com/containerd/nerdctl/v2/pkg/testutil" + "github.com/containerd/nerdctl/v2/pkg/testutil/nerdtest" ) func TestComposePort(t *testing.T) { @@ -75,3 +79,42 @@ services: base.ComposeCmd("-f", comp.YAMLFullPath(), "port", "--protocol", "udp", "svc0", "10000").AssertFail() base.ComposeCmd("-f", comp.YAMLFullPath(), "port", "--protocol", "tcp", "svc0", "10001").AssertFail() } + +// TestComposeMultiplePorts tests whether it is possible to allocate a large +// number of ports. (https://github.com/containerd/nerdctl/issues/4027) +func TestComposeMultiplePorts(t *testing.T) { + var dockerComposeYAML = fmt.Sprintf(` +services: + svc0: + image: %s + command: "sleep infinity" + ports: + - '32000-32060:32000-32060' +`, testutil.AlpineImage) + + testCase := nerdtest.Setup() + + testCase.Setup = func(data test.Data, helpers test.Helpers) { + compYamlPath := data.Temp().Save(dockerComposeYAML, "compose.yaml") + data.Labels().Set("composeYaml", compYamlPath) + + helpers.Ensure("compose", "-f", compYamlPath, "up", "-d") + } + + testCase.Cleanup = func(data test.Data, helpers test.Helpers) { + helpers.Anyhow("compose", "-f", data.Temp().Path("compose.yaml"), "down", "-v") + } + + testCase.SubTests = []*test.Case{ + { + Description: "Issue #4027 - Allocate a large number of ports.", + NoParallel: true, + Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { + return helpers.Command("compose", "-f", data.Labels().Get("composeYaml"), "port", "svc0", "32000") + }, + Expected: test.Expects(expect.ExitCodeSuccess, nil, expect.Contains("0.0.0.0:32000")), + }, + } + + testCase.Run(t) +} diff --git a/cmd/nerdctl/compose/compose_ps.go b/cmd/nerdctl/compose/compose_ps.go index badee1755b9..f73b3407d09 100644 --- a/cmd/nerdctl/compose/compose_ps.go +++ b/cmd/nerdctl/compose/compose_ps.go @@ -29,9 +29,9 @@ import ( "github.com/containerd/containerd/v2/core/runtime/restart" "github.com/containerd/errdefs" "github.com/containerd/go-cni" - "github.com/containerd/log" "github.com/containerd/nerdctl/v2/cmd/nerdctl/helpers" + "github.com/containerd/nerdctl/v2/pkg/api/types" "github.com/containerd/nerdctl/v2/pkg/clientutil" "github.com/containerd/nerdctl/v2/pkg/cmd/compose" "github.com/containerd/nerdctl/v2/pkg/containerutil" @@ -183,9 +183,9 @@ func psAction(cmd *cobra.Command, args []string) error { var p composeContainerPrintable var err error if format == "json" { - p, err = composeContainerPrintableJSON(ctx, container) + p, err = composeContainerPrintableJSON(ctx, container, globalOptions) } else { - p, err = composeContainerPrintableTab(ctx, container) + p, err = composeContainerPrintableTab(ctx, container, globalOptions) } if err != nil { return err @@ -234,7 +234,7 @@ func psAction(cmd *cobra.Command, args []string) error { // composeContainerPrintableTab constructs composeContainerPrintable with fields // only for console output. -func composeContainerPrintableTab(ctx context.Context, container containerd.Container) (composeContainerPrintable, error) { +func composeContainerPrintableTab(ctx context.Context, container containerd.Container, gOptions types.GlobalCommandOptions) (composeContainerPrintable, error) { info, err := container.Info(ctx, containerd.WithoutRefreshedMetadata) if err != nil { return composeContainerPrintable{}, err @@ -251,6 +251,18 @@ func composeContainerPrintableTab(ctx context.Context, container containerd.Cont if err != nil { return composeContainerPrintable{}, err } + dataStore, err := clientutil.DataStore(gOptions.DataRoot, gOptions.Address) + if err != nil { + return composeContainerPrintable{}, err + } + containerLabels, err := container.Labels(ctx) + if err != nil { + return composeContainerPrintable{}, err + } + ports, err := portutil.LoadPortMappings(dataStore, gOptions.Namespace, info.ID, containerLabels) + if err != nil { + return composeContainerPrintable{}, err + } return composeContainerPrintable{ Name: info.Labels[labels.Name], @@ -258,13 +270,13 @@ func composeContainerPrintableTab(ctx context.Context, container containerd.Cont Command: formatter.InspectContainerCommandTrunc(spec), Service: info.Labels[labels.ComposeService], State: status, - Ports: formatter.FormatPorts(info.Labels), + Ports: formatter.FormatPorts(ports), }, nil } // composeContainerPrintableJSON constructs composeContainerPrintable with fields // only for json output and compatible docker output. -func composeContainerPrintableJSON(ctx context.Context, container containerd.Container) (composeContainerPrintable, error) { +func composeContainerPrintableJSON(ctx context.Context, container containerd.Container, gOptions types.GlobalCommandOptions) (composeContainerPrintable, error) { info, err := container.Info(ctx, containerd.WithoutRefreshedMetadata) if err != nil { return composeContainerPrintable{}, err @@ -294,6 +306,18 @@ func composeContainerPrintableJSON(ctx context.Context, container containerd.Con if err != nil { return composeContainerPrintable{}, err } + dataStore, err := clientutil.DataStore(gOptions.DataRoot, gOptions.Address) + if err != nil { + return composeContainerPrintable{}, err + } + containerLabels, err := container.Labels(ctx) + if err != nil { + return composeContainerPrintable{}, err + } + portMappings, err := portutil.LoadPortMappings(dataStore, gOptions.Namespace, info.ID, containerLabels) + if err != nil { + return composeContainerPrintable{}, err + } return composeContainerPrintable{ ID: container.ID(), @@ -305,7 +329,7 @@ func composeContainerPrintableJSON(ctx context.Context, container containerd.Con State: state, Health: "", ExitCode: exitCode, - Publishers: formatPublishers(info.Labels), + Publishers: formatPublishers(portMappings), }, nil } @@ -321,7 +345,7 @@ type PortPublisher struct { // formatPublishers parses and returns docker-compatible []PortPublisher from // label map. If an error happens, an empty slice is returned. -func formatPublishers(labelMap map[string]string) []PortPublisher { +func formatPublishers(portMappings []cni.PortMapping) []PortPublisher { mapper := func(pm cni.PortMapping) PortPublisher { return PortPublisher{ URL: pm.HostIP, @@ -332,12 +356,8 @@ func formatPublishers(labelMap map[string]string) []PortPublisher { } var dockerPorts []PortPublisher - if portMappings, err := portutil.ParsePortsLabel(labelMap); err == nil { - for _, p := range portMappings { - dockerPorts = append(dockerPorts, mapper(p)) - } - } else { - log.L.Error(err.Error()) + for _, p := range portMappings { + dockerPorts = append(dockerPorts, mapper(p)) } return dockerPorts } diff --git a/cmd/nerdctl/container/container_port.go b/cmd/nerdctl/container/container_port.go index a6237749789..180cacb3d12 100644 --- a/cmd/nerdctl/container/container_port.go +++ b/cmd/nerdctl/container/container_port.go @@ -29,6 +29,7 @@ import ( "github.com/containerd/nerdctl/v2/pkg/clientutil" "github.com/containerd/nerdctl/v2/pkg/containerutil" "github.com/containerd/nerdctl/v2/pkg/idutil/containerwalker" + "github.com/containerd/nerdctl/v2/pkg/portutil" ) func PortCommand() *cobra.Command { @@ -81,13 +82,26 @@ func portAction(cmd *cobra.Command, args []string) error { } defer cancel() + dataStore, err := clientutil.DataStore(globalOptions.DataRoot, globalOptions.Address) + if err != nil { + return err + } + walker := &containerwalker.ContainerWalker{ Client: client, OnFound: func(ctx context.Context, found containerwalker.Found) error { if found.MatchCount > 1 { return fmt.Errorf("multiple IDs found with provided prefix: %s", found.Req) } - return containerutil.PrintHostPort(ctx, cmd.OutOrStdout(), found.Container, argPort, argProto) + containerLabels, err := found.Container.Labels(ctx) + if err != nil { + return err + } + ports, err := portutil.LoadPortMappings(dataStore, globalOptions.Namespace, found.Container.ID(), containerLabels) + if err != nil { + return err + } + return containerutil.PrintHostPort(ctx, cmd.OutOrStdout(), found.Container, argPort, argProto, ports) }, } req := args[0] diff --git a/cmd/nerdctl/container/container_run_network_linux_test.go b/cmd/nerdctl/container/container_run_network_linux_test.go index b8e0c144f1c..8fb99d42c5c 100644 --- a/cmd/nerdctl/container/container_run_network_linux_test.go +++ b/cmd/nerdctl/container/container_run_network_linux_test.go @@ -36,7 +36,6 @@ import ( "github.com/containerd/containerd/v2/defaults" "github.com/containerd/containerd/v2/pkg/netns" - "github.com/containerd/errdefs" "github.com/containerd/nerdctl/mod/tigron/expect" "github.com/containerd/nerdctl/mod/tigron/require" "github.com/containerd/nerdctl/mod/tigron/test" @@ -409,21 +408,21 @@ func TestRunPort(t *testing.T) { baseTestRunPort(t, testutil.NginxAlpineImage, testutil.NginxAlpineIndexHTMLSnippet, true) } -func TestRunWithInvalidPortThenCleanUp(t *testing.T) { +func TestRunWithManyPortsThenCleanUp(t *testing.T) { testCase := nerdtest.Setup() // docker does not set label restriction to 4096 bytes testCase.Require = require.Not(nerdtest.Docker) testCase.SubTests = []*test.Case{ { - Description: "Run a container with invalid ports, and then clean up.", + Description: "Run a container with many ports, and then clean up.", Command: func(data test.Data, helpers test.Helpers) test.TestableCommand { return helpers.Command("run", "--data-root", data.Temp().Path(), "--rm", "-p", "22200-22299:22200-22299", testutil.CommonImage) }, Expected: func(data test.Data, helpers test.Helpers) *test.Expected { return &test.Expected{ - ExitCode: 1, - Errors: []error{errdefs.ErrInvalidArgument}, + ExitCode: 0, + Errors: []error{}, Output: func(stdout string, t tig.T) { getAddrHash := func(addr string) string { const addrHashLen = 8 diff --git a/pkg/cmd/container/create.go b/pkg/cmd/container/create.go index 76b0ee24137..0dc2cfdc52e 100644 --- a/pkg/cmd/container/create.go +++ b/pkg/cmd/container/create.go @@ -37,7 +37,6 @@ import ( "github.com/containerd/containerd/v2/core/containers" "github.com/containerd/containerd/v2/pkg/cio" "github.com/containerd/containerd/v2/pkg/oci" - "github.com/containerd/go-cni" "github.com/containerd/log" "github.com/containerd/nerdctl/v2/pkg/annotations" @@ -61,6 +60,7 @@ import ( "github.com/containerd/nerdctl/v2/pkg/mountutil" "github.com/containerd/nerdctl/v2/pkg/namestore" "github.com/containerd/nerdctl/v2/pkg/platformutil" + "github.com/containerd/nerdctl/v2/pkg/portutil" "github.com/containerd/nerdctl/v2/pkg/referenceutil" "github.com/containerd/nerdctl/v2/pkg/rootlessutil" "github.com/containerd/nerdctl/v2/pkg/store" @@ -390,6 +390,11 @@ func Create(ctx context.Context, client *containerd.Client, args []string, netMa } cOpts = append(cOpts, ilOpt) + err = portutil.GeneratePortMappingsConfig(dataStore, options.GOptions.Namespace, id, netLabelOpts.PortMappings) + if err != nil { + return nil, generateRemoveOrphanedDirsFunc(ctx, id, dataStore, internalLabels), fmt.Errorf("Error writing to network-config.json: %v", err) + } + opts = append(opts, propagateInternalContainerdLabelsToOCIAnnotations(), oci.WithAnnotations(strutil.ConvertKVStringsToMap(options.Annotations))) @@ -689,7 +694,6 @@ type internalLabels struct { networks []string ipAddress string ip6Address string - ports []cni.PortMapping macAddress string dnsServers []string dnsSearchDomains []string @@ -741,13 +745,6 @@ func withInternalLabels(internalLabels internalLabels) (containerd.NewContainerO return nil, err } m[labels.Networks] = string(networksJSON) - if len(internalLabels.ports) > 0 { - portsJSON, err := json.Marshal(internalLabels.ports) - if err != nil { - return nil, err - } - m[labels.Ports] = string(portsJSON) - } if internalLabels.logURI != "" { m[labels.LogURI] = internalLabels.logURI logConfigJSON, err := json.Marshal(internalLabels.logConfig) @@ -909,7 +906,6 @@ func withHealthcheck(options types.ContainerCreateOptions, ensuredImage *imgutil func (il *internalLabels) loadNetOpts(opts types.NetworkOptions) { il.hostname = opts.Hostname il.domainname = opts.Domainname - il.ports = opts.PortMappings il.ipAddress = opts.IPAddress il.ip6Address = opts.IP6Address il.networks = opts.NetworkSlice diff --git a/pkg/cmd/container/inspect.go b/pkg/cmd/container/inspect.go index 63c359ae51a..f9cdb18308a 100644 --- a/pkg/cmd/container/inspect.go +++ b/pkg/cmd/container/inspect.go @@ -25,19 +25,28 @@ import ( "github.com/containerd/containerd/v2/core/snapshots" "github.com/containerd/nerdctl/v2/pkg/api/types" + "github.com/containerd/nerdctl/v2/pkg/clientutil" "github.com/containerd/nerdctl/v2/pkg/containerdutil" "github.com/containerd/nerdctl/v2/pkg/containerinspector" "github.com/containerd/nerdctl/v2/pkg/idutil/containerwalker" "github.com/containerd/nerdctl/v2/pkg/imgutil" "github.com/containerd/nerdctl/v2/pkg/inspecttypes/dockercompat" + "github.com/containerd/nerdctl/v2/pkg/portutil" ) // Inspect prints detailed information for each container in `containers`. func Inspect(ctx context.Context, client *containerd.Client, containers []string, options types.ContainerInspectOptions) ([]any, error) { + dataStore, err := clientutil.DataStore(options.GOptions.DataRoot, options.GOptions.Address) + if err != nil { + return []any{}, err + } + f := &containerInspector{ mode: options.Mode, size: options.Size, snapshotter: containerdutil.SnapshotService(client, options.GOptions.Snapshotter), + dataStore: dataStore, + namespace: options.GOptions.Namespace, } walker := &containerwalker.ContainerWalker{ @@ -45,7 +54,7 @@ func Inspect(ctx context.Context, client *containerd.Client, containers []string OnFound: f.Handler, } - err := walker.WalkAll(ctx, containers, true) + err = walker.WalkAll(ctx, containers, true) if err != nil { return []any{}, err } @@ -58,6 +67,8 @@ type containerInspector struct { size bool snapshotter snapshots.Snapshotter entries []interface{} + dataStore string + namespace string } func (x *containerInspector) Handler(ctx context.Context, found containerwalker.Found) error { @@ -68,6 +79,19 @@ func (x *containerInspector) Handler(ctx context.Context, found containerwalker. if err != nil { return err } + + containerLabels, err := found.Container.Labels(ctx) + if err != nil { + return err + } + ports, err := portutil.LoadPortMappings(x.dataStore, x.namespace, n.ID, containerLabels) + if err != nil { + return err + } + if n.Process != nil && n.Process.NetNS != nil && len(ports) > 0 { + n.Process.NetNS.PortMappings = ports + } + switch x.mode { case "native": x.entries = append(x.entries, n) diff --git a/pkg/cmd/container/kill.go b/pkg/cmd/container/kill.go index 4f750d54784..080336d9f87 100644 --- a/pkg/cmd/container/kill.go +++ b/pkg/cmd/container/kill.go @@ -33,6 +33,7 @@ import ( "github.com/containerd/log" "github.com/containerd/nerdctl/v2/pkg/api/types" + "github.com/containerd/nerdctl/v2/pkg/clientutil" "github.com/containerd/nerdctl/v2/pkg/containerutil" "github.com/containerd/nerdctl/v2/pkg/idutil/containerwalker" "github.com/containerd/nerdctl/v2/pkg/labels" @@ -122,14 +123,18 @@ func killContainer(ctx context.Context, container containerd.Container, signal s // cleanupNetwork removes cni network setup, specifically the forwards func cleanupNetwork(ctx context.Context, container containerd.Container, globalOpts types.GlobalCommandOptions) error { return rootlessutil.WithDetachedNetNSIfAny(func() error { - // retrieve info to get current active port mappings - info, err := container.Info(ctx, containerd.WithoutRefreshedMetadata) + // retrieve current active port mappings + dataStore, err := clientutil.DataStore(globalOpts.DataRoot, globalOpts.Address) if err != nil { return err } - ports, portErr := portutil.ParsePortsLabel(info.Labels) - if portErr != nil { - return fmt.Errorf("no oci spec: %q", portErr) + containerLabels, err := container.Labels(ctx) + if err != nil { + return err + } + ports, err := portutil.LoadPortMappings(dataStore, globalOpts.Namespace, container.ID(), containerLabels) + if err != nil { + return fmt.Errorf("no oci spec: %q", err) } portMappings := []cni.NamespaceOpts{ cni.WithCapabilityPortMap(ports), diff --git a/pkg/cmd/container/list.go b/pkg/cmd/container/list.go index b23dbb9e14b..3a1d28269e9 100644 --- a/pkg/cmd/container/list.go +++ b/pkg/cmd/container/list.go @@ -32,11 +32,13 @@ import ( "github.com/containerd/log" "github.com/containerd/nerdctl/v2/pkg/api/types" + "github.com/containerd/nerdctl/v2/pkg/clientutil" "github.com/containerd/nerdctl/v2/pkg/containerdutil" "github.com/containerd/nerdctl/v2/pkg/containerutil" "github.com/containerd/nerdctl/v2/pkg/formatter" "github.com/containerd/nerdctl/v2/pkg/imgutil" "github.com/containerd/nerdctl/v2/pkg/labels" + "github.com/containerd/nerdctl/v2/pkg/portutil" ) // List prints containers according to `options`. @@ -162,6 +164,18 @@ func prepareContainers(ctx context.Context, client *containerd.Client, container } else { return nil, fmt.Errorf("can't get container %s status", c.ID()) } + dataStore, err := clientutil.DataStore(options.GOptions.DataRoot, options.GOptions.Address) + if err != nil { + return nil, err + } + containerLabels, err := c.Labels(ctx) + if err != nil { + return nil, err + } + ports, err := portutil.LoadPortMappings(dataStore, options.GOptions.Namespace, c.ID(), containerLabels) + if err != nil { + return nil, err + } li := ListItem{ Command: formatter.InspectContainerCommand(spec, options.Truncate, true), CreatedAt: info.CreatedAt, @@ -169,7 +183,7 @@ func prepareContainers(ctx context.Context, client *containerd.Client, container Image: info.Image, Platform: info.Labels[labels.Platform], Names: containerutil.GetContainerName(info.Labels), - Ports: formatter.FormatPorts(info.Labels), + Ports: formatter.FormatPorts(ports), Status: status, Runtime: info.Runtime.Name, Labels: formatter.FormatLabels(info.Labels), diff --git a/pkg/cmd/container/remove.go b/pkg/cmd/container/remove.go index 1fedcc50432..28048a2f6a1 100644 --- a/pkg/cmd/container/remove.go +++ b/pkg/cmd/container/remove.go @@ -39,6 +39,7 @@ import ( "github.com/containerd/nerdctl/v2/pkg/labels" "github.com/containerd/nerdctl/v2/pkg/mountutil/volumestore" "github.com/containerd/nerdctl/v2/pkg/namestore" + "github.com/containerd/nerdctl/v2/pkg/portutil" "github.com/containerd/nerdctl/v2/pkg/store" ) @@ -191,6 +192,18 @@ func RemoveContainer(ctx context.Context, c containerd.Container, globalOptions } netOpts, err := containerutil.NetworkOptionsFromSpec(spec) + if err != nil { + retErr = err + return + } + + portSlice, err := portutil.LoadPortMappings(dataStore, globalOptions.Namespace, id, containerLabels) + if err != nil { + retErr = err + return + } + netOpts.PortMappings = portSlice + if err == nil { networkManager, err := containerutil.NewNetworkingOptionsManager(globalOptions, netOpts, client) if err != nil { diff --git a/pkg/composer/port.go b/pkg/composer/port.go index f786b4a3923..db2dac8befb 100644 --- a/pkg/composer/port.go +++ b/pkg/composer/port.go @@ -22,6 +22,7 @@ import ( "io" "github.com/containerd/nerdctl/v2/pkg/containerutil" + "github.com/containerd/nerdctl/v2/pkg/portutil" ) // PortOptions has args for getting the public port of a given private port/protocol @@ -31,6 +32,8 @@ type PortOptions struct { Index int Port int Protocol string + DataStore string + Namespace string } // Port gets the corresponding public port of a given private port/protocol @@ -48,6 +51,13 @@ func (c *Composer) Port(ctx context.Context, writer io.Writer, po PortOptions) e po.Index, len(containers), po.ServiceName) } container := containers[po.Index-1] - - return containerutil.PrintHostPort(ctx, writer, container, po.Port, po.Protocol) + containerLabels, err := container.Labels(ctx) + if err != nil { + return err + } + ports, err := portutil.LoadPortMappings(po.DataStore, po.Namespace, container.ID(), containerLabels) + if err != nil { + return err + } + return containerutil.PrintHostPort(ctx, writer, container, po.Port, po.Protocol, ports) } diff --git a/pkg/containerutil/container_network_manager.go b/pkg/containerutil/container_network_manager.go index d28e720a915..d41e3c7e17a 100644 --- a/pkg/containerutil/container_network_manager.go +++ b/pkg/containerutil/container_network_manager.go @@ -893,12 +893,6 @@ func NetworkOptionsFromSpec(spec *specs.Spec) (types.NetworkOptions, error) { } opts.NetworkSlice = networks - if portsJSON := spec.Annotations[labels.Ports]; portsJSON != "" { - if err := json.Unmarshal([]byte(portsJSON), &opts.PortMappings); err != nil { - return opts, err - } - } - return opts, nil } diff --git a/pkg/containerutil/containerutil.go b/pkg/containerutil/containerutil.go index 0bebf2310ea..1559e203196 100644 --- a/pkg/containerutil/containerutil.go +++ b/pkg/containerutil/containerutil.go @@ -42,6 +42,7 @@ import ( "github.com/containerd/containerd/v2/pkg/cio" "github.com/containerd/containerd/v2/pkg/oci" "github.com/containerd/errdefs" + "github.com/containerd/go-cni" "github.com/containerd/log" "github.com/containerd/nerdctl/v2/pkg/consoleutil" @@ -50,7 +51,6 @@ import ( "github.com/containerd/nerdctl/v2/pkg/ipcutil" "github.com/containerd/nerdctl/v2/pkg/labels" "github.com/containerd/nerdctl/v2/pkg/labels/k8slabels" - "github.com/containerd/nerdctl/v2/pkg/portutil" "github.com/containerd/nerdctl/v2/pkg/rootlessutil" "github.com/containerd/nerdctl/v2/pkg/signalutil" "github.com/containerd/nerdctl/v2/pkg/strutil" @@ -59,16 +59,7 @@ import ( // PrintHostPort writes to `writer` the public (HostIP:HostPort) of a given `containerPort/protocol` in a container. // if `containerPort < 0`, it writes all public ports of the container. -func PrintHostPort(ctx context.Context, writer io.Writer, container containerd.Container, containerPort int, proto string) error { - l, err := container.Labels(ctx) - if err != nil { - return err - } - ports, err := portutil.ParsePortsLabel(l) - if err != nil { - return err - } - +func PrintHostPort(ctx context.Context, writer io.Writer, container containerd.Container, containerPort int, proto string, ports []cni.PortMapping) error { if containerPort < 0 { for _, p := range ports { fmt.Fprintf(writer, "%d/%s -> %s:%d\n", p.ContainerPort, p.Protocol, p.HostIP, p.HostPort) diff --git a/pkg/formatter/formatter.go b/pkg/formatter/formatter.go index 3801e1ab208..b72952ce68a 100644 --- a/pkg/formatter/formatter.go +++ b/pkg/formatter/formatter.go @@ -33,9 +33,7 @@ import ( "github.com/containerd/containerd/v2/core/runtime/restart" "github.com/containerd/containerd/v2/pkg/oci" "github.com/containerd/errdefs" - "github.com/containerd/log" - - "github.com/containerd/nerdctl/v2/pkg/portutil" + "github.com/containerd/go-cni" ) func ContainerStatus(ctx context.Context, c containerd.Container) string { @@ -112,11 +110,7 @@ func Ellipsis(str string, maxDisplayWidth int) string { return str[:maxDisplayWidth-1] + "…" } -func FormatPorts(labelMap map[string]string) string { - ports, err := portutil.ParsePortsLabel(labelMap) - if err != nil { - log.L.Error(err.Error()) - } +func FormatPorts(ports []cni.PortMapping) string { if len(ports) == 0 { return "" } diff --git a/pkg/inspecttypes/dockercompat/dockercompat.go b/pkg/inspecttypes/dockercompat/dockercompat.go index fbcf57d0c75..407d7985ab6 100644 --- a/pkg/inspecttypes/dockercompat/dockercompat.go +++ b/pkg/inspecttypes/dockercompat/dockercompat.go @@ -694,7 +694,7 @@ func statusFromNative(x containerd.Status, labels map[string]string) string { } } -func networkSettingsFromNative(n *native.NetNS, sp *specs.Spec) (*NetworkSettings, error) { +func networkSettingsFromNative(n *native.NetNS, _ *specs.Spec) (*NetworkSettings, error) { res := &NetworkSettings{ Networks: make(map[string]*NetworkEndpointSettings), } @@ -737,19 +737,12 @@ func networkSettingsFromNative(n *native.NetNS, sp *specs.Spec) (*NetworkSetting fakeDockerNetworkName := fmt.Sprintf("unknown-%s", x.Name) res.Networks[fakeDockerNetworkName] = nes - if portsLabel, ok := sp.Annotations[labels.Ports]; ok { - var ports []cni.PortMapping - err := json.Unmarshal([]byte(portsLabel), &ports) - if err != nil { - return nil, err - } - nports, err := convertToNatPort(ports) - if err != nil { - return nil, err - } - for portLabel, portBindings := range *nports { - resPortMap[portLabel] = portBindings - } + nports, err := convertToNatPort(n.PortMappings) + if err != nil { + return nil, err + } + for portLabel, portBindings := range *nports { + resPortMap[portLabel] = portBindings } if x.Index == n.PrimaryInterface { diff --git a/pkg/inspecttypes/dockercompat/dockercompat_test.go b/pkg/inspecttypes/dockercompat/dockercompat_test.go index 3e7602d8e67..621e64bf2ff 100644 --- a/pkg/inspecttypes/dockercompat/dockercompat_test.go +++ b/pkg/inspecttypes/dockercompat/dockercompat_test.go @@ -33,6 +33,7 @@ import ( containerd "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/core/containers" "github.com/containerd/containerd/v2/core/images" + "github.com/containerd/go-cni" "github.com/containerd/nerdctl/v2/pkg/healthcheck" "github.com/containerd/nerdctl/v2/pkg/inspecttypes/native" @@ -413,11 +414,17 @@ func TestNetworkSettingsFromNative(t *testing.T) { Addrs: []string{"10.0.4.30/24"}, }, }, + PortMappings: []cni.PortMapping{ + { + HostPort: 8075, + ContainerPort: 77, + Protocol: "tcp", + HostIP: "127.0.0.1", + }, + }, }, s: &specs.Spec{ - Annotations: map[string]string{ - "nerdctl/ports": "[{\"HostPort\":8075,\"ContainerPort\":77,\"Protocol\":\"tcp\",\"HostIP\":\"127.0.0.1\"}]", - }, + Annotations: map[string]string{}, }, expected: &NetworkSettings{ Ports: &nat.PortMap{ diff --git a/pkg/inspecttypes/native/container.go b/pkg/inspecttypes/native/container.go index de015dd5f94..1bd421a2d62 100644 --- a/pkg/inspecttypes/native/container.go +++ b/pkg/inspecttypes/native/container.go @@ -21,6 +21,7 @@ import ( containerd "github.com/containerd/containerd/v2/client" "github.com/containerd/containerd/v2/core/containers" + "github.com/containerd/go-cni" ) // Container corresponds to a containerd-native container object. @@ -43,6 +44,7 @@ type NetNS struct { // Zero means unset. PrimaryInterface int `json:"PrimaryInterface,omitempty"` Interfaces []NetInterface `json:"Interfaces,omitempty"` + PortMappings []cni.PortMapping } // NetInterface wraps net.Interface for JSON marshallability. diff --git a/pkg/labels/labels.go b/pkg/labels/labels.go index 0c50324fee2..792c74dbf9f 100644 --- a/pkg/labels/labels.go +++ b/pkg/labels/labels.go @@ -57,6 +57,7 @@ const ( // Currently, the length of the slice must be 1. Networks = Prefix + "networks" + // DEPRECATED : https://github.com/containerd/nerdctl/pull/4290 // Ports is a JSON-marshalled string of []cni.PortMapping . Ports = Prefix + "ports" diff --git a/pkg/netutil/networkstore/networkstore.go b/pkg/netutil/networkstore/networkstore.go new file mode 100644 index 00000000000..0faa78ba9cf --- /dev/null +++ b/pkg/netutil/networkstore/networkstore.go @@ -0,0 +1,110 @@ +/* + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package networkstore + +import ( + "encoding/json" + "errors" + "fmt" + "path/filepath" + + "github.com/containerd/go-cni" + + "github.com/containerd/nerdctl/v2/pkg/store" +) + +const ( + containersDirBaseName = "containers" + networkConfigName = "network-config.json" +) + +var ErrNetworkStore = errors.New("network-store error") + +func New(dataStore, namespace, containerID string) (ns *NetworkStore, err error) { + defer func() { + if err != nil { + err = errors.Join(ErrNetworkStore, err) + } + }() + + if dataStore == "" || namespace == "" || containerID == "" { + return nil, fmt.Errorf("either dataStore or namespace or containerID is empty") + } + + st, err := store.New(filepath.Join(dataStore, containersDirBaseName, namespace, containerID), 0, 0o600) + if err != nil { + return nil, err + } + + return &NetworkStore{ + safeStore: st, + }, nil +} + +type NetworkStore struct { + safeStore store.Store + + PortMappings []cni.PortMapping +} + +func (ns *NetworkStore) Acquire(portMappings []cni.PortMapping) (err error) { + defer func() { + if err != nil { + err = errors.Join(ErrNetworkStore, err) + } + }() + + portsJSON, err := json.Marshal(portMappings) + if err != nil { + return fmt.Errorf("failed to marshal port mappings to JSON: %w", err) + } + + return ns.safeStore.WithLock(func() error { + return ns.safeStore.Set(portsJSON, networkConfigName) + }) +} + +func (ns *NetworkStore) Load() (err error) { + defer func() { + if err != nil { + err = errors.Join(ErrNetworkStore, err) + } + }() + + return ns.safeStore.WithLock(func() error { + doesExist, err := ns.safeStore.Exists(networkConfigName) + if err != nil || !doesExist { + return err + } + + data, err := ns.safeStore.Get(networkConfigName) + if err != nil { + if errors.Is(err, store.ErrNotFound) { + err = nil + } + return err + } + + var ports []cni.PortMapping + if err := json.Unmarshal(data, &ports); err != nil { + return fmt.Errorf("failed to parse port mappings %v: %w", ports, err) + } + ns.PortMappings = ports + + return err + }) +} diff --git a/pkg/ocihook/ocihook.go b/pkg/ocihook/ocihook.go index a83275e907c..89b6c6b1410 100644 --- a/pkg/ocihook/ocihook.go +++ b/pkg/ocihook/ocihook.go @@ -45,6 +45,7 @@ import ( "github.com/containerd/nerdctl/v2/pkg/netutil" "github.com/containerd/nerdctl/v2/pkg/netutil/nettype" "github.com/containerd/nerdctl/v2/pkg/ocihook/state" + "github.com/containerd/nerdctl/v2/pkg/portutil" "github.com/containerd/nerdctl/v2/pkg/rootlessutil" "github.com/containerd/nerdctl/v2/pkg/store" ) @@ -208,11 +209,11 @@ func newHandlerOpts(state *specs.State, dataStore, cniPath, cniNetconfPath, brid } } - if portsJSON := o.state.Annotations[labels.Ports]; portsJSON != "" { - if err := json.Unmarshal([]byte(portsJSON), &o.ports); err != nil { - return nil, err - } + ports, err := portutil.LoadPortMappings(o.dataStore, namespace, o.state.ID, o.state.Annotations) + if err != nil { + return nil, err } + o.ports = ports if ipAddress, ok := o.state.Annotations[labels.IPAddress]; ok { o.containerIP = ipAddress diff --git a/pkg/portutil/portutil.go b/pkg/portutil/portutil.go index a832470abce..681988c654f 100644 --- a/pkg/portutil/portutil.go +++ b/pkg/portutil/portutil.go @@ -28,6 +28,7 @@ import ( "github.com/containerd/log" "github.com/containerd/nerdctl/v2/pkg/labels" + "github.com/containerd/nerdctl/v2/pkg/netutil/networkstore" "github.com/containerd/nerdctl/v2/pkg/rootlessutil" ) @@ -139,16 +140,35 @@ func ParseFlagP(s string) ([]cni.PortMapping, error) { return mr, nil } -// ParsePortsLabel parses JSON-marshalled string from label map -// (under `labels.Ports` key) and returns []cni.PortMapping. -func ParsePortsLabel(labelMap map[string]string) ([]cni.PortMapping, error) { - portsJSON := labelMap[labels.Ports] - if portsJSON == "" { - return []cni.PortMapping{}, nil +func GeneratePortMappingsConfig(dataStore, namespace, id string, portMappings []cni.PortMapping) error { + ns, err := networkstore.New(dataStore, namespace, id) + if err != nil { + return err } + return ns.Acquire(portMappings) +} + +func LoadPortMappings(dataStore, namespace, id string, containerLabels map[string]string) ([]cni.PortMapping, error) { var ports []cni.PortMapping + + ns, err := networkstore.New(dataStore, namespace, id) + if err != nil { + return ports, err + } + if err = ns.Load(); err != nil { + return ports, err + } + if len(ns.PortMappings) != 0 { + return ns.PortMappings, nil + } + + portsJSON := containerLabels[labels.Ports] + if portsJSON == "" { + return ports, nil + } if err := json.Unmarshal([]byte(portsJSON), &ports); err != nil { - return nil, fmt.Errorf("failed to parse label %q=%q: %s", labels.Ports, portsJSON, err.Error()) + return ports, fmt.Errorf("failed to parse label %q=%q: %s", labels.Ports, portsJSON, err.Error()) } + log.L.Warnf("container %s (%s) is using legacy port mapping configuration. To ensure compatibility with the new port mapping logic, please recreate this container. For more details, see: https://github.com/containerd/nerdctl/pull/4290", containerLabels[labels.Name], id[:12]) return ports, nil } diff --git a/pkg/portutil/portutil_test.go b/pkg/portutil/portutil_test.go index 46b9eff7544..02f390bb9ab 100644 --- a/pkg/portutil/portutil_test.go +++ b/pkg/portutil/portutil_test.go @@ -26,7 +26,6 @@ import ( "github.com/containerd/go-cni" - "github.com/containerd/nerdctl/v2/pkg/labels" "github.com/containerd/nerdctl/v2/pkg/rootlessutil" ) @@ -152,81 +151,6 @@ func TestParseFlagPWithPlatformSpec(t *testing.T) { } } -func TestParsePortsLabel(t *testing.T) { - tests := []struct { - name string - labelMap map[string]string - want []cni.PortMapping - wantErr bool - }{ - { - name: "normal", - labelMap: map[string]string{ - labels.Ports: "[{\"HostPort\":12345,\"ContainerPort\":10000,\"Protocol\":\"tcp\",\"HostIP\":\"0.0.0.0\"}]", - }, - want: []cni.PortMapping{ - { - HostPort: 12345, - ContainerPort: 10000, - Protocol: "tcp", - HostIP: "0.0.0.0", - }, - }, - wantErr: false, - }, - { - name: "empty ports (value empty)", - labelMap: map[string]string{ - labels.Ports: "", - }, - want: []cni.PortMapping{}, - wantErr: false, - }, - { - name: "empty ports (key not exists)", - labelMap: map[string]string{}, - want: []cni.PortMapping{}, - wantErr: false, - }, - { - name: "parse error (wrong format)", - labelMap: map[string]string{ - labels.Ports: "{\"HostPort\":12345,\"ContainerPort\":10000,\"Protocol\":\"tcp\",\"HostIP\":\"0.0.0.0\"}", - }, - want: nil, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := ParsePortsLabel(tt.labelMap) - if err != nil { - t.Log(err) - assert.Equal(t, true, tt.wantErr) - } - if !reflect.DeepEqual(got, tt.want) { - assert.Equal(t, len(got), len(tt.want)) - if len(got) > 0 { - sort.Slice(got, func(i, j int) bool { - return got[i].HostPort < got[j].HostPort - }) - assert.Equal( - t, - got[len(got)-1].HostPort-got[0].HostPort, - got[len(got)-1].ContainerPort-got[0].ContainerPort, - ) - for i := range len(got) { - assert.Equal(t, got[i].HostPort, tt.want[i].HostPort) - assert.Equal(t, got[i].ContainerPort, tt.want[i].ContainerPort) - assert.Equal(t, got[i].Protocol, tt.want[i].Protocol) - assert.Equal(t, got[i].HostIP, tt.want[i].HostIP) - } - } - } - }) - } -} - func TestParseFlagP(t *testing.T) { type args struct { s string From c13417d74a5d1d310f16dc830edef6e24f3b0f00 Mon Sep 17 00:00:00 2001 From: Hayato Kiwata Date: Wed, 25 Jun 2025 09:34:46 +0000 Subject: [PATCH 2/2] docs: add network-config.json description to dir.md Signed-off-by: Hayato Kiwata --- docs/dir.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/dir.md b/docs/dir.md index b3350cddabc..61f5efae3a7 100644 --- a/docs/dir.md +++ b/docs/dir.md @@ -35,6 +35,7 @@ Files: - `-json.log`: used by `nerdctl logs` - `oci-hook.*.log`: logs of the OCI hook - `lifecycle.json`: used to store stateful information about the container that can only be retrieved through OCI hooks +- `network-config.json`: used to store port mapping information for containers run with the `-p` option. ### `//names/` e.g. `/var/lib/nerdctl/1935db59/names/default`