Gpu: Allow/denylist support to ignore devices (e.g. integrated)

cmd/gpu_plugin/README.md

@@ -57,6 +57,8 @@ For workloads on different KMDs, see [KMD and UMD](#kmd-and-umd).
 | -health-management | - | disabled | Enable health management by requesting data from oneAPI/Level-Zero interface. Requires [GPU Level-Zero](../gpu_levelzero/) sidecar. See [health management](#health-management) |
 | -wsl | - | disabled | Adapt plugin to run in the WSL environment. Requires [GPU Level-Zero](../gpu_levelzero/) sidecar. |
 | -shared-dev-num | int | 1 | Number of containers that can share the same GPU device |
+| -allow-ids | string | "" | A list of PCI Device IDs that are allowed to be registered as resources. Default is empty (=all registered). Cannot be used together with `deny-ids`. |
+| -deny-ids | string | "" | A list of PCI Device IDs that are denied to be registered as resources. Default is empty (=all registered). Cannot be used together with `allow-ids`. |
 | -allocation-policy | string | none | 3 possible values: balanced, packed, none. For shared-dev-num > 1: _balanced_ mode spreads workloads among GPU devices, _packed_ mode fills one GPU fully before moving to next, and _none_ selects first available device from kubelet. Default is _none_. |
 
 The plugin also accepts a number of other arguments (common to all plugins) related to logging.

cmd/gpu_plugin/gpu_plugin.go

@@ -69,6 +69,8 @@ const (
 
 type cliOptions struct {
 	preferredAllocationPolicy string
+	allowIDs                  string
+	denyIDs                   string
 	sharedDevNum              int
 	temperatureLimit          int
 	enableMonitoring          bool
@@ -204,6 +206,23 @@ func packedPolicy(req *pluginapi.ContainerPreferredAllocationRequest) []string {
 	return deviceIds
 }
 
+func validatePCIDeviceIDs(pciIDList string) error {
+	r := regexp.MustCompile(`^0x[0-9a-f]{4}$`)
+
+	for id := range strings.SplitSeq(pciIDList, ",") {
+		id = strings.TrimSpace(id)
+		if id == "" {
+			return os.ErrNotExist
+		}
+
+		if !r.MatchString(id) {
+			return os.ErrInvalid
+		}
+	}
+
+	return nil
+}
+
 func (dp *devicePlugin) pciAddressForCard(cardPath, cardName string) (string, error) {
 	linkPath, err := os.Readlink(cardPath)
 	if err != nil {
@@ -585,6 +604,31 @@ func (dp *devicePlugin) filterOutInvalidCards(files []fs.DirEntry) []fs.DirEntry
 			continue
 		}
 
+		allowlist := len(dp.options.allowIDs) > 0
+		denylist := len(dp.options.denyIDs) > 0
+
+		// Skip if the device is either not allowed or denied.
+		if allowlist || denylist {
+			pciID, err := pciDeviceIDForCard(path.Join(dp.sysfsDir, f.Name()))
+			if err != nil {
+				klog.Warningf("Failed to get PCI ID for device %s: %+v", f.Name(), err)
+
+				continue
+			}
+
+			if allowlist && !strings.Contains(dp.options.allowIDs, pciID) {
+				klog.V(4).Infof("Skipping device %s (%s), not in allowlist: %s", f.Name(), pciID, dp.options.allowIDs)
+
+				continue
+			}
+
+			if denylist && strings.Contains(dp.options.denyIDs, pciID) {
+				klog.V(4).Infof("Skipping device %s (%s), in denylist: %s", f.Name(), pciID, dp.options.denyIDs)
+
+				continue
+			}
+		}
+
 		filtered = append(filtered, f)
 	}
 
@@ -710,6 +754,25 @@ func (dp *devicePlugin) Allocate(request *pluginapi.AllocateRequest) (*pluginapi
 	return nil, &dpapi.UseDefaultMethodError{}
 }
 
+func checkAllowDenyOptions(opts cliOptions) bool {
+	if len(opts.allowIDs) > 0 && len(opts.denyIDs) > 0 {
+		klog.Error("Cannot use both allow-ids and deny-ids options at the same time. Please use only one of them.")
+		return false
+	}
+
+	if err := validatePCIDeviceIDs(opts.allowIDs); err != nil {
+		klog.Error("Failed to validate allow-ids: ", err)
+		return false
+	}
+
+	if err := validatePCIDeviceIDs(opts.denyIDs); err != nil {
+		klog.Error("Failed to validate deny-ids: ", err)
+		return false
+	}
+
+	return true
+}
+
 func main() {
 	var (
 		prefix string
@@ -723,6 +786,9 @@ func main() {
 	flag.IntVar(&opts.sharedDevNum, "shared-dev-num", 1, "number of containers sharing the same GPU device")
 	flag.IntVar(&opts.temperatureLimit, "temp-limit", 100, "temperature limit at which device is marked unhealthy")
 	flag.StringVar(&opts.preferredAllocationPolicy, "allocation-policy", "none", "modes of allocating GPU devices: balanced, packed and none")
+	flag.StringVar(&opts.allowIDs, "allow-ids", "", "comma-separated list of device IDs to allow (e.g. 0x49c5,0x49c6)")
+	flag.StringVar(&opts.denyIDs, "deny-ids", "", "comma-separated list of device IDs to deny (e.g. 0x49c5,0x49c6)")
+
 	flag.Parse()
 
 	if opts.sharedDevNum < 1 {
@@ -736,6 +802,12 @@ func main() {
 		os.Exit(1)
 	}
 
+	if !checkAllowDenyOptions(opts) {
+		klog.Error("Invalid allow/deny options.")
+
+		os.Exit(1)
+	}
+
 	klog.V(1).Infof("GPU device plugin started with %s preferred allocation policy", opts.preferredAllocationPolicy)
 
 	plugin := newDevicePlugin(prefix+sysfsDrmDirectory, prefix+devfsDriDirectory, opts)

cmd/gpu_plugin/gpu_plugin_test.go

@@ -361,6 +361,93 @@ func TestScan(t *testing.T) {
 			expectedI915Devs:     1,
 			expectedI915Monitors: 1,
 		},
+		{
+			name:      "two devices with only one allowed",
+			sysfsdirs: []string{"card0/device/drm/card0", "card0/device/drm/controlD64", "card1/device/drm/card1"},
+			sysfsfiles: map[string][]byte{
+				"card0/device/vendor": []byte("0x8086"),
+				"card0/device/device": []byte("0x1234"),
+				"card1/device/vendor": []byte("0x8086"),
+				"card1/device/device": []byte("0x9876"),
+			},
+			symlinkfiles: map[string]string{
+				"card0/device/driver": "drivers/xe",
+				"card1/device/driver": "drivers/i915",
+			},
+			devfsdirs: []string{
+				"card0",
+				"by-path/pci-0000:00:00.0-card",
+				"by-path/pci-0000:00:00.0-render",
+				"card1",
+				"by-path/pci-0000:00:01.0-card",
+				"by-path/pci-0000:00:01.0-render",
+			},
+			options:              cliOptions{enableMonitoring: true, allowIDs: "0x1234"},
+			expectedXeDevs:       1,
+			expectedXeMonitors:   1,
+			expectedI915Devs:     0,
+			expectedI915Monitors: 0,
+		},
+		{
+			name:      "three devices with two allowed",
+			sysfsdirs: []string{"card0/device/drm/card0", "card0/device/drm/controlD64", "card1/device/drm/card1", "card2/device/drm/card2"},
+			sysfsfiles: map[string][]byte{
+				"card0/device/vendor": []byte("0x8086"),
+				"card0/device/device": []byte("0x1234"),
+				"card1/device/vendor": []byte("0x8086"),
+				"card1/device/device": []byte("0x9876"),
+				"card2/device/vendor": []byte("0x8086"),
+				"card2/device/device": []byte("0x0101"),
+			},
+			symlinkfiles: map[string]string{
+				"card0/device/driver": "drivers/xe",
+				"card1/device/driver": "drivers/i915",
+				"card2/device/driver": "drivers/i915",
+			},
+			devfsdirs: []string{
+				"card0",
+				"by-path/pci-0000:00:00.0-card",
+				"by-path/pci-0000:00:00.0-render",
+				"card1",
+				"by-path/pci-0000:00:01.0-card",
+				"by-path/pci-0000:00:01.0-render",
+				"card2",
+				"by-path/pci-0000:00:02.0-card",
+				"by-path/pci-0000:00:02.0-render",
+			},
+			options:              cliOptions{enableMonitoring: true, allowIDs: "0x1234,0x9876"},
+			expectedXeDevs:       1,
+			expectedXeMonitors:   1,
+			expectedI915Devs:     1,
+			expectedI915Monitors: 1,
+		},
+		{
+			name:      "two devices with one denied",
+			sysfsdirs: []string{"card0/device/drm/card0", "card0/device/drm/controlD64", "card1/device/drm/card1"},
+			sysfsfiles: map[string][]byte{
+				"card0/device/vendor": []byte("0x8086"),
+				"card0/device/device": []byte("0x1234"),
+				"card1/device/vendor": []byte("0x8086"),
+				"card1/device/device": []byte("0x9876"),
+			},
+			symlinkfiles: map[string]string{
+				"card0/device/driver": "drivers/xe",
+				"card1/device/driver": "drivers/i915",
+			},
+			devfsdirs: []string{
+				"card0",
+				"by-path/pci-0000:00:00.0-card",
+				"by-path/pci-0000:00:00.0-render",
+				"card1",
+				"by-path/pci-0000:00:01.0-card",
+				"by-path/pci-0000:00:01.0-render",
+			},
+			options:              cliOptions{enableMonitoring: true, denyIDs: "0x1234"},
+			expectedXeDevs:       0,
+			expectedXeMonitors:   0,
+			expectedI915Devs:     1,
+			expectedI915Monitors: 1,
+		},
 		{
 			name:      "sriov-1-pf-no-vfs + monitoring",
 			sysfsdirs: []string{"card0/device/drm/card0", "card0/device/drm/controlD64"},
@@ -1048,3 +1135,61 @@ func TestCDIDeviceInclusion(t *testing.T) {
 		t.Error("Invalid count for device (xe)")
 	}
 }
+
+func TestParsePCIDeviceIDs(t *testing.T) {
+	tests := []struct {
+		name      string
+		input     string
+		wantError bool
+	}{
+		{
+			name:      "valid single ID",
+			input:     "0x1234",
+			wantError: false,
+		},
+		{
+			name:      "valid multiple IDs",
+			input:     "0x1234,0x5678,0x9abc",
+			wantError: false,
+		},
+		{
+			name:      "valid IDs with spaces",
+			input:     " 0x1234 , 0x5678 ",
+			wantError: false,
+		},
+		{
+			name:      "empty string",
+			input:     "",
+			wantError: true,
+		},
+		{
+			name:      "invalid ID format",
+			input:     "0x1234,abcd",
+			wantError: true,
+		},
+		{
+			name:      "invalid hex length",
+			input:     "0x123,0x5678",
+			wantError: true,
+		},
+		{
+			name:      "extra comma",
+			input:     "0x1234,",
+			wantError: true,
+		},
+		{
+			name:      "capita hex",
+			input:     "0xAA12,",
+			wantError: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			err := validatePCIDeviceIDs(tt.input)
+			if (err != nil) != tt.wantError {
+				t.Errorf("parsePCIDeviceIDs() error = %v, wantError %v", err, tt.wantError)
+			}
+		})
+	}
+}

deployments/gpu_plugin/overlays/allowlist-arc/add-args.yaml

@@ -0,0 +1,12 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: intel-gpu-plugin
+spec:
+  template:
+    spec:
+      containers:
+      - name: intel-gpu-plugin
+        args:
+        - "-v=4"
+        - "-allow-ids=0x56a6,0x56a5,0x56a1,0x56a0,0x5694,0x5693,0x5692,0x5691,0x5690,0x56b3,0x56b2,0x56a4,0x56a3,0x5697,0x5696,0x5695,0x56b1,0x56b0,0x56a2,0x56ba,0x56bc,0x56bd,0x56bb"

deployments/gpu_plugin/overlays/allowlist-arc/kustomization.yaml

@@ -0,0 +1,4 @@
+resources:
+  - ../../base
+patches:
+  - path: add-args.yaml

deployments/operator/crd/bases/deviceplugin.intel.com_gpudeviceplugins.yaml

@@ -55,6 +55,20 @@ spec:
           spec:
             description: GpuDevicePluginSpec defines the desired state of GpuDevicePlugin.
             properties:
+              allowIDs:
+                description: |-
+                  AllowIDs is a comma-separated list of PCI IDs of GPU devices that should only be advertised by the plugin.
+                  If not set, all devices are advertised.
+                  The list can contain IDs in the form of '0x1234,0x49a4,0x50b4'.
+                  Cannot be used together with DenyIDs.
+                type: string
+              denyIDs:
+                description: |-
+                  DenyIDs is a comma-separated list of PCI IDs of GPU devices that should only be denied by the plugin.
+                  If not set, all devices are advertised.
+                  The list can contain IDs in the form of '0x1234,0x49a4,0x50b4'.
+                  Cannot be used together with AllowIDs.
+                type: string
               enableMonitoring:
                 description: |-
                   EnableMonitoring enables the monitoring resource ('i915_monitoring')

pkg/apis/deviceplugin/v1/gpudeviceplugin_types.go

@@ -34,6 +34,18 @@ type GpuDevicePluginSpec struct {
 	// InitImage is a container image with tools (e.g., GPU NFD source hook) installed on each node.
 	InitImage string `json:"initImage,omitempty"`
 
+	// AllowIDs is a comma-separated list of PCI IDs of GPU devices that should only be advertised by the plugin.
+	// If not set, all devices are advertised.
+	// The list can contain IDs in the form of '0x1234,0x49a4,0x50b4'.
+	// Cannot be used together with DenyIDs.
+	AllowIDs string `json:"allowIDs,omitempty"`
+
+	// DenyIDs is a comma-separated list of PCI IDs of GPU devices that should only be denied by the plugin.
+	// If not set, all devices are advertised.
+	// The list can contain IDs in the form of '0x1234,0x49a4,0x50b4'.
+	// Cannot be used together with AllowIDs.
+	DenyIDs string `json:"denyIDs,omitempty"`
+
 	// PreferredAllocationPolicy sets the mode of allocating GPU devices on a node.
 	// See documentation for detailed description of the policies. Only valid when SharedDevNum > 1 is set.
 	// +kubebuilder:validation:Enum=balanced;packed;none

pkg/apis/deviceplugin/v1/gpudeviceplugin_webhook.go

@@ -16,14 +16,20 @@ package v1
 
 import (
 	"fmt"
+	"regexp"
+	"strings"
 
 	ctrl "sigs.k8s.io/controller-runtime"
 
 	"github.com/intel/intel-device-plugins-for-kubernetes/pkg/controllers"
 )
 
+var pciIDRegex regexp.Regexp
+
 // SetupWebhookWithManager sets up a webhook for GpuDevicePlugin custom resources.
 func (r *GpuDevicePlugin) SetupWebhookWithManager(mgr ctrl.Manager) error {
+	pciIDRegex = *regexp.MustCompile(`^0x[0-9a-f]{4}$`)
+
 	return ctrl.NewWebhookManagedBy(mgr).
 		For(r).
 		WithDefaulter(&commonDevicePluginDefaulter{
@@ -44,5 +50,33 @@ func (r *GpuDevicePlugin) validatePlugin(ref *commonDevicePluginValidator) error
 		return fmt.Errorf("%w: PreferredAllocationPolicy is valid only when setting sharedDevNum > 1", errValidation)
 	}
 
+	if r.Spec.AllowIDs != "" {
+		for id := range strings.SplitSeq(r.Spec.AllowIDs, ",") {
+			if id == "" {
+				return fmt.Errorf("%w: Empty PCI Device ID in AllowIDs", errValidation)
+			}
+
+			if !pciIDRegex.MatchString(id) {
+				return fmt.Errorf("%w: Invalid PCI Device ID: %s", errValidation, id)
+			}
+		}
+	}
+
+	if r.Spec.DenyIDs != "" {
+		for id := range strings.SplitSeq(r.Spec.DenyIDs, ",") {
+			if id == "" {
+				return fmt.Errorf("%w: Empty PCI Device ID in DenyIDs", errValidation)
+			}
+
+			if !pciIDRegex.MatchString(id) {
+				return fmt.Errorf("%w: Invalid PCI Device ID: %s", errValidation, id)
+			}
+		}
+	}
+
+	if len(r.Spec.AllowIDs) > 0 && len(r.Spec.DenyIDs) > 0 {
+		return fmt.Errorf("%w: AllowIDs and DenyIDs cannot be used together", errValidation)
+	}
+
 	return validatePluginImage(r.Spec.Image, ref.expectedImage, &ref.expectedVersion)
 }