From 7642de0cfabaa50012c062fabdfb0719ed7b1bc9 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Thu, 18 Dec 2025 19:19:49 +0300 Subject: [PATCH 01/37] add dra usbip Signed-off-by: Yaroslav Borbat --- .../virtualization-dra-plugin/werf.inc.yaml | 10 +- .../mount-points.yaml | 4 + .../werf.inc.yaml | 28 + images/virtualization-dra/Taskfile.yaml | 12 + images/virtualization-dra/api/doc.go | 20 + images/virtualization-dra/api/types.go | 48 ++ .../api/zz_generated.deepcopy.go | 51 ++ .../cmd/go-usbip/app/app.go | 51 ++ .../cmd/go-usbip/app/attach.go | 59 ++ .../cmd/go-usbip/app/bind.go | 49 ++ .../cmd/go-usbip/app/detach.go | 55 ++ .../cmd/go-usbip/app/ports.go | 59 ++ .../cmd/go-usbip/app/run.go | 89 +++ .../cmd/go-usbip/app/unbind.go | 49 ++ .../virtualization-dra/cmd/go-usbip/main.go | 36 ++ .../cmd/usb-gateway/app/app.go | 183 ++++++ .../cmd/usb-gateway/app/init.go | 52 ++ .../cmd/usb-gateway/main.go | 36 ++ images/virtualization-dra/go.mod | 57 +- images/virtualization-dra/go.sum | 126 ++-- .../hack/boilerplate.go.txt | 15 + .../virtualization-dra/hack/update-codegen.sh | 28 + .../internal/common/consts.go | 29 + .../internal/featuregates/featuregates.go | 76 +++ .../internal/plugin/driver.go | 6 +- .../controller/resourceclaim/controller.go | 610 ++++++++++++++++++ .../internal/usb-gateway/informer/informer.go | 150 +++++ .../internal/usb-gateway/labeler/labeler.go | 83 +++ .../internal/usb-gateway/prepare/labels.go | 32 + .../internal/usb-gateway/tlsproxy/proxy.go | 97 +++ .../internal/usb/convert.go | 41 +- .../virtualization-dra/internal/usb/device.go | 178 +---- .../internal/usb/discovery.go | 44 +- .../virtualization-dra/internal/usb/store.go | 134 ++-- .../internal/usbip/attacher.go | 302 +++++++++ .../internal/usbip/binder.go | 224 +++++++ .../internal/usbip/interfaces.go | 80 +++ .../internal/usbip/protocol/common.go | 145 +++++ .../internal/usbip/protocol/convert.go | 52 ++ .../internal/usbip/protocol/device_list.go | 249 +++++++ .../internal/usbip/protocol/import.go | 88 +++ .../internal/usbip/sysfs.go | 79 +++ .../internal/usbip/usbip.go | 17 + .../internal/usbip/usbipd.go | 442 +++++++++++++ .../internal/usbip/usbipd_config.go | 164 +++++ .../virtualization-dra/internal/usbip/vhci.go | 213 ++++++ .../pkg/modprobe/modprobe.go | 58 ++ .../virtualization-dra/pkg/usb/discovery.go | 87 +++ images/virtualization-dra/pkg/usb/monitor.go | 180 ++++++ images/virtualization-dra/pkg/usb/speed.go | 46 ++ images/virtualization-dra/pkg/usb/usb.go | 452 +++++++++++++ .../test/pod-with-template-3.yaml | 16 + .../test/resourceclaim-template-2.yaml | 17 + images/virtualization-dra/werf.inc.yaml | 11 +- .../_helper.tpl | 5 + .../daemonset.yaml | 126 ++++ .../rbac-for-us.yaml | 34 + 57 files changed, 5379 insertions(+), 305 deletions(-) create mode 100644 images/virtualization-dra-usb-gateway/mount-points.yaml create mode 100644 images/virtualization-dra-usb-gateway/werf.inc.yaml create mode 100644 images/virtualization-dra/api/doc.go create mode 100644 images/virtualization-dra/api/types.go create mode 100644 images/virtualization-dra/api/zz_generated.deepcopy.go create mode 100644 images/virtualization-dra/cmd/go-usbip/app/app.go create mode 100644 images/virtualization-dra/cmd/go-usbip/app/attach.go create mode 100644 images/virtualization-dra/cmd/go-usbip/app/bind.go create mode 100644 images/virtualization-dra/cmd/go-usbip/app/detach.go create mode 100644 images/virtualization-dra/cmd/go-usbip/app/ports.go create mode 100644 images/virtualization-dra/cmd/go-usbip/app/run.go create mode 100644 images/virtualization-dra/cmd/go-usbip/app/unbind.go create mode 100644 images/virtualization-dra/cmd/go-usbip/main.go create mode 100644 images/virtualization-dra/cmd/usb-gateway/app/app.go create mode 100644 images/virtualization-dra/cmd/usb-gateway/app/init.go create mode 100644 images/virtualization-dra/cmd/usb-gateway/main.go create mode 100644 images/virtualization-dra/hack/boilerplate.go.txt create mode 100755 images/virtualization-dra/hack/update-codegen.sh create mode 100644 images/virtualization-dra/internal/common/consts.go create mode 100644 images/virtualization-dra/internal/featuregates/featuregates.go create mode 100644 images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go create mode 100644 images/virtualization-dra/internal/usb-gateway/informer/informer.go create mode 100644 images/virtualization-dra/internal/usb-gateway/labeler/labeler.go create mode 100644 images/virtualization-dra/internal/usb-gateway/prepare/labels.go create mode 100644 images/virtualization-dra/internal/usb-gateway/tlsproxy/proxy.go create mode 100644 images/virtualization-dra/internal/usbip/attacher.go create mode 100644 images/virtualization-dra/internal/usbip/binder.go create mode 100644 images/virtualization-dra/internal/usbip/interfaces.go create mode 100644 images/virtualization-dra/internal/usbip/protocol/common.go create mode 100644 images/virtualization-dra/internal/usbip/protocol/convert.go create mode 100644 images/virtualization-dra/internal/usbip/protocol/device_list.go create mode 100644 images/virtualization-dra/internal/usbip/protocol/import.go create mode 100644 images/virtualization-dra/internal/usbip/sysfs.go create mode 100644 images/virtualization-dra/internal/usbip/usbip.go create mode 100644 images/virtualization-dra/internal/usbip/usbipd.go create mode 100644 images/virtualization-dra/internal/usbip/usbipd_config.go create mode 100644 images/virtualization-dra/internal/usbip/vhci.go create mode 100644 images/virtualization-dra/pkg/modprobe/modprobe.go create mode 100644 images/virtualization-dra/pkg/usb/discovery.go create mode 100644 images/virtualization-dra/pkg/usb/monitor.go create mode 100644 images/virtualization-dra/pkg/usb/speed.go create mode 100644 images/virtualization-dra/pkg/usb/usb.go create mode 100644 images/virtualization-dra/test/pod-with-template-3.yaml create mode 100644 images/virtualization-dra/test/resourceclaim-template-2.yaml create mode 100644 templates/virtualization-dra-usb-gateway/_helper.tpl create mode 100644 templates/virtualization-dra-usb-gateway/daemonset.yaml create mode 100644 templates/virtualization-dra-usb-gateway/rbac-for-us.yaml diff --git a/images/virtualization-dra-plugin/werf.inc.yaml b/images/virtualization-dra-plugin/werf.inc.yaml index 7615500ef5..dc3f154093 100644 --- a/images/virtualization-dra-plugin/werf.inc.yaml +++ b/images/virtualization-dra-plugin/werf.inc.yaml @@ -9,10 +9,10 @@ import: to: /app/virtualization-dra-plugin after: install {{- if eq $.DEBUG_COMPONENT "delve/virtualization-dra-plugin" }} -- image: debugger - add: /app/dlv - to: /app/dlv - after: install + - image: debugger + add: /app/dlv + to: /app/dlv + after: install {{- end }} imageSpec: config: @@ -22,7 +22,7 @@ imageSpec: env: PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/app/dlv" XDG_CONFIG_HOME: "/tmp" - entrypoint: ["/app/dlv", "--listen=:2345", "--headless=true", "--continue", "--log=true", "--log-output=debugger,debuglineerr,gdbwire,lldbout,rpc", "--accept-multiclient", "--api-version=2", "exec", "/app/virtualization-dra-plugin", "--", "--leader-election=false"] + entrypoint: ["/app/dlv", "--listen=:2345", "--headless=true", "--continue", "--log=true", "--log-output=debugger,debuglineerr,gdbwire,lldbout,rpc", "--accept-multiclient", "--api-version=2", "exec", "/app/virtualization-dra-plugin", "--"] {{- else }} entrypoint: ["/app/virtualization-dra-plugin"] {{- end }} diff --git a/images/virtualization-dra-usb-gateway/mount-points.yaml b/images/virtualization-dra-usb-gateway/mount-points.yaml new file mode 100644 index 0000000000..448596c34b --- /dev/null +++ b/images/virtualization-dra-usb-gateway/mount-points.yaml @@ -0,0 +1,4 @@ +# A list of pre-created mount points for containerd strict mode. + +dirs: [] + diff --git a/images/virtualization-dra-usb-gateway/werf.inc.yaml b/images/virtualization-dra-usb-gateway/werf.inc.yaml new file mode 100644 index 0000000000..e5fbc673e7 --- /dev/null +++ b/images/virtualization-dra-usb-gateway/werf.inc.yaml @@ -0,0 +1,28 @@ +--- +image: {{ .ModuleNamePrefix }}{{ .ImageName }} +fromImage: {{ .ModuleNamePrefix }}distroless +git: + {{- include "image mount points" . }} +import: + - image: {{ .ModuleNamePrefix }}virtualization-dra-builder + add: /out/virtualization-dra-usb-gateway + to: /app/virtualization-dra-usb-gateway + after: install + {{- if eq $.DEBUG_COMPONENT "delve/virtualization-dra-usb-gateway" }} + - image: debugger + add: /app/dlv + to: /app/dlv + after: install + {{- end }} +imageSpec: + config: + user: 64535 + workingDir: "/app" + {{- if eq $.DEBUG_COMPONENT "delve/virtualization-dra-usb-gateway" }} + env: + PATH: "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/app/dlv" + XDG_CONFIG_HOME: "/tmp" + entrypoint: ["/app/dlv", "--listen=:2345", "--headless=true", "--continue", "--log=true", "--log-output=debugger,debuglineerr,gdbwire,lldbout,rpc", "--accept-multiclient", "--api-version=2", "exec", "/app/virtualization-dra-usb-gateway", "--"] + {{- else }} + entrypoint: ["/app/virtualization-dra-usb-gateway"] + {{- end }} diff --git a/images/virtualization-dra/Taskfile.yaml b/images/virtualization-dra/Taskfile.yaml index a28e5823b4..145a70c862 100644 --- a/images/virtualization-dra/Taskfile.yaml +++ b/images/virtualization-dra/Taskfile.yaml @@ -57,3 +57,15 @@ tasks: cmds: - | golangci-lint run + + + build:go-usbip: + desc: "Build go-usbip binary" + cmds: + - go build -o bin/go-usbip cmd/go-usbip/main.go + + api:generate: + desc: "Generate API code" + cmds: + - hack/update-codegen.sh + diff --git a/images/virtualization-dra/api/doc.go b/images/virtualization-dra/api/doc.go new file mode 100644 index 0000000000..01a9011a22 --- /dev/null +++ b/images/virtualization-dra/api/doc.go @@ -0,0 +1,20 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// +k8s:deepcopy-gen=package +// +groupName=dra.virtualization.deckhouse.io + +package api diff --git a/images/virtualization-dra/api/types.go b/images/virtualization-dra/api/types.go new file mode 100644 index 0000000000..abdadcbaa6 --- /dev/null +++ b/images/virtualization-dra/api/types.go @@ -0,0 +1,48 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" +) + +// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object + +type USBGatewayStatus struct { + metav1.TypeMeta `json:",inline"` + + BusNum int `json:"busNum"` + DeviceNum int `json:"deviceNum"` + DevicePath string `json:"devicePath"` + + TargetIP string `json:"targetIP"` + TargetPort int `json:"targetPort"` + Bound bool `json:"bound"` + Attached bool `json:"attached"` +} + +func FromData(data *runtime.RawExtension) *USBGatewayStatus { + if data == nil { + return nil + } + status, ok := data.Object.(*USBGatewayStatus) + if !ok { + return nil + } + return status +} diff --git a/images/virtualization-dra/api/zz_generated.deepcopy.go b/images/virtualization-dra/api/zz_generated.deepcopy.go new file mode 100644 index 0000000000..21ac5d1595 --- /dev/null +++ b/images/virtualization-dra/api/zz_generated.deepcopy.go @@ -0,0 +1,51 @@ +//go:build !ignore_autogenerated +// +build !ignore_autogenerated + +/* +Copyright Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Code generated by deepcopy-gen. DO NOT EDIT. + +package api + +import ( + runtime "k8s.io/apimachinery/pkg/runtime" +) + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *USBGatewayStatus) DeepCopyInto(out *USBGatewayStatus) { + *out = *in + out.TypeMeta = in.TypeMeta + return +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new USBGatewayStatus. +func (in *USBGatewayStatus) DeepCopy() *USBGatewayStatus { + if in == nil { + return nil + } + out := new(USBGatewayStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. +func (in *USBGatewayStatus) DeepCopyObject() runtime.Object { + if c := in.DeepCopy(); c != nil { + return c + } + return nil +} diff --git a/images/virtualization-dra/cmd/go-usbip/app/app.go b/images/virtualization-dra/cmd/go-usbip/app/app.go new file mode 100644 index 0000000000..fbb1fe327f --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/app.go @@ -0,0 +1,51 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import "github.com/spf13/cobra" + +const long = ` + _ _ + __ _ ___ _ _ ___| |__ (_)_ __ + / _' |/ _ \ _____| | | / __| '_ \| | '_ \ +| (_| | (_) |_____| |_| \__ \ |_) | | |_) | +\__, | \___/ \__,_|___/_.__/|_| .__/ +|___/ |_| + + go-usbip is a implementation of USBIP server and client. +` + +func NewUSBIPCommand() *cobra.Command { + cmd := &cobra.Command{ + Use: "usbip", + Short: "USBIP command line tool", + Long: long, + SilenceUsage: true, + SilenceErrors: true, + } + + cmd.AddCommand( + NewRunCommand(), + NewBindCommand(), + NewUnbindCommand(), + NewAttachCommand(), + NewDetachCommand(), + NewUsedPortsCommand(), + ) + + return cmd +} diff --git a/images/virtualization-dra/cmd/go-usbip/app/attach.go b/images/virtualization-dra/cmd/go-usbip/app/attach.go new file mode 100644 index 0000000000..818f983a00 --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/attach.go @@ -0,0 +1,59 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +func NewAttachCommand() *cobra.Command { + o := &attachOptions{} + cmd := &cobra.Command{ + Use: "attach [:host:] [:busID:]", + Short: "Attach USB devices to USBIP server", + Example: o.Usage(), + RunE: o.Run, + Args: cobra.ExactArgs(2), + } + + o.AddFlags(cmd.Flags()) + + return cmd +} + +type attachOptions struct { + port int +} + +func (o *attachOptions) Usage() string { + return ` # Attach USB devices to USBIP server + $ go-usbip attach 192.168.1.1 3-2.1.1 +` +} + +func (o *attachOptions) AddFlags(fs *pflag.FlagSet) { + fs.IntVar(&o.port, "port", 3240, "Remote port for attaching") +} + +func (o *attachOptions) Run(_ *cobra.Command, args []string) error { + host := args[0] + busID := args[1] + return usbip.NewUSBAttacher().Attach(host, busID, o.port) +} diff --git a/images/virtualization-dra/cmd/go-usbip/app/bind.go b/images/virtualization-dra/cmd/go-usbip/app/bind.go new file mode 100644 index 0000000000..31b505ac5b --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/bind.go @@ -0,0 +1,49 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +func NewBindCommand() *cobra.Command { + o := &bindOptions{} + cmd := &cobra.Command{ + Use: "bind [:busID:]", + Short: "Bind USB devices to USBIP server", + Example: o.Usage(), + RunE: o.Run, + Args: cobra.ExactArgs(1), + } + + return cmd +} + +type bindOptions struct{} + +func (o *bindOptions) Usage() string { + return ` # Bind USB devices to USBIP server + $ go-usbip bind 3-2.1.1 +` +} + +func (o *bindOptions) Run(_ *cobra.Command, args []string) error { + busID := args[0] + return usbip.NewUSBBinder().Bind(busID) +} diff --git a/images/virtualization-dra/cmd/go-usbip/app/detach.go b/images/virtualization-dra/cmd/go-usbip/app/detach.go new file mode 100644 index 0000000000..15c2654407 --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/detach.go @@ -0,0 +1,55 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "strconv" + + "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +func NewDetachCommand() *cobra.Command { + o := &detachOptions{} + cmd := &cobra.Command{ + Use: "detach [:port:]", + Short: "Detach USB devices from USBIP server", + Example: o.Usage(), + RunE: o.Run, + Args: cobra.ExactArgs(1), + } + + return cmd +} + +type detachOptions struct{} + +func (o *detachOptions) Usage() string { + return ` # Detach USB devices from USBIP server + $ go-usbip detach 0 +` +} + +func (o *detachOptions) Run(_ *cobra.Command, args []string) error { + port, err := strconv.Atoi(args[0]) + if err != nil { + return fmt.Errorf("invalid port: %w", err) + } + return usbip.NewUSBAttacher().Detach(port) +} diff --git a/images/virtualization-dra/cmd/go-usbip/app/ports.go b/images/virtualization-dra/cmd/go-usbip/app/ports.go new file mode 100644 index 0000000000..634e519071 --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/ports.go @@ -0,0 +1,59 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +func NewUsedPortsCommand() *cobra.Command { + o := &usedPortsOptions{} + cmd := &cobra.Command{ + Use: "ports", + Short: "List used ports", + Example: o.Usage(), + RunE: o.Run, + } + + return cmd +} + +type usedPortsOptions struct{} + +func (o *usedPortsOptions) Usage() string { + return ` # List used ports + $ go-usbip ports +` +} + +func (o *usedPortsOptions) Run(cmd *cobra.Command, _ []string) error { + ports, err := usbip.NewUSBAttacher().GetUsedPorts() + if err != nil { + return err + } + + cmd.Println("Used ports:") + for _, port := range ports { + cmd.Println(fmt.Sprintf("- %d", port)) + } + + return nil +} diff --git a/images/virtualization-dra/cmd/go-usbip/app/run.go b/images/virtualization-dra/cmd/go-usbip/app/run.go new file mode 100644 index 0000000000..0962f174b2 --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/run.go @@ -0,0 +1,89 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "context" + "time" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + + "github.com/deckhouse/virtualization-dra/internal/usbip" + "github.com/deckhouse/virtualization-dra/pkg/usb" +) + +func NewRunCommand() *cobra.Command { + o := &runOptions{} + cmd := &cobra.Command{ + Use: "run", + Short: "Run USBIP server", + Example: o.Usage(), + RunE: o.Run, + Args: cobra.NoArgs, + } + + o.AddFlags(cmd.Flags()) + + return cmd +} + +type runOptions struct { + port int + resyncPeriod time.Duration +} + +func (o *runOptions) Usage() string { + return ` # Run USBIP server + $ go-usbip run +` +} + +func (o *runOptions) AddFlags(fs *pflag.FlagSet) { + fs.IntVar(&o.port, "port", 3240, "Port to listen on") + fs.DurationVar(&o.resyncPeriod, "resync-period", time.Second*300, "Resync period") +} + +func (o *runOptions) Run(cmd *cobra.Command, _ []string) error { + monitor, err := usb.NewMonitor(context.Background(), o.resyncPeriod) + if err != nil { + return err + } + + config := usbip.USBIPDConfig{ + Port: o.port, + Monitor: monitor, + } + err = config.Validate() + if err != nil { + return err + } + + usbipd, err := config.Complete() + if err != nil { + return err + } + + err = usbipd.Start(cmd.Context()) + if err != nil { + return err + } + + <-cmd.Context().Done() + + return nil +} diff --git a/images/virtualization-dra/cmd/go-usbip/app/unbind.go b/images/virtualization-dra/cmd/go-usbip/app/unbind.go new file mode 100644 index 0000000000..a2bc72d792 --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/unbind.go @@ -0,0 +1,49 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +func NewUnbindCommand() *cobra.Command { + o := &unbindOptions{} + cmd := &cobra.Command{ + Use: "unbind [:busID:]", + Short: "Unbind USB devices from USBIP server", + Example: o.Usage(), + RunE: o.Run, + Args: cobra.ExactArgs(1), + } + + return cmd +} + +type unbindOptions struct{} + +func (o *unbindOptions) Usage() string { + return ` # Unbind USB devices from USBIP server + $ go-usbip unbind 3-2.1.1 +` +} + +func (o *unbindOptions) Run(_ *cobra.Command, args []string) error { + busID := args[0] + return usbip.NewUSBBinder().Unbind(busID) +} diff --git a/images/virtualization-dra/cmd/go-usbip/main.go b/images/virtualization-dra/cmd/go-usbip/main.go new file mode 100644 index 0000000000..ee2334bb7b --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/main.go @@ -0,0 +1,36 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "log/slog" + "os" + "os/signal" + "syscall" + + "github.com/deckhouse/virtualization-dra/cmd/go-usbip/app" +) + +func main() { + ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + + if err := app.NewUSBIPCommand().ExecuteContext(ctx); err != nil { + slog.Error("failed to execute command", slog.Any("err", err)) + os.Exit(1) + } +} diff --git a/images/virtualization-dra/cmd/usb-gateway/app/app.go b/images/virtualization-dra/cmd/usb-gateway/app/app.go new file mode 100644 index 0000000000..0bd715c0df --- /dev/null +++ b/images/virtualization-dra/cmd/usb-gateway/app/app.go @@ -0,0 +1,183 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + "net" + "os" + "time" + + "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + "k8s.io/component-base/cli/flag" + + "github.com/deckhouse/virtualization-dra/internal/featuregates" + "github.com/deckhouse/virtualization-dra/internal/usb-gateway/controller/resourceclaim" + "github.com/deckhouse/virtualization-dra/internal/usb-gateway/informer" + "github.com/deckhouse/virtualization-dra/internal/usb-gateway/prepare" + "github.com/deckhouse/virtualization-dra/internal/usbip" + "github.com/deckhouse/virtualization-dra/pkg/logger" + "github.com/deckhouse/virtualization-dra/pkg/usb" +) + +func NewUSBGatewayCommand() *cobra.Command { + o := newUsbOptions() + + cmd := &cobra.Command{ + Use: "usb-gateway", + Short: "USB gateway", + Long: "USB gateway", + SilenceUsage: true, + SilenceErrors: true, + PreRunE: func(cmd *cobra.Command, args []string) error { + if err := o.Validate(); err != nil { + return err + } + log := o.Logging.Complete() + logger.SetDefaultLogger(log) + return nil + }, + RunE: o.Run, + } + + cmd.AddCommand(NewInitCommand()) + + fs := cmd.Flags() + for _, f := range o.NamedFlags().FlagSets { + fs.AddFlagSet(f) + } + + return cmd +} + +func newUsbOptions() *usbOptions { + return &usbOptions{ + Logging: &logger.Options{}, + featureGates: featuregates.AddFlags, + } +} + +type usbOptions struct { + Kubeconfig string + NodeName string + PodIP string + USBIPPort int + USBResyncPeriod time.Duration + Logging *logger.Options + featureGates featuregates.AddFlagsFunc +} + +func (o *usbOptions) NamedFlags() (fs flag.NamedFlagSets) { + mfs := fs.FlagSet("usb-gateway") + mfs.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to kubeconfig file") + mfs.StringVar(&o.NodeName, "node-name", os.Getenv("NODE_NAME"), "Node name") + mfs.StringVar(&o.PodIP, "pod-ip", os.Getenv("POD_IP"), "Pod IP") + mfs.IntVar(&o.USBIPPort, "usbip-port", 3240, "USBIP port") + mfs.DurationVar(&o.USBResyncPeriod, "usb-resync-period", 5*time.Minute, "USB resync period") + + o.Logging.AddFlags(fs.FlagSet("logging")) + + o.featureGates(fs.FlagSet("feature-gates")) + + return fs +} + +func (o *usbOptions) Validate() error { + if o.NodeName == "" { + return fmt.Errorf("NodeName is required") + } + if o.PodIP == "" { + return fmt.Errorf("PodIP is required") + } + if net.ParseIP(o.PodIP) == nil { + return fmt.Errorf("PodIP is not a valid IP address") + } + if o.USBIPPort < 1 || o.USBIPPort > 65535 { + return fmt.Errorf("USBIPPort is not a valid port number") + } + + return nil +} + +func (o *usbOptions) Run(cmd *cobra.Command, _ []string) error { + monitor, err := usb.NewMonitor(cmd.Context(), o.USBResyncPeriod) + if err != nil { + return err + } + config := usbip.USBIPDConfig{ + Port: o.USBIPPort, + Monitor: monitor, + } + err = config.Validate() + if err != nil { + return err + } + + usbipd, err := config.Complete() + if err != nil { + return err + } + + cfg, err := clientcmd.BuildConfigFromFlags("", o.Kubeconfig) + if err != nil { + return fmt.Errorf("failed to get rest config: %w", err) + } + + client, err := kubernetes.NewForConfig(cfg) + if err != nil { + return fmt.Errorf("failed to create kubernetes client: %w", err) + } + + dynamicClient, err := dynamic.NewForConfig(cfg) + if err != nil { + return fmt.Errorf("failed to create dynamic client: %w", err) + } + + if err = prepare.MarkNodeForUSBGateway(cmd.Context(), o.NodeName, dynamicClient); err != nil { + return fmt.Errorf("failed to mark node for USB gateway: %w", err) + } + + f := informer.NewFactory(client, nil) + resourceClaimInformer := f.ResourceClaim() + resourceSliceInformer := f.ResourceSlice() + nodeInformer := f.Nodes() + podInformer := f.Pods() + + f.Start(cmd.Context().Done()) + f.WaitForCacheSync(cmd.Context().Done()) + + ip := net.ParseIP(o.PodIP) + usbIPInterface := usbip.New() + c, err := resourceclaim.NewController(o.NodeName, ip, o.USBIPPort, client, resourceClaimInformer, resourceSliceInformer, nodeInformer, podInformer, usbIPInterface) + if err != nil { + return fmt.Errorf("failed to create resourceclaim controller: %w", err) + } + + group, ctx := errgroup.WithContext(cmd.Context()) + group.Go(func() error { + return usbipd.Run(ctx) + }) + group.Go(func() error { + return c.Run(ctx, 1) + }) + + return group.Wait() +} diff --git a/images/virtualization-dra/cmd/usb-gateway/app/init.go b/images/virtualization-dra/cmd/usb-gateway/app/init.go new file mode 100644 index 0000000000..a435e880dd --- /dev/null +++ b/images/virtualization-dra/cmd/usb-gateway/app/init.go @@ -0,0 +1,52 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization-dra/pkg/modprobe" +) + +func NewInitCommand() *cobra.Command { + o := &initOptions{} + + cmd := &cobra.Command{ + Use: "init", + Short: "Init USB gateway", + RunE: o.Run, + } + + return cmd +} + +type initOptions struct{} + +func (o *initOptions) Run(_ *cobra.Command, _ []string) error { + modules := []string{ + "kernel/drivers/usb/usbip/usbip-core.ko", + "kernel/drivers/usb/usbip/vhci-hcd.ko", + } + + if err := modprobe.LoadModules(modules); err != nil { + return fmt.Errorf("failed to load modules: %w", err) + } + + return nil +} diff --git a/images/virtualization-dra/cmd/usb-gateway/main.go b/images/virtualization-dra/cmd/usb-gateway/main.go new file mode 100644 index 0000000000..4ab9d51646 --- /dev/null +++ b/images/virtualization-dra/cmd/usb-gateway/main.go @@ -0,0 +1,36 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "context" + "log/slog" + "os" + "os/signal" + "syscall" + + "github.com/deckhouse/virtualization-dra/cmd/usb-gateway/app" +) + +func main() { + ctx, _ := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + + if err := app.NewUSBGatewayCommand().ExecuteContext(ctx); err != nil { + slog.Error("failed to execute command", slog.Any("err", err)) + os.Exit(1) + } +} diff --git a/images/virtualization-dra/go.mod b/images/virtualization-dra/go.mod index a6363e2f74..9bac9000d9 100644 --- a/images/virtualization-dra/go.mod +++ b/images/virtualization-dra/go.mod @@ -1,38 +1,47 @@ module github.com/deckhouse/virtualization-dra -go 1.24.7 +go 1.25.0 -tool github.com/onsi/ginkgo/v2/ginkgo +tool ( + github.com/onsi/ginkgo/v2/ginkgo + k8s.io/code-generator +) require ( github.com/containerd/nri v0.10.0 github.com/deckhouse/deckhouse/pkg/log v0.1.0 - github.com/go-logr/logr v1.4.2 + github.com/fsnotify/fsnotify v1.5.1 + github.com/go-logr/logr v1.4.3 github.com/godbus/dbus/v5 v5.2.0 - github.com/onsi/ginkgo/v2 v2.21.0 - github.com/onsi/gomega v1.35.1 + github.com/onsi/ginkgo/v2 v2.27.2 + github.com/onsi/gomega v1.38.2 github.com/spf13/cobra v1.10.1 github.com/spf13/pflag v1.0.9 + golang.org/x/sync v0.18.0 + golang.org/x/sys v0.38.0 google.golang.org/grpc v1.72.1 k8s.io/api v0.34.2 - k8s.io/apimachinery v0.34.2 + k8s.io/apimachinery v0.35.0 k8s.io/client-go v0.34.2 k8s.io/component-base v0.34.2 k8s.io/dynamic-resource-allocation v0.34.2 k8s.io/klog/v2 v2.130.1 k8s.io/kubelet v0.34.2 - k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 + k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 tags.cncf.io/container-device-interface v1.0.1 tags.cncf.io/container-device-interface/specs-go v1.0.0 ) require ( github.com/DataDog/gostackparse v0.7.0 // indirect + github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver/v4 v4.0.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/containerd/ttrpc v1.2.7 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/emicklei/go-restful/v3 v3.12.2 // indirect - github.com/fsnotify/fsnotify v1.5.1 // indirect github.com/fxamacker/cbor/v2 v2.9.0 // indirect github.com/go-openapi/jsonpointer v0.21.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect @@ -41,7 +50,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/google/gnostic-models v0.7.0 // indirect github.com/google/go-cmp v0.7.0 // indirect - github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db // indirect + github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 // indirect github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -53,32 +62,38 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect - github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_golang v1.22.0 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.62.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect github.com/sirupsen/logrus v1.9.3 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect github.com/tetratelabs/wazero v1.9.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.etcd.io/etcd/client/pkg/v3 v3.6.4 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect - go.yaml.in/yaml/v2 v2.4.2 // indirect + go.yaml.in/yaml/v2 v2.4.3 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect - golang.org/x/mod v0.21.0 // indirect - golang.org/x/net v0.38.0 // indirect + golang.org/x/mod v0.29.0 // indirect + golang.org/x/net v0.47.0 // indirect golang.org/x/oauth2 v0.27.0 // indirect - golang.org/x/sys v0.31.0 // indirect - golang.org/x/term v0.30.0 // indirect - golang.org/x/text v0.23.0 // indirect + golang.org/x/term v0.37.0 // indirect + golang.org/x/text v0.31.0 // indirect golang.org/x/time v0.9.0 // indirect - golang.org/x/tools v0.26.0 // indirect + golang.org/x/tools v0.38.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb // indirect - google.golang.org/protobuf v1.36.5 // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + google.golang.org/protobuf v1.36.8 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + k8s.io/code-generator v0.35.0 // indirect + k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b // indirect + k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect sigs.k8s.io/yaml v1.6.0 // indirect diff --git a/images/virtualization-dra/go.sum b/images/virtualization-dra/go.sum index dea9d5d69f..8127885373 100644 --- a/images/virtualization-dra/go.sum +++ b/images/virtualization-dra/go.sum @@ -1,7 +1,13 @@ github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4= github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM= +github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= +github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/nri v0.10.0 h1:bt2NzfvlY6OJE0i+fB5WVeGQEycxY7iFVQpEbh7J3Go= @@ -21,8 +27,14 @@ github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWp github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= -github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= -github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= +github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= +github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= +github.com/gkampitakis/go-diff v1.3.2/go.mod h1:LLgOrpqleQe26cte8s36HTWcTmMEur6OPYerdAAS9tk= +github.com/gkampitakis/go-snaps v0.5.15 h1:amyJrvM1D33cPHwVrjo9jQxX8g/7E2wYdZ+01KS3zGE= +github.com/gkampitakis/go-snaps v0.5.15/go.mod h1:HNpx/9GoKisdhw9AFOBT1N7DBs9DiHo/hGheFGBZ+mc= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= @@ -35,6 +47,8 @@ github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+Gr github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= +github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= +github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/godbus/dbus/v5 v5.2.0 h1:3WexO+U+yg9T70v9FdHr9kCxYlazaAXUhx2VMkbfax8= github.com/godbus/dbus/v5 v5.2.0/go.mod h1:3AAv2+hPq5rdnr5txxxRwiGjPXamgoIHgz9FPBfOp3c= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -46,8 +60,8 @@ github.com/google/gnostic-models v0.7.0/go.mod h1:whL5G0m6dmc5cPxKc5bdKdEN3UjI7O github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db h1:097atOisP2aRj7vFgYQBbFN4U4JNXUNYpxael3UzMyo= -github.com/google/pprof v0.0.0-20241029153458-d1b30febd7db/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6 h1:BHT72Gu3keYf3ZEu2J0b1vyeLSOYI8bm5wbJM/8yDe8= +github.com/google/pprof v0.0.0-20250403155104-27863c87afa6/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -59,10 +73,14 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= +github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= github.com/knqyf263/go-plugin v0.9.0 h1:CQs2+lOPIlkZVtcb835ZYDEoyyWJWLbSTWeCs0EwTwI= github.com/knqyf263/go-plugin v0.9.0/go.mod h1:2z5lCO1/pez6qGo8CvCxSlBFSEat4MEp1DrnA+f7w8Q= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -72,8 +90,14 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= +github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= +github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= +github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= @@ -84,10 +108,10 @@ github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWu github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= -github.com/onsi/ginkgo/v2 v2.21.0 h1:7rg/4f3rB88pb5obDgNZrNHrQ4e6WpjonchcpuBRnZM= -github.com/onsi/ginkgo/v2 v2.21.0/go.mod h1:7Du3c42kxCUegi0IImZ1wUQzMBVecgIHjR1C+NkhLQo= -github.com/onsi/gomega v1.35.1 h1:Cwbd75ZBPxFSuZ6T+rN/WCb/gOc6YgFBXLlZLhC7Ds4= -github.com/onsi/gomega v1.35.1/go.mod h1:PvZbdDc8J6XJEpDK4HCuRBm8a6Fzp9/DmhC9C7yFlog= +github.com/onsi/ginkgo/v2 v2.27.2 h1:LzwLj0b89qtIy6SSASkzlNvX6WktqurSHwkk2ipF/Ns= +github.com/onsi/ginkgo/v2 v2.27.2/go.mod h1:ArE1D/XhNXBXCBkKOLkbsb2c81dQHCRcF5zwn/ykDRo= +github.com/onsi/gomega v1.38.2 h1:eZCjf2xjZAqe+LeWvKb5weQ+NcPwX84kqJ0cZNxok2A= +github.com/onsi/gomega v1.38.2/go.mod h1:W2MJcYxRGV63b418Ai34Ud0hEdTVXq9NW9+Sx6uXf3k= github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -96,14 +120,18 @@ github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go. github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU= github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.62.0 h1:xasJaQlnWAeyHdUBeGjXmutelfJHWMRr+Fg4QszZ2Io= +github.com/prometheus/common v0.62.0/go.mod h1:vyBcEuLSvWos9B1+CyL7JZ2up+uFzXhkqml0W5zIY1I= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -123,12 +151,20 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= -github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tetratelabs/wazero v1.9.0 h1:IcZ56OuxrtaEz8UYNRHBrUa9bYeX9oVY93KspZZBf/I= github.com/tetratelabs/wazero v1.9.0/go.mod h1:TSbcXCfFP0L2FGkRPxHphadXPjo1T6W+CseNNY7EkjM= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= +github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= @@ -160,8 +196,8 @@ go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -go.yaml.in/yaml/v2 v2.4.2 h1:DzmwEr2rDGHl7lsFgAHxmNz/1NlQ7xLIrlN2h5d1eGI= -go.yaml.in/yaml/v2 v2.4.2/go.mod h1:081UH+NErpNdqlCXm3TtEran0rJZGxAYx9hb/ELlsPU= +go.yaml.in/yaml/v2 v2.4.3 h1:6gvOSjQoTB3vt1l+CU+tSyi/HOjfOjRLJ4YwYZGwRO0= +go.yaml.in/yaml/v2 v2.4.3/go.mod h1:zSxWcmIDjOzPXpjlTTbAsKokqkDNAVtZO0WOMiT90s8= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -169,19 +205,21 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= -golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= +golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= -golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= golang.org/x/oauth2 v0.27.0 h1:da9Vo7/tDv5RH/7nZDz1eMGS/q1Vv1N/7FCrBhI9I3M= golang.org/x/oauth2 v0.27.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -189,22 +227,26 @@ golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= -golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= -golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= -golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= +golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc= +golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/term v0.37.0 h1:8EGAD0qCmHYZg6J17DvsMy9/wJ7/D/4pV/wfnld5lTU= +golang.org/x/term v0.37.0/go.mod h1:5pB4lxRNYYVZuTLmy8oR2BH8dflOR+IbTYFD8fi3254= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.23.0 h1:D71I7dUrlY+VX0gQShAThNGHFxZ13dGLBHQLVl1mJlY= -golang.org/x/text v0.23.0/go.mod h1:/BLNzu4aZCJ1+kcD0DNRotWKage4q2rGVAg4o22unh4= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= -golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= +golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +golang.org/x/tools/go/expect v0.1.1-deprecated h1:jpBZDwmgPhXsKZC6WhL20P4b/wmnpsEAGHaNy0n/rJM= +golang.org/x/tools/go/expect v0.1.1-deprecated/go.mod h1:eihoPOH+FgIqa3FpoTwguz/bVUSGBlGQU67vpBeOrBY= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated h1:1h2MnaIAIXISqTFKdENegdpAgUXz6NrPEsbIeWaBRvM= +golang.org/x/tools/go/packages/packagestest v0.1.1-deprecated/go.mod h1:RVAQXBGNv1ib0J382/DPCRS/BPnsGebyM1Gj5VSDpG8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -213,13 +255,13 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20250303144028-a0af3efb3deb/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I= google.golang.org/grpc v1.72.1 h1:HR03wO6eyZ7lknl75XlxABNVLLFc2PAb6mHlYh756mA= google.golang.org/grpc v1.72.1/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= -google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM= -google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/protobuf v1.36.8 h1:xHScyCOEuuwZEc6UtSOvPbAT4zRh0xcNRYekJwfqyMc= +google.golang.org/protobuf v1.36.8/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= @@ -228,24 +270,28 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= k8s.io/api v0.34.2 h1:fsSUNZhV+bnL6Aqrp6O7lMTy6o5x2C4XLjnh//8SLYY= k8s.io/api v0.34.2/go.mod h1:MMBPaWlED2a8w4RSeanD76f7opUoypY8TFYkSM+3XHw= -k8s.io/apimachinery v0.34.2 h1:zQ12Uk3eMHPxrsbUJgNF8bTauTVR2WgqJsTmwTE/NW4= -k8s.io/apimachinery v0.34.2/go.mod h1:/GwIlEcWuTX9zKIg2mbw0LRFIsXwrfoVxn+ef0X13lw= +k8s.io/apimachinery v0.35.0 h1:Z2L3IHvPVv/MJ7xRxHEtk6GoJElaAqDCCU0S6ncYok8= +k8s.io/apimachinery v0.35.0/go.mod h1:jQCgFZFR1F4Ik7hvr2g84RTJSZegBc8yHgFWKn//hns= k8s.io/client-go v0.34.2 h1:Co6XiknN+uUZqiddlfAjT68184/37PS4QAzYvQvDR8M= k8s.io/client-go v0.34.2/go.mod h1:2VYDl1XXJsdcAxw7BenFslRQX28Dxz91U9MWKjX97fE= +k8s.io/code-generator v0.35.0 h1:TvrtfKYZTm9oDF2z+veFKSCcgZE3Igv0svY+ehCmjHQ= +k8s.io/code-generator v0.35.0/go.mod h1:iS1gvVf3c/T71N5DOGYO+Gt3PdJ6B9LYSvIyQ4FHzgc= k8s.io/component-base v0.34.2 h1:HQRqK9x2sSAsd8+R4xxRirlTjowsg6fWCPwWYeSvogQ= k8s.io/component-base v0.34.2/go.mod h1:9xw2FHJavUHBFpiGkZoKuYZ5pdtLKe97DEByaA+hHbM= k8s.io/dynamic-resource-allocation v0.34.2 h1:SjlRGSWl6CZXoJwQNL+Y0wRfdH8PkJ4mHRNK6MMj0bY= k8s.io/dynamic-resource-allocation v0.34.2/go.mod h1:ul6I+gfrCmC+OCuVdN0/iykyB2sPrIqh2WyKQ3RQPCU= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b h1:gMplByicHV/TJBizHd9aVEsTYoJBnnUAT5MHlTkbjhQ= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b/go.mod h1:CgujABENc3KuTrcsdpGmrrASjtQsWCT7R99mEV4U/fM= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912 h1:Y3gxNAuB0OBLImH611+UDZcmKS3g6CthxToOb37KgwE= +k8s.io/kube-openapi v0.0.0-20250910181357-589584f1c912/go.mod h1:kdmbQkyfwUagLfXIad1y2TdrjPFWp2Q89B3qkRwf/pQ= k8s.io/kubelet v0.34.2 h1:Dl+1uh7xwJr70r+SHKyIpvu6XvzuoPu0uDIC4cqgJUs= k8s.io/kubelet v0.34.2/go.mod h1:RfwR03iuKeVV7Z1qD9XKH98c3tlPImJpQ3qHIW40htM= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4 h1:SjGebBtkBqHFOli+05xYbK8YF1Dzkbzn+gDM4X9T4Ck= +k8s.io/utils v0.0.0-20251002143259-bc988d571ff4/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= diff --git a/images/virtualization-dra/hack/boilerplate.go.txt b/images/virtualization-dra/hack/boilerplate.go.txt new file mode 100644 index 0000000000..b8911cc97b --- /dev/null +++ b/images/virtualization-dra/hack/boilerplate.go.txt @@ -0,0 +1,15 @@ +/* +Copyright Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ diff --git a/images/virtualization-dra/hack/update-codegen.sh b/images/virtualization-dra/hack/update-codegen.sh new file mode 100755 index 0000000000..def707d625 --- /dev/null +++ b/images/virtualization-dra/hack/update-codegen.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +# Copyright 2025 Flant JSC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd -P)" +API_ROOT="${SCRIPT_DIR}/../api" +CODEGEN_PKG="$(go env GOMODCACHE)/$(go list -f '{{.Path}}@{{.Version}}' -m k8s.io/code-generator)" +source "${CODEGEN_PKG}/kube_codegen.sh" + +function generate { + kube::codegen::gen_helpers \ + --boilerplate "${SCRIPT_DIR}/boilerplate.go.txt" \ + "${API_ROOT}" +} + +generate diff --git a/images/virtualization-dra/internal/common/consts.go b/images/virtualization-dra/internal/common/consts.go new file mode 100644 index 0000000000..e9a5bdc0eb --- /dev/null +++ b/images/virtualization-dra/internal/common/consts.go @@ -0,0 +1,29 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package common + +const ( + USBGatewayLabel = "virtualization.deckhouse.io/usb-gateway" +) + +const ( + USBGatewayAnnotation = "virtualization.deckhouse.io/usb-gateway" +) + +const ( + VirtualizationDraPluginName = "virtualization-dra" +) diff --git a/images/virtualization-dra/internal/featuregates/featuregates.go b/images/virtualization-dra/internal/featuregates/featuregates.go new file mode 100644 index 0000000000..91fac715a1 --- /dev/null +++ b/images/virtualization-dra/internal/featuregates/featuregates.go @@ -0,0 +1,76 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package featuregates + +import ( + "github.com/spf13/pflag" + "k8s.io/component-base/featuregate" +) + +const ( + USBGateway featuregate.Feature = "USBGateway" +) + +var featureSpecs = map[featuregate.Feature]featuregate.FeatureSpec{ + USBGateway: { + Default: false, + PreRelease: featuregate.Alpha, + }, +} + +var ( + instance *FeatureGate + addFlags func(fs *pflag.FlagSet) +) + +func init() { + gate, gateAddFlags, _, err := New() + if err != nil { + panic(err) + } + instance = gate + addFlags = gateAddFlags +} + +func AddFlags(fs *pflag.FlagSet) { + addFlags(fs) +} + +func Default() *FeatureGate { + return instance +} + +type ( + AddFlagsFunc func(fs *pflag.FlagSet) + SetFromMapFunc func(m map[string]bool) error +) + +func New() (*FeatureGate, AddFlagsFunc, SetFromMapFunc, error) { + gate := featuregate.NewFeatureGate() + if err := gate.Add(featureSpecs); err != nil { + return nil, nil, nil, err + } + return &FeatureGate{gate}, gate.AddFlag, gate.SetFromMap, nil +} + +type FeatureGate struct { + featuregate.FeatureGate +} + +func (f *FeatureGate) USBGatewayEnabled() bool { + return f.Enabled(USBGateway) +} diff --git a/images/virtualization-dra/internal/plugin/driver.go b/images/virtualization-dra/internal/plugin/driver.go index c8251e2020..a8605d29a9 100644 --- a/images/virtualization-dra/internal/plugin/driver.go +++ b/images/virtualization-dra/internal/plugin/driver.go @@ -30,9 +30,11 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/dynamic-resource-allocation/kubeletplugin" "k8s.io/dynamic-resource-allocation/resourceslice" + + "github.com/deckhouse/virtualization-dra/internal/common" ) -const DriverName = "virtualization-dra" +const DriverName = common.VirtualizationDraPluginName func NewDriver(nodeName string, kubeClient kubernetes.Interface, allocator Allocator, log *slog.Logger) *Driver { return &Driver{ @@ -169,8 +171,6 @@ func (d *Driver) startPublisher(ctx context.Context) { return case devices := <-ch: d.log.Info("Publishing devices", slog.Any("devices", devices)) - if len(devices) > 0 { - } resources := d.makeResources(devices) err := d.helper.PublishResources(ctx, resources) if err != nil { diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go new file mode 100644 index 0000000000..85ad158e98 --- /dev/null +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -0,0 +1,610 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourceclaim + +import ( + "context" + "fmt" + "log/slog" + "net" + "strings" + "time" + + corev1 "k8s.io/api/core/v1" + resourcev1beta1 "k8s.io/api/resource/v1beta1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/util/workqueue" + + vdraapi "github.com/deckhouse/virtualization-dra/api" + "github.com/deckhouse/virtualization-dra/internal/common" + "github.com/deckhouse/virtualization-dra/internal/usb-gateway/informer" + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +const controllerName = "resourceclaim-controller" +const finalizer = "virtualization.deckhouse.io/usb-gateway" + +var ( + keyFunc = cache.DeletionHandlingMetaNamespaceKeyFunc +) + +func controllerKeyFunc(namespace, name string) string { + return types.NamespacedName{ + Namespace: namespace, + Name: name, + }.String() +} + +type Controller struct { + nodeName string + podIP net.IP + usbipPort int + client kubernetes.Interface + resourceClaimIndexer cache.Indexer + resourceSliceIndexer cache.Indexer + nodeIndexer cache.Indexer + podIndexer cache.Indexer + usbIP usbip.Interface + queue workqueue.TypedRateLimitingInterface[string] + log *slog.Logger + hasSynced cache.InformerSynced +} + +func NewController(nodeName string, podIP net.IP, usbipPort int, client kubernetes.Interface, resourceClaimInformer, resourceSliceInformer, nodeInformer, podInformer cache.SharedIndexInformer, usbIP usbip.Interface) (*Controller, error) { + queue := workqueue.NewTypedRateLimitingQueueWithConfig( + workqueue.DefaultTypedControllerRateLimiter[string](), + workqueue.TypedRateLimitingQueueConfig[string]{Name: controllerName}, + ) + + c := &Controller{ + nodeName: nodeName, + podIP: podIP, + usbipPort: usbipPort, + client: client, + resourceClaimIndexer: resourceClaimInformer.GetIndexer(), + resourceSliceIndexer: resourceSliceInformer.GetIndexer(), + nodeIndexer: nodeInformer.GetIndexer(), + podIndexer: podInformer.GetIndexer(), + usbIP: usbIP, + queue: queue, + log: slog.With(slog.String("controller", controllerName)), + } + + _, err := resourceClaimInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + AddFunc: c.addResourceClaim, + UpdateFunc: c.updateResourceClaim, + DeleteFunc: c.deleteResourceClaim, + }) + if err != nil { + return nil, fmt.Errorf("unable to add event handler to resourceclaim informer: %w", err) + } + + c.hasSynced = func() bool { + return resourceClaimInformer.HasSynced() && nodeInformer.HasSynced() && podInformer.HasSynced() && resourceSliceInformer.HasSynced() + } + + return c, nil +} + +func (c *Controller) addResourceClaim(obj interface{}) { + if rc, ok := obj.(*resourcev1beta1.ResourceClaim); ok { + c.enqueueResourceClaim(rc) + } +} + +func (c *Controller) deleteResourceClaim(obj interface{}) { + if rc, ok := obj.(*resourcev1beta1.ResourceClaim); ok { + c.enqueueResourceClaim(rc) + } +} + +func (c *Controller) updateResourceClaim(_, newObj interface{}) { + newRC, ok := newObj.(*resourcev1beta1.ResourceClaim) + if !ok { + return + } + + if newRC.Status.Allocation != nil { + c.enqueueResourceClaim(newRC) + } +} + +func (c *Controller) enqueueResourceClaim(rc *resourcev1beta1.ResourceClaim) { + key, err := keyFunc(rc) + if err != nil { + utilruntime.HandleError(fmt.Errorf("couldn't get key for object %#v: %w", rc, err)) + return + } + c.queueAdd(key) +} + +func (c *Controller) queueAdd(key string) { + c.queue.Add(key) +} + +func (c *Controller) queueAfterAdd(key string, duration time.Duration) { + c.queue.AddAfter(key, duration) +} + +func (c *Controller) Run(ctx context.Context, workers int) error { + defer utilruntime.HandleCrash() + defer c.queue.ShutDown() + + c.log.Info("Starting controller") + defer c.log.Info("Shutting down controller") + + if !cache.WaitForCacheSync(ctx.Done(), c.hasSynced) { + return fmt.Errorf("failed to wait for caches to sync") + } + + c.log.Info("Starting workers controller") + for i := 0; i < workers; i++ { + go wait.UntilWithContext(ctx, c.worker, time.Second) + } + + <-ctx.Done() + return nil +} + +func (c *Controller) worker(ctx context.Context) { + workFunc := func(ctx context.Context) bool { + key, quit := c.queue.Get() + if quit { + return true + } + defer c.queue.Done(key) + + if err := c.sync(key); err != nil { + c.log.Error("re-enqueuing", slog.String("key", key), slog.Any("err", err)) + c.queue.AddRateLimited(key) + } else { + c.log.Info(fmt.Sprintf("processed ResourceClaim %v", key)) + c.queue.Forget(key) + } + return false + } + for { + quit := workFunc(ctx) + + if quit { + return + } + } +} + +func (c *Controller) sync(key string) error { + log := c.log.With("key", key) + log.Info("syncing resource claim") + + rc, err := c.getResourceClaim(key) + if err != nil { + return err + } + if rc == nil { + return nil + } + if !rc.GetDeletionTimestamp().IsZero() { + return c.handleDelete(rc) + } + + pod, err := c.getReservedFor(rc) + if err != nil { + return err + } + if pod == nil { + c.log.Info("no reserved pod found for resource claim, re-enqueue after 10s") + c.queueAfterAdd(key, time.Second*10) + return nil + } + + myAllocationDevices, otherAllocationDevices, err := c.getAllocationDevices(rc) + if err != nil { + return err + } + + onMyNode := c.podOnMyNode(pod) + shouldShare := !onMyNode && len(myAllocationDevices) > 0 + shouldAttach := onMyNode && len(otherAllocationDevices) > 0 + + switch { + case shouldShare: + log.Info("sharing usb to other node") + if err = c.handleServer(rc, myAllocationDevices); err != nil { + return fmt.Errorf("failed to handle server: %w", err) + } + case shouldAttach: + log.Info("attaching usb to my node") + if err = c.handleClient(rc, otherAllocationDevices); err != nil { + return fmt.Errorf("failed to handle client: %w", err) + } + + } + + return nil +} + +// TODO: handle detach too +func (c *Controller) handleDelete(rc *resourcev1beta1.ResourceClaim) error { + if c.allUnBound(rc) { + return c.removeFinalizer(rc) + } + + myAllocationDevices, _, err := c.getAllocationDevices(rc) + if err != nil { + return err + } + + myAllocationDevicesByName := make(map[string]resourcev1beta1.Device) + for _, device := range myAllocationDevices { + myAllocationDevicesByName[device.Name] = device + } + + shouldUpdate := false + + for i := range rc.Status.Devices { + allocatedDeviceStatus := &rc.Status.Devices[i] + + usbGatewayStatus := vdraapi.FromData(allocatedDeviceStatus.Data) + if usbGatewayStatus == nil { + continue + } + if !usbGatewayStatus.Bound { + continue + } + + device, ok := myAllocationDevicesByName[allocatedDeviceStatus.Device] + if !ok { + continue + } + + busID := "" + if attr, ok := device.Basic.Attributes["busID"]; ok && attr.StringValue != nil { + busID = *attr.StringValue + } else { + continue + } + + // TODO: device can be added to other resource claims. Not supported yet. + c.log.Info("unbinding usb") + if err = c.usbIP.Unbind(busID); err != nil { + return fmt.Errorf("failed to unbind usb: %w", err) + } + + usbGatewayStatus.Bound = false + allocatedDeviceStatus.Data.Object = usbGatewayStatus + shouldUpdate = true + } + + if shouldUpdate { + _, err = c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).UpdateStatus(context.Background(), rc, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update resource claim status: %w", err) + } + } + + return nil +} + +func (c *Controller) allUnBound(rc *resourcev1beta1.ResourceClaim) bool { + for _, deviceStatus := range rc.Status.Devices { + usbGatewayStatus := vdraapi.FromData(deviceStatus.Data) + if usbGatewayStatus == nil { + continue + } + if usbGatewayStatus.Bound { + return false + } + } + + return true +} + +func (c *Controller) addFinalizer(rc *resourcev1beta1.ResourceClaim) error { + var newFinalizers []string + for _, fin := range rc.GetFinalizers() { + if fin == finalizer { + return nil + } + newFinalizers = append(newFinalizers, fin) + } + newFinalizers = append(newFinalizers, finalizer) + rc.SetFinalizers(newFinalizers) + _, err := c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).Update(context.Background(), rc, metav1.UpdateOptions{}) + return err +} + +func (c *Controller) removeFinalizer(rc *resourcev1beta1.ResourceClaim) error { + var newFinalizers []string + for _, fin := range rc.GetFinalizers() { + if fin == finalizer { + continue + } + newFinalizers = append(newFinalizers, fin) + } + if len(newFinalizers) == len(rc.GetFinalizers()) { + return nil + } + + rc.SetFinalizers(newFinalizers) + _, err := c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).Update(context.Background(), rc, metav1.UpdateOptions{}) + return err +} + +func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocationDevices []resourcev1beta1.Device) error { + indexAllocDevice := make(map[string]int) + for i, allocDeviceStatus := range rc.Status.Devices { + indexAllocDevice[allocDeviceStatus.Device] = i + } + + shouldUpdate := false + + for _, device := range myAllocationDevices { + if device.Basic == nil { + continue + } + + index, ok := indexAllocDevice[device.Name] + if !ok { + continue + } + + allocDeviceStatus := &rc.Status.Devices[index] + + usbGatewayStatus := vdraapi.FromData(allocDeviceStatus.Data) + + targetIPAlreadySet := usbGatewayStatus != nil && usbGatewayStatus.TargetIP != "" + if targetIPAlreadySet { + continue + } + usbGatewayStatus = &vdraapi.USBGatewayStatus{} + + busID := "" + if attr, ok := device.Basic.Attributes["busID"]; ok && attr.StringValue != nil { + busID = *attr.StringValue + } else { + continue + } + + bound, err := c.usbIP.IsBound(busID) + if err != nil { + return fmt.Errorf("failed to check if usb is bound: %w", err) + } + + if !bound { + if err = c.usbIP.Bind(busID); err != nil { + return fmt.Errorf("failed to bind usb: %w", err) + } + } + + usbGatewayStatus.TargetIP = c.podIP.String() + usbGatewayStatus.TargetPort = c.usbipPort + usbGatewayStatus.Bound = true + + if allocDeviceStatus.Data == nil { + allocDeviceStatus.Data = &runtime.RawExtension{} + } + allocDeviceStatus.Data.Object = usbGatewayStatus + shouldUpdate = true + } + + if shouldUpdate { + err := c.addFinalizer(rc) + if err != nil { + return err + } + _, err = c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).UpdateStatus(context.Background(), rc, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update resource claim status: %w", err) + } + } + + return nil +} + +func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAllocationDevices []resourcev1beta1.Device) error { + indexAllocDevice := make(map[string]int) + for i, allocDeviceStatus := range rc.Status.Devices { + indexAllocDevice[allocDeviceStatus.Device] = i + } + + shouldUpdate := false + + for _, device := range otherAllocationDevices { + if device.Basic == nil { + continue + } + index, ok := indexAllocDevice[device.Name] + if !ok { + continue + } + allocDeviceStatus := &rc.Status.Devices[index] + usbGatewayStatus := vdraapi.FromData(allocDeviceStatus.Data) + if usbGatewayStatus == nil { + continue + } + if !usbGatewayStatus.Bound { + continue + } + if usbGatewayStatus.Attached { + continue + } + busID := "" + if attr, ok := device.Basic.Attributes["busID"]; ok && attr.StringValue != nil { + busID = *attr.StringValue + } else { + continue + } + + err := c.usbIP.Attach(usbGatewayStatus.TargetIP, busID, usbGatewayStatus.TargetPort) + if err != nil { + return fmt.Errorf("failed to attach usb: %w", err) + } + + usbGatewayStatus.Attached = true + if allocDeviceStatus.Data == nil { + allocDeviceStatus.Data = &runtime.RawExtension{} + } + allocDeviceStatus.Data.Object = usbGatewayStatus + shouldUpdate = true + } + + if shouldUpdate { + _, err := c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).UpdateStatus(context.Background(), rc, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("failed to update resource claim status: %w", err) + } + } + + return nil +} + +func (c *Controller) getResourceClaim(key string) (*resourcev1beta1.ResourceClaim, error) { + obj, exists, err := c.resourceClaimIndexer.GetByKey(key) + if err != nil && !k8serrors.IsNotFound(err) { + return nil, fmt.Errorf("failed to get resourceclaim: %w", err) + } + if !exists { + return nil, nil + } + + rc, ok := obj.(*resourcev1beta1.ResourceClaim) + if !ok { + return nil, fmt.Errorf("unexpected type of resourceclaim: %T", obj) + } + + return rc.DeepCopy(), nil +} + +func (c *Controller) getPod(name, namespace string) (*corev1.Pod, error) { + obj, exists, err := c.podIndexer.GetByKey(controllerKeyFunc(namespace, name)) + if err != nil && !k8serrors.IsNotFound(err) { + return nil, err + } + if !exists { + return nil, nil + } + + pod, ok := obj.(*corev1.Pod) + if !ok { + return nil, fmt.Errorf("unexpected type of pod: %T", obj) + } + + return pod.DeepCopy(), nil +} + +func (c *Controller) getVirtualizationDraResourceSlices() ([]resourcev1beta1.ResourceSlice, error) { + slicesObj, err := c.resourceSliceIndexer.ByIndex(informer.DriverIndex, common.VirtualizationDraPluginName) + if err != nil { + return nil, err + } + var slices []resourcev1beta1.ResourceSlice + for _, obj := range slicesObj { + slice, ok := obj.(resourcev1beta1.ResourceSlice) + if !ok { + return nil, fmt.Errorf("unexpected type of resource slice: %T", obj) + } + slices = append(slices, *slice.DeepCopy()) + } + return slices, nil +} + +// TODO: refactor me. only one pod supports now +func (c *Controller) getReservedFor(rc *resourcev1beta1.ResourceClaim) (*corev1.Pod, error) { + for _, rFor := range rc.Status.ReservedFor { + if rFor.Resource == "pods" { + pod, err := c.getPod(rFor.Name, rc.Namespace) + if err != nil { + return nil, err + } + if pod == nil || pod.GetUID() != rFor.UID { + return nil, nil + } + return pod, nil + } + } + return nil, nil +} + +func (c *Controller) podOnMyNode(pod *corev1.Pod) bool { + return pod.Spec.NodeName == c.nodeName +} + +func (c *Controller) getAllocationDevices(rc *resourcev1beta1.ResourceClaim) ([]resourcev1beta1.Device, []resourcev1beta1.Device, error) { + if rc.Status.Allocation == nil { + return nil, nil, fmt.Errorf("resource claim %s/%s has no allocation", rc.Namespace, rc.Name) + } + + virtualizationDraSlices, err := c.getVirtualizationDraResourceSlices() + if err != nil { + return nil, nil, err + } + + byPoolSlices := make(map[string][]resourcev1beta1.ResourceSlice) + for _, slice := range virtualizationDraSlices { + byPoolSlices[slice.Spec.Pool.Name] = append(byPoolSlices[slice.Spec.Pool.Name], slice) + } + + allocResultsByPool := make(map[string]map[string]resourcev1beta1.DeviceRequestAllocationResult) + + for _, status := range rc.Status.Allocation.Devices.Results { + if status.Driver != common.VirtualizationDraPluginName { + continue + } + // now, driver virtualization-dra supports only usb, but we can add more devices later + // so we need to check if the device is usb + if strings.HasPrefix(status.Device, "usb") { + continue + } + + allocResultsByPool[status.Pool][status.Device] = status + } + + var myDevices []resourcev1beta1.Device + var otherDevices []resourcev1beta1.Device + + for pool, allocResultsByDevice := range allocResultsByPool { + slices, ok := byPoolSlices[pool] + if !ok { + return nil, nil, fmt.Errorf("no resource slices found for pool %s", pool) + } + + for _, slice := range slices { + for _, device := range slice.Spec.Devices { + allocResult, ok := allocResultsByDevice[device.Name] + if !ok { + continue + } + + // virtualization-dra creates slices with pool name by node name + if allocResult.Pool == c.nodeName { + myDevices = append(myDevices, device) + } else { + otherDevices = append(otherDevices, device) + } + } + } + } + + return myDevices, otherDevices, nil +} diff --git a/images/virtualization-dra/internal/usb-gateway/informer/informer.go b/images/virtualization-dra/internal/usb-gateway/informer/informer.go new file mode 100644 index 0000000000..3d32df4fb6 --- /dev/null +++ b/images/virtualization-dra/internal/usb-gateway/informer/informer.go @@ -0,0 +1,150 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package informer + +import ( + "log/slog" + "math/rand/v2" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + resourcev1beta1 "k8s.io/api/resource/v1beta1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/cache" +) + +const ( + NodeIndex = "node" + PoolIndex = "pool" + DriverIndex = "driver" +) + +func NewFactory(clientSet *kubernetes.Clientset, resync *time.Duration) *Factory { + var defaultResync time.Duration + if resync != nil { + defaultResync = *resync + } else { + defaultResync = resyncPeriod(12 * time.Hour) + } + + return &Factory{ + clientSet: clientSet, + defaultResync: defaultResync, + informers: make(map[string]cache.SharedIndexInformer), + } +} + +type Factory struct { + clientSet *kubernetes.Clientset + defaultResync time.Duration + + informers map[string]cache.SharedIndexInformer + startedInformers map[string]struct{} + mu sync.Mutex +} + +func (f *Factory) Start(stopCh <-chan struct{}) { + f.mu.Lock() + defer f.mu.Unlock() + + for name, informer := range f.informers { + if _, found := f.startedInformers[name]; found { + // skip informers that have already started. + slog.Info("SKIPPING informer", slog.String("name", name)) + continue + } + slog.Info("STARTING informer", slog.String("name", name)) + go informer.Run(stopCh) + f.startedInformers[name] = struct{}{} + } +} + +func (f *Factory) WaitForCacheSync(stopCh <-chan struct{}) { + var syncs []cache.InformerSynced + + f.mu.Lock() + for name, informer := range f.informers { + slog.Info("Waiting for cache sync of informer", slog.String("name", name)) + syncs = append(syncs, informer.HasSynced) + } + f.mu.Unlock() + + cache.WaitForCacheSync(stopCh, syncs...) +} + +func (f *Factory) ResourceClaim() cache.SharedIndexInformer { + return f.getInformer("resourceClaimInformer", func() cache.SharedIndexInformer { + lw := cache.NewListWatchFromClient(f.clientSet.ResourceV1beta1().RESTClient(), "resourceclaims", corev1.NamespaceAll, fields.Everything()) + return cache.NewSharedIndexInformer(lw, &resourcev1beta1.ResourceClaim{}, f.defaultResync, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + }) +} + +func (f *Factory) ResourceSlice() cache.SharedIndexInformer { + return f.getInformer("resourceSliceInformer", func() cache.SharedIndexInformer { + lw := cache.NewListWatchFromClient(f.clientSet.ResourceV1beta1().RESTClient(), "resourceslices", corev1.NamespaceAll, fields.Everything()) + return cache.NewSharedIndexInformer(lw, &resourcev1beta1.ResourceSlice{}, f.defaultResync, cache.Indexers{ + PoolIndex: func(obj interface{}) ([]string, error) { + return []string{obj.(*resourcev1beta1.ResourceSlice).Spec.Pool.Name}, nil + }, + DriverIndex: func(obj interface{}) ([]string, error) { + return []string{obj.(*resourcev1beta1.ResourceSlice).Spec.Driver}, nil + }, + }) + }) +} + +func (f *Factory) Nodes() cache.SharedIndexInformer { + return f.getInformer("nodesInformer", func() cache.SharedIndexInformer { + lw := cache.NewListWatchFromClient(f.clientSet.CoreV1().RESTClient(), "nodes", corev1.NamespaceAll, fields.Everything()) + return cache.NewSharedIndexInformer(lw, &corev1.Node{}, f.defaultResync, cache.Indexers{}) + }) +} + +func (f *Factory) Pods() cache.SharedIndexInformer { + return f.getInformer("podsInformer", func() cache.SharedIndexInformer { + lw := cache.NewListWatchFromClient(f.clientSet.CoreV1().RESTClient(), "pods", corev1.NamespaceAll, fields.Everything()) + return cache.NewSharedIndexInformer(lw, &corev1.Pod{}, f.defaultResync, cache.Indexers{ + cache.NamespaceIndex: cache.MetaNamespaceIndexFunc, + NodeIndex: func(obj interface{}) ([]string, error) { + return []string{obj.(*corev1.Pod).Spec.NodeName}, nil + }, + }) + }) +} + +func (f *Factory) getInformer(key string, newFunc func() cache.SharedIndexInformer) cache.SharedIndexInformer { + f.mu.Lock() + defer f.mu.Unlock() + + informer, ok := f.informers[key] + if ok { + return informer + } + + informer = newFunc() + f.informers[key] = informer + + return informer +} + +// resyncPeriod computes the time interval a shared informer waits before resyncing with the api server +func resyncPeriod(minResyncPeriod time.Duration) time.Duration { + factor := rand.Float64() + 1 + return time.Duration(float64(minResyncPeriod.Nanoseconds()) * factor) +} diff --git a/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go b/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go new file mode 100644 index 0000000000..2e36e3fdfc --- /dev/null +++ b/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go @@ -0,0 +1,83 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package labeler + +import ( + "context" + "encoding/json" + "fmt" + "maps" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/dynamic" +) + +type Labeler interface { + Label(ctx context.Context, name, namespace string, labels map[string]string) error +} + +type genericLabeler struct { + client dynamic.Interface + gvr schema.GroupVersionResource +} + +func NewGenericLabeler(client dynamic.Interface, gvr schema.GroupVersionResource) Labeler { + return &genericLabeler{ + client: client, + gvr: gvr, + } +} + +func (l *genericLabeler) Label(ctx context.Context, name, namespace string, newLabels map[string]string) error { + obj, err := l.client.Resource(l.gvr).Namespace(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + + labels := obj.GetLabels() + maps.Copy(labels, newLabels) + + value, err := json.Marshal(labels) + if err != nil { + return err + } + + patch := []byte(fmt.Sprintf("[{'op': 'replace', 'path': '/metadata/labels', 'value': %s}]", value)) + + _, err = l.client.Resource(l.gvr).Namespace(namespace).Patch(ctx, name, types.JSONPatchType, patch, metav1.PatchOptions{}) + return err +} + +type NodeLabeler struct { + generic Labeler +} + +func NewNodeLabeler(client dynamic.Interface) NodeLabeler { + return NodeLabeler{ + generic: NewGenericLabeler(client, schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "nodes", + }), + } +} + +func (l NodeLabeler) Label(ctx context.Context, name, namespace string, newLabels map[string]string) error { + return l.generic.Label(ctx, name, namespace, newLabels) +} diff --git a/images/virtualization-dra/internal/usb-gateway/prepare/labels.go b/images/virtualization-dra/internal/usb-gateway/prepare/labels.go new file mode 100644 index 0000000000..904c288f85 --- /dev/null +++ b/images/virtualization-dra/internal/usb-gateway/prepare/labels.go @@ -0,0 +1,32 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package prepare + +import ( + "context" + + "k8s.io/client-go/dynamic" + + "github.com/deckhouse/virtualization-dra/internal/common" + "github.com/deckhouse/virtualization-dra/internal/usb-gateway/labeler" +) + +func MarkNodeForUSBGateway(ctx context.Context, nodeName string, dynamicClient dynamic.Interface) error { + return labeler.NewNodeLabeler(dynamicClient).Label(ctx, nodeName, "", map[string]string{ + common.USBGatewayLabel: "true", + }) +} diff --git a/images/virtualization-dra/internal/usb-gateway/tlsproxy/proxy.go b/images/virtualization-dra/internal/usb-gateway/tlsproxy/proxy.go new file mode 100644 index 0000000000..c502bde742 --- /dev/null +++ b/images/virtualization-dra/internal/usb-gateway/tlsproxy/proxy.go @@ -0,0 +1,97 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tlsproxy + +import ( + "context" + "crypto/tls" + "io" + "log/slog" + "net" + "strconv" +) + +type TLSProxy struct { + tlsConfig *tls.Config + port int +} + +func NewTLSProxy(tlsConfig *tls.Config, port int) *TLSProxy { + return &TLSProxy{ + tlsConfig: tlsConfig, + port: port, + } +} + +func (p *TLSProxy) Start(ctx context.Context, plainConn net.Conn) error { + listener, err := tls.Listen("tcp", net.JoinHostPort("", strconv.Itoa(p.port)), p.tlsConfig) + if err != nil { + return err + } + + go func() { + defer listener.Close() + defer plainConn.Close() + + tlsConn, err := listener.Accept() + if err != nil { + select { + case <-ctx.Done(): + return + default: + slog.Error("accept error", slog.Any("error", err)) + return + } + } + defer tlsConn.Close() + + done := make(chan struct{}, 2) + + // TLS -> plain + go func() { + _, err = io.Copy(plainConn, tlsConn) + if err != nil { + slog.Error("copy error from TLS to plain", + slog.Any("error", err), + slog.String("tlsRemoteAddr", tlsConn.RemoteAddr().String()), + slog.String("proxyRemoteAddr", plainConn.RemoteAddr().String()), + ) + } + done <- struct{}{} + }() + + // plain -> TLS + go func() { + _, err = io.Copy(tlsConn, plainConn) + if err != nil { + slog.Error("copy error from plain to TLS", + slog.Any("error", err), + slog.String("tlsRemoteAddr", tlsConn.RemoteAddr().String()), + slog.String("proxyRemoteAddr", plainConn.RemoteAddr().String()), + ) + } + done <- struct{}{} + }() + + select { + case <-ctx.Done(): + case <-done: + } + }() + + return nil +} diff --git a/images/virtualization-dra/internal/usb/convert.go b/images/virtualization-dra/internal/usb/convert.go index 95f4cb65aa..8cbeb53f9a 100644 --- a/images/virtualization-dra/internal/usb/convert.go +++ b/images/virtualization-dra/internal/usb/convert.go @@ -17,16 +17,27 @@ limitations under the License. package usb import ( + corev1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" "k8s.io/utils/ptr" + + "github.com/deckhouse/virtualization-dra/internal/common" + "github.com/deckhouse/virtualization-dra/internal/featuregates" ) -func convertToAPIDevice(usbDevice Device) *resourceapi.Device { - return &resourceapi.Device{ - Name: usbDevice.GetName(), +func convertToAPIDevice(usbDevice Device, nodeName string) *resourceapi.Device { + name := usbDevice.GetName(nodeName) + device := &resourceapi.Device{ + Name: name, Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ "name": { - StringValue: ptr.To(usbDevice.Name), + StringValue: ptr.To(name), + }, + "path": { + StringValue: ptr.To(usbDevice.Path), + }, + "busID": { + StringValue: ptr.To(usbDevice.BusID), }, "manufacturer": { StringValue: ptr.To(usbDevice.Manufacturer), @@ -69,4 +80,26 @@ func convertToAPIDevice(usbDevice Device) *resourceapi.Device { }, }, } + + if featuregates.Default().USBGatewayEnabled() { + device.NodeSelector = &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: common.USBGatewayLabel, + Operator: corev1.NodeSelectorOpExists, + Values: []string{"true"}, + }, + }, + }, + }, + } + // TODO: add support for multiple allocations + // device.AllowMultipleAllocations = ptr.To(true) + } else { + device.NodeName = ptr.To(nodeName) + } + + return device } diff --git a/images/virtualization-dra/internal/usb/device.go b/images/virtualization-dra/internal/usb/device.go index 534a9d3465..0a53c8d303 100644 --- a/images/virtualization-dra/internal/usb/device.go +++ b/images/virtualization-dra/internal/usb/device.go @@ -17,15 +17,12 @@ limitations under the License. package usb import ( - "bufio" "fmt" - "log/slog" - "os" - "path/filepath" "strconv" "strings" "github.com/deckhouse/virtualization-dra/pkg/set" + "github.com/deckhouse/virtualization-dra/pkg/usb" ) type DeviceSet = set.Set[Device] @@ -35,7 +32,8 @@ func NewDeviceSet() *DeviceSet { } type Device struct { - Name string + Path string + BusID string Manufacturer string Product string VendorID int4x @@ -49,13 +47,16 @@ type Device struct { DevicePath string } -func (d *Device) GetName() string { - // usb---- +func (d *Device) GetName(nodeName string) string { + // usb----- // usb-003-005-e39-f100 - return fmt.Sprintf("usb-%s-%s-%s-%s", d.Bus.String(), d.DeviceNumber.String(), d.VendorID.String(), d.ProductID.String()) + return fmt.Sprintf("usb-%s-%s-%s-%s-%s", d.Bus.String(), d.DeviceNumber.String(), d.VendorID.String(), d.ProductID.String(), nodeName) } func (d *Device) Validate() error { + if d.BusID == "" { + return fmt.Errorf("BusID is required") + } if d.VendorID == 0 { return fmt.Errorf("VendorID is required") } @@ -80,153 +81,22 @@ func (d *Device) Validate() error { return nil } -func LoadDevice(path string) (device Device, err error) { - if err = parseSysUeventFile(path, &device); err != nil { - return - } - if err = parseSerial(path, &device); err != nil { - return - } - if err = parseManufacturer(path, &device); err != nil { - return - } - if err = parseProduct(path, &device); err != nil { - return - } - return -} - -func parseSysUeventFile(path string, device *Device) error { - // Example uevent file: - // MAJOR=189 - // MINOR=257 - // DEVNAME=bus/usb/003/002 - // DEVTYPE=usb_device - // DRIVER=usb - // PRODUCT=e39/f100/35d - // TYPE=0/0/0 - // BUSNUM=003 - // DEVNUM=002 - file, err := os.Open(filepath.Join(path, "uevent")) - if err != nil { - return fmt.Errorf("unable to open the file %s: %w", path, err) - } - defer file.Close() - - scanner := bufio.NewScanner(file) - for scanner.Scan() { - line := scanner.Text() - values := strings.Split(line, "=") - if len(values) != 2 { - slog.Info("Skipping %s due not being key=value", slog.String("line", line)) - continue - } - switch values[0] { - case "MAJOR": - val, err := strconv.ParseInt(values[1], 10, 32) - if err != nil { - slog.Error("Failed to parse MAJOR", slog.String("value", values[1]), slog.Any("err", err)) - return nil - } - device.Major = int(val) - case "MINOR": - val, err := strconv.ParseInt(values[1], 10, 32) - if err != nil { - slog.Error("Failed to parse MINOR", slog.String("value", values[1]), slog.Any("err", err)) - return nil - } - device.Minor = int(val) - case "BUSNUM": - val, err := strconv.ParseInt(values[1], 10, 32) - if err != nil { - slog.Error("Failed to parse BUSNUM", slog.String("value", values[1]), slog.Any("err", err)) - return nil - } - device.Bus = int3d(val) - case "DEVNUM": - val, err := strconv.ParseInt(values[1], 10, 32) - if err != nil { - slog.Error("Failed to parse DEVNUM", slog.String("value", values[1]), slog.Any("err", err)) - return nil - } - device.DeviceNumber = int3d(val) - case "PRODUCT": - products := strings.Split(values[1], "/") - if len(products) != 3 { - slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) - return nil - } - - val, err := strconv.ParseInt(products[0], 16, 32) - if err != nil { - slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) - return nil - } - device.VendorID = int4x(val) - - val, err = strconv.ParseInt(products[1], 16, 32) - if err != nil { - slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) - return nil - } - device.ProductID = int4x(val) - - val, err = strconv.ParseInt(products[2], 16, 32) - if err != nil { - slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) - return nil - } - device.BCD = int4x(val) - case "DEVNAME": - device.DevicePath = filepath.Join("/dev", values[1]) - default: - slog.Info("Skipping unhandled line", slog.String("line", line)) - } - } - return nil -} - -func parseSerial(path string, device *Device) error { - b, err := os.ReadFile(filepath.Join(path, "serial")) - if err != nil { - return err - } - lines := strings.Split(string(b), "\n") - if len(lines) >= 1 { - device.Serial = strings.TrimSpace(lines[0]) - } else { - device.Serial = "unknown" +func toDevice(device *usb.Device) Device { + return Device{ + Path: device.Path, + BusID: device.BusID, + Manufacturer: device.Manufacturer, + Product: device.Product, + VendorID: int4x(device.VendorID), + ProductID: int4x(device.ProductID), + BCD: int4x(device.BCD), + Bus: int3d(device.Bus), + DeviceNumber: int3d(device.DeviceNumber), + Major: int(device.Major), + Minor: int(device.Minor), + Serial: device.Serial, + DevicePath: device.DevicePath, } - - return nil -} - -func parseManufacturer(path string, device *Device) error { - b, err := os.ReadFile(filepath.Join(path, "manufacturer")) - if err != nil { - return err - } - lines := strings.Split(string(b), "\n") - if len(lines) >= 1 { - device.Manufacturer = strings.TrimSpace(lines[0]) - } else { - device.Manufacturer = "unknown" - } - return nil -} - -func parseProduct(path string, device *Device) error { - b, err := os.ReadFile(filepath.Join(path, "product")) - if err != nil { - return err - } - lines := strings.Split(string(b), "\n") - if len(lines) >= 1 { - device.Product = strings.TrimSpace(lines[0]) - } else { - device.Product = "unknown" - } - return nil } type int4x int diff --git a/images/virtualization-dra/internal/usb/discovery.go b/images/virtualization-dra/internal/usb/discovery.go index 304ea83d88..28b72771d3 100644 --- a/images/virtualization-dra/internal/usb/discovery.go +++ b/images/virtualization-dra/internal/usb/discovery.go @@ -17,46 +17,20 @@ limitations under the License. package usb import ( - "fmt" - "log/slog" - "os" - "path/filepath" - "strings" + "github.com/deckhouse/virtualization-dra/pkg/usb" ) -const PathToUSBDevices = "/sys/bus/usb/devices" +const PathToUSBDevices = usb.PathToUSBDevices func discoverPluggedUSBDevices(pathToUSBDevices string) (*DeviceSet, error) { - usbDeviceSet := NewDeviceSet() - err := filepath.Walk(pathToUSBDevices, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - // Ignore named usb controllers - if strings.HasPrefix(info.Name(), "usb") { - return nil - } - // We are interested in actual USB devices information that - // contains idVendor and idProduct. We can skip all others. - if _, err := os.Stat(filepath.Join(path, "idVendor")); err != nil { - return nil - } - - // Get device information - device, err := LoadDevice(path) - if err = device.Validate(); err != nil { - slog.Error("failed to validate device, skip...", slog.Any("device", device), slog.String("error", err.Error())) - return nil - } - if err != nil { - return err - } - usbDeviceSet.Add(device) - return nil - }) - + devices, err := usb.DiscoverPluggedUSBDevices(pathToUSBDevices) if err != nil { - return nil, fmt.Errorf("failed when walking usb devices tree: %w", err) + return nil, err } + usbDeviceSet := NewDeviceSet() + for _, device := range devices { + usbDeviceSet.Add(toDevice(device)) + } + return usbDeviceSet, nil } diff --git a/images/virtualization-dra/internal/usb/store.go b/images/virtualization-dra/internal/usb/store.go index 5ae5d2b50d..aa658170d5 100644 --- a/images/virtualization-dra/internal/usb/store.go +++ b/images/virtualization-dra/internal/usb/store.go @@ -20,18 +20,24 @@ import ( "context" "fmt" "log/slog" + "strconv" "strings" "sync" "time" "github.com/containerd/nri/pkg/api" resourceapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1beta1" "k8s.io/utils/ptr" cdiapi "tags.cncf.io/container-device-interface/pkg/cdi" cdispec "tags.cncf.io/container-device-interface/specs-go" + vdraapi "github.com/deckhouse/virtualization-dra/api" + "github.com/deckhouse/virtualization-dra/internal/common" + "github.com/deckhouse/virtualization-dra/internal/featuregates" + "github.com/deckhouse/virtualization-dra/internal/cdi" "github.com/deckhouse/virtualization-dra/pkg/set" ) @@ -103,7 +109,7 @@ func (s *AllocationStore) sync() error { allocatableDevices := make([]resourceapi.Device, discoverPluggedUSBDevices.Len()) for i, usbDevice := range discoverPluggedUSBDevices.Slice() { - allocatableDevices[i] = *convertToAPIDevice(usbDevice) + allocatableDevices[i] = *convertToAPIDevice(usbDevice, s.nodeName) } allocatableDevicesByName := make(map[string]resourceapi.Device, len(allocatableDevices)) @@ -183,7 +189,12 @@ func (s *AllocationStore) Prepare(_ context.Context, claim *resourceapi.Resource return nil, fmt.Errorf("device %v is already allocated", result.Device) } - edits, err := s.makeContainerEdits(claimUID, &usbDevice) + containerEditsOptions, err := newContainerEditsOptions(&usbDevice, claim) + if err != nil { + return nil, err + } + + edits, err := s.makeContainerEdits(claimUID, containerEditsOptions) if err != nil { return nil, err } @@ -213,75 +224,114 @@ func (s *AllocationStore) Prepare(_ context.Context, claim *resourceapi.Resource return devices, nil } -// TODO: refactor me -func (s *AllocationStore) makeContainerEdits(claimUID string, device *resourceapi.Device) (*cdiapi.ContainerEdits, error) { - var ( - devicePath string - deviceNum string - bus string - major int64 - minor int64 - ) - - if attr, ok := device.Attributes["devicePath"]; ok { - if val := attr.StringValue; val != nil { - devicePath = *val - } else { - return nil, fmt.Errorf("devicePath attribute is not exist") - } +func newContainerEditsOptions(device *resourceapi.Device, claim *resourceapi.ResourceClaim) (containerEditsOptions, error) { + opts := containerEditsOptions{ + Name: device.Name, } - if attr, ok := device.Attributes["deviceNumber"]; ok { - if val := attr.StringValue; val != nil { - deviceNum = *val - } else { - return nil, fmt.Errorf("deviceNum attribute is not exist") + if featuregates.Default().USBGatewayEnabled() && isUSBGateway(claim) { + var data *runtime.RawExtension + for _, deviceStatus := range claim.Status.Devices { + if deviceStatus.Device == device.Name { + data = deviceStatus.Data + break + } + } + if data == nil { + return opts, fmt.Errorf("device status data is not found") } - } - if attr, ok := device.Attributes["bus"]; ok { - if val := attr.StringValue; val != nil { - bus = *val - } else { - return nil, fmt.Errorf("bus attribute is not exist") + usbGatewayStatus, ok := data.Object.(*vdraapi.USBGatewayStatus) + if !ok { + return opts, fmt.Errorf("device status data is not a USBGatewayStatus") + } + if usbGatewayStatus == nil { + return opts, fmt.Errorf("device status data Object is nil") + } + + opts.Bus = strconv.Itoa(usbGatewayStatus.BusNum) + opts.DeviceNum = strconv.Itoa(usbGatewayStatus.DeviceNum) + opts.DevicePath = usbGatewayStatus.DevicePath + + } else { + if attr, ok := device.Attributes["devicePath"]; ok { + if val := attr.StringValue; val != nil { + opts.DevicePath = *val + } else { + return opts, fmt.Errorf("devicePath attribute is not exist") + } + } + + if attr, ok := device.Attributes["deviceNumber"]; ok { + if val := attr.StringValue; val != nil { + opts.DeviceNum = *val + } else { + return opts, fmt.Errorf("deviceNum attribute is not exist") + } + } + + if attr, ok := device.Attributes["bus"]; ok { + if val := attr.StringValue; val != nil { + opts.Bus = *val + } else { + return opts, fmt.Errorf("bus attribute is not exist") + } } } if attr, ok := device.Attributes["major"]; ok { if val := attr.IntValue; val != nil { - major = *val + opts.Major = *val } else { - return nil, fmt.Errorf("major attribute is not exist") + return opts, fmt.Errorf("major attribute is not exist") } } if attr, ok := device.Attributes["minor"]; ok { if val := attr.IntValue; val != nil { - minor = *val + opts.Minor = *val } else { - return nil, fmt.Errorf("minor attribute is not exist") + return opts, fmt.Errorf("minor attribute is not exist") } } + return opts, nil +} + +func isUSBGateway(claim *resourceapi.ResourceClaim) bool { + return claim.Annotations[common.USBGatewayAnnotation] == "true" +} + +type containerEditsOptions struct { + Name string + DevicePath string + DeviceNum string + Bus string + Major int64 + Minor int64 +} + +// TODO: refactor me +func (s *AllocationStore) makeContainerEdits(claimUID string, opts containerEditsOptions) (*cdiapi.ContainerEdits, error) { claimUIDUpper := strings.ToUpper(claimUID) - deviceNameUpper := strings.ToUpper(device.Name) + deviceNameUpper := strings.ToUpper(opts.Name) edits := &cdiapi.ContainerEdits{ ContainerEdits: &cdispec.ContainerEdits{ Env: []string{ fmt.Sprintf("DRA_USB_CLAIM_UID_%s=%s", claimUIDUpper, claimUID), - fmt.Sprintf("DRA_USB_DEVICE_NAME_%s=%s", deviceNameUpper, device.Name), - fmt.Sprintf("DRA_USB_CLAIM_UID_%s_DEVICE_NAME=%s", claimUIDUpper, device.Name), - fmt.Sprintf("DRA_USB_%s_DEVICE_PATH=%s", deviceNameUpper, devicePath), - fmt.Sprintf("DRA_USB_%s_BUS_DEVICENUMBER=%s:%s", deviceNameUpper, bus, deviceNum), + fmt.Sprintf("DRA_USB_DEVICE_NAME_%s=%s", deviceNameUpper, opts.Name), + fmt.Sprintf("DRA_USB_CLAIM_UID_%s_DEVICE_NAME=%s", claimUIDUpper, opts.Name), + fmt.Sprintf("DRA_USB_%s_DEVICE_PATH=%s", deviceNameUpper, opts.DevicePath), + fmt.Sprintf("DRA_USB_%s_BUS_DEVICENUMBER=%s:%s", deviceNameUpper, opts.Bus, opts.DeviceNum), }, DeviceNodes: []*cdispec.DeviceNode{ { - Path: devicePath, - HostPath: devicePath, + Path: opts.DevicePath, + HostPath: opts.DevicePath, Type: "c", - Major: major, - Minor: minor, + Major: opts.Major, + Minor: opts.Minor, Permissions: "mrw", UID: ptr.To(uint32(107)), // qemu user. TODO: make this configurable GID: ptr.To(uint32(107)), // qemu group. TODO: make this configurable diff --git a/images/virtualization-dra/internal/usbip/attacher.go b/images/virtualization-dra/internal/usbip/attacher.go new file mode 100644 index 0000000000..c23f24d1c9 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/attacher.go @@ -0,0 +1,302 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usbip + +import ( + "fmt" + "log/slog" + "net" + "os" + "strconv" + "syscall" + + "github.com/deckhouse/virtualization-dra/internal/usbip/protocol" + "github.com/deckhouse/virtualization-dra/pkg/usb" +) + +func NewUSBAttacher() USBAttacher { + return &usbAttacher{} +} + +type usbAttacher struct{} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_attach.c#L174 +func (a usbAttacher) Attach(host, busID string, port int) error { + conn, err := a.usbipNetTCPConnect(host, fmt.Sprintf("%d", port)) + if err != nil { + return fmt.Errorf("failed to connect to usbipd: %w", err) + } + + rhport, err := a.queryImportDevice(conn, busID) + if err != nil { + return fmt.Errorf("failed to query import device: %w", err) + } + + err = a.recordConnection(host, strconv.Itoa(port), busID, rhport) + if err != nil { + return fmt.Errorf("failed to record connection: %w", err) + } + + return nil +} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_detach.c#L32 +func (a usbAttacher) Detach(port int) error { + driver, err := newVhciDriver() + if err != nil { + return fmt.Errorf("failed to get vhci driver: %w", err) + } + + found := false + for i := 0; i < driver.nports; i++ { + idev := &driver.idevs[i] + + if idev.port == port { + found = true + vstatus := protocol.DeviceStatus(idev.status) + if vstatus != protocol.VDeviceStatusNull { + break + } + + slog.Info("Port is already detached", slog.Int("port", port)) + return fmt.Errorf("port is already detached") + + } + } + + if !found { + slog.Error("Invalid port > maxports", slog.Int("port", port), slog.Int("maxports", driver.nports)) + return fmt.Errorf("port %d not found", port) + } + + path := vhciStatePortPath(port) + + if err = os.Remove(path); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove vhci state port file: %w", err) + } + + if err = os.RemoveAll(vhciStatePath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove vhci state path: %w", err) + } + + if err = writeSysfsAttr(vhciHcdDetach, detachAttr{port: port}); err != nil { + return fmt.Errorf("failed to write detach attribute: %w", err) + } + + slog.Info("Port detached", slog.Int("port", port)) + return nil +} + +func (a usbAttacher) GetUsedPorts() ([]int, error) { + driver, err := newVhciDriver() + if err != nil { + return nil, fmt.Errorf("failed to get vhci driver: %w", err) + } + + var ports []int + + for i := 0; i < driver.nports; i++ { + idev := &driver.idevs[i] + + vstatus := protocol.DeviceStatus(idev.status) + if vstatus == protocol.VDeviceStatusUsed { + ports = append(ports, idev.port) + } + } + + return ports, nil +} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_network.c#L261 +func (a usbAttacher) usbipNetTCPConnect(host, port string) (*net.TCPConn, error) { + tcpAddr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(host, port)) + if err != nil { + return nil, fmt.Errorf("resolve TCP address: %w", err) + } + + conn, err := net.DialTCP("tcp", nil, tcpAddr) + if err != nil { + return nil, fmt.Errorf("dial TCP: %w", err) + } + + if err := conn.SetNoDelay(true); err != nil { + conn.Close() + return nil, fmt.Errorf("set TCP_NODELAY: %w", err) + } + + if err := conn.SetKeepAlive(true); err != nil { + conn.Close() + return nil, fmt.Errorf("set keepalive: %w", err) + } + + return conn, nil +} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_attach.c#L120 +func (a usbAttacher) queryImportDevice(conn *net.TCPConn, busID string) (int, error) { + opCommon := protocol.NewOpCommon(protocol.OpReqImport, protocol.OpStatusOk) + importReq := protocol.NewImportRequest(busID) + + if err := opCommon.Encode(conn); err != nil { + return -1, fmt.Errorf("failed to encode OpCommon: %w", err) + } + + if err := importReq.Encode(conn); err != nil { + return -1, fmt.Errorf("failed to encode ImportRequest: %w", err) + } + + importReply := &protocol.ImportReply{} + if err := importReply.Decode(conn); err != nil { + return -1, fmt.Errorf("failed to decode ImportReply: %w", err) + } + + if importReply.Version != protocol.Version { + return -1, fmt.Errorf("unsupported USBIP version: %d", importReply.Version) + } + + if importReply.Status != protocol.OpStatusOk { + return -1, fmt.Errorf("reply failed: %d", importReply.Status) + } + + if importReply.USBDevice.GetBusID() != busID { + return -1, fmt.Errorf("busID mismatch: %s != %s", importReply.USBDevice.GetBusID(), busID) + } + + return a.importDevice(conn, importReply.USBDevice) +} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_attach.c#L81 +func (a usbAttacher) importDevice(conn *net.TCPConn, usbDevice protocol.USBDevice) (int, error) { + port, err := a.getFreePort(usbDevice.Speed) + if err != nil { + return -1, fmt.Errorf("failed to get free port: %w", err) + } + + sockFd, err := a.getSockFd(conn) + if err != nil { + return -1, fmt.Errorf("failed to get socket fd: %w", err) + } + + devID := getDevId(usbDevice.Busnum, usbDevice.Devnum) + + attr := attachAttr{ + port: port, + sockFd: sockFd, + devId: devID, + speed: usbDevice.Speed, + } + + err = writeSysfsAttr(vhciHcdAttach, attr) + if err != nil { + return -1, fmt.Errorf("failed to write attach attribute: %w", err) + } + + return port, nil +} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/libsrc/vhci_driver.c#L334 +func (a usbAttacher) getFreePort(speed uint32) (int, error) { + driver, err := newVhciDriver() + if err != nil { + return -1, err + } + + deviceSpeed := usb.DeviceSpeed(speed) + + for i := 0; i < driver.nports; i++ { + switch deviceSpeed { + case usb.DeviceSpeedSuper: + if driver.idevs[i].hub != hubSpeedSuper { + continue + } + break + default: + if driver.idevs[i].hub != hubSpeedHigh { + continue + } + break + } + vstatus := protocol.DeviceStatus(driver.idevs[i].status) + if vstatus == protocol.VDeviceStatusNull { + return driver.idevs[i].port, nil + + } + } + + return -1, nil +} + +func (a usbAttacher) getSockFd(conn *net.TCPConn) (int, error) { + file, err := conn.File() + if err != nil { + return -1, err + } + defer file.Close() + + fd := int(file.Fd()) + + newFd, err := syscall.Dup(fd) + if err != nil { + return -1, err + } + + return newFd, nil +} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_attach.c#L39 +func (a usbAttacher) recordConnection(host, port, busID string, rhport int) error { + err := os.MkdirAll(vhciStatePath, 0700) + if err != nil { + return fmt.Errorf("failed to create vhci state path: %w", err) + } + + path := vhciStatePortPath(rhport) + + file, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0700) + if err != nil { + return fmt.Errorf("failed to open vhci state port file: %w", err) + } + defer file.Close() + + value := fmt.Sprintf("%s %s %s", host, port, busID) + + _, err = file.WriteString(value) + if err != nil { + return fmt.Errorf("failed to write vhci state port file: %w", err) + } + + return nil +} + +type attachAttr struct { + port int + sockFd int + devId int + speed uint32 +} + +func (a attachAttr) Complete() string { + return fmt.Sprintf("%d %d %d %d", a.port, a.sockFd, a.devId, a.speed) +} + +type detachAttr struct { + port int +} + +func (a detachAttr) Complete() string { + return fmt.Sprintf("%d", a.port) +} diff --git a/images/virtualization-dra/internal/usbip/binder.go b/images/virtualization-dra/internal/usbip/binder.go new file mode 100644 index 0000000000..226eabc6a8 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/binder.go @@ -0,0 +1,224 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usbip + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +func NewUSBBinder() USBBinder { + return &usbBinder{} +} + +type usbBinder struct{} + +// Bind binds the USB device to the USBIP server. +// https://github.com/torvalds/linux/blob/40fbbd64bba6c6e7a72885d2f59b6a3be9991eeb/tools/usb/usbip/src/usbip_bind.c#L130 +func (b *usbBinder) Bind(busID string) error { + devInfo, err := b.getUSBDeviceInfo(busID) + if err != nil { + return fmt.Errorf("device with bus ID %s does not exist: %w", busID, err) + } + + if strings.Contains(devInfo.DevPath, "vhci_hcd") { + return fmt.Errorf("bind loop detected: device %s is already attached to vhci_hcd", busID) + } + + err = b.unbindOther(devInfo) + if err != nil { + return fmt.Errorf("failed to unbind other devices: %w", err) + } + + if err = b.modifyMatchBusID(busID, true); err != nil { + return err + } + + if err = b.bindUsbip(busID); err != nil { + return fmt.Errorf("failed to bind usb device: %w: %w", err, b.modifyMatchBusID(busID, false)) + } + + return b.storeBind(busID, true) +} + +// Unbind unbinds the USB device from the USBIP server. +// https://github.com/torvalds/linux/blob/40fbbd64bba6c6e7a72885d2f59b6a3be9991eeb/tools/usb/usbip/src/usbip_unbind.c#L30 +func (b *usbBinder) Unbind(busID string) error { + devInfo, err := b.getUSBDeviceInfo(busID) + if err != nil { + return fmt.Errorf("device with bus ID %s does not exist: %w", busID, err) + } + + if devInfo.Driver != usbipHostDriverName { + return fmt.Errorf("device %s is not bound to %s driver", devInfo.BusID, usbipHostDriverName) + } + + if err = b.unbindUsbip(busID); err != nil { + return fmt.Errorf("failed to unbind usb device %s: %w", busID, err) + } + + // notify driver of unbind + if err = b.modifyMatchBusID(busID, false); err != nil { + return fmt.Errorf("failed to modify match bus ID %s: %w", busID, err) + } + + // Trigger new probing + if err = b.rebindUsbip(busID); err != nil { + return fmt.Errorf("failed to rebind usb device %s: %w", busID, err) + } + + return b.storeBind(busID, false) +} + +func (b *usbBinder) IsBound(busID string) (bool, error) { + return b.isBound(busID) +} + +type usbDeviceInfo struct { + BusID string + Driver string + DevPath string + IsHub bool +} + +func (b *usbBinder) getUSBDeviceInfo(busID string) (*usbDeviceInfo, error) { + path := getUSBDevicePath(busID) + + if _, err := os.Stat(path); err != nil { + return nil, err + } + + info := &usbDeviceInfo{ + BusID: busID, + } + + bDevClassPath := filepath.Join(path, "bDeviceClass") + if data, err := os.ReadFile(bDevClassPath); err == nil { + info.IsHub = strings.TrimSpace(string(data)) == "09" // 09 = USB Hub class + } + + driverLink := filepath.Join(path, "driver") + if link, err := os.Readlink(driverLink); err == nil { + info.Driver = filepath.Base(link) + } + + ueventPath := filepath.Join(path, "uevent") + if data, err := os.ReadFile(ueventPath); err == nil { + lines := strings.Split(string(data), "\n") + for _, line := range lines { + if strings.HasPrefix(line, "DEVNAME=") { + info.DevPath = filepath.Join("/dev", strings.TrimPrefix(line, "DEVNAME=")) + break + } + } + } + + return info, nil +} + +func (b *usbBinder) storeBind(busID string, bind bool) error { + bound, err := b.isBound(busID) + if err != nil { + return err + } + if bound == bind { + return nil + } + path := bindPath(busID) + if bind { + _, err = os.Create(path) + return err + } + return os.Remove(path) +} + +func (b *usbBinder) isBound(busID string) (bool, error) { + path := bindPath(busID) + _, err := os.Stat(path) + if err != nil { + if os.IsNotExist(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func bindPath(busID string) string { + return filepath.Join(getUSBDevicePath(busID), "usbip_bound") +} + +func (b *usbBinder) unbindOther(devInfo *usbDeviceInfo) error { + if devInfo.IsHub { + return fmt.Errorf("skip unbinding of hub %s", devInfo.BusID) + } + + if devInfo.Driver == "" { + // no driver bound to the device + return nil + } + + if devInfo.Driver == usbipHostDriverName { + return fmt.Errorf("device %s is already bound to %s", devInfo.BusID, usbipHostDriverName) + } + + unbindPath := unbindAttrPath(devInfo.Driver) + + if err := writeSysfsAttr(unbindPath, busIDAttr{busID: devInfo.BusID}); err != nil { + return fmt.Errorf("error unbinding device %s from driver %s: %w", devInfo.BusID, devInfo.Driver, err) + } + + return nil +} + +func (b *usbBinder) bindUsbip(busID string) error { + return writeSysfsAttr(bindAttrPath(usbipHostDriverName), busIDAttr{busID: busID}) +} + +func (b *usbBinder) unbindUsbip(busID string) error { + return writeSysfsAttr(unbindAttrPath(usbipHostDriverName), busIDAttr{busID: busID}) +} + +func (b *usbBinder) rebindUsbip(busID string) error { + return writeSysfsAttr(rebindAttrPath(usbipHostDriverName), busIDAttr{busID: busID}) +} + +func (b *usbBinder) modifyMatchBusID(busID string, add bool) error { + return writeSysfsAttr(matchBusIDAttrPath(usbipHostDriverName), modifyMatchBusIDAttr{busID: busID, add: add}) +} + +type modifyMatchBusIDAttr struct { + busID string + add bool +} + +func (a modifyMatchBusIDAttr) Complete() string { + if a.add { + return fmt.Sprintf("add %s", a.busID) + } + return fmt.Sprintf("del %s", a.busID) +} + +type busIDAttr struct { + busID string +} + +func (a busIDAttr) Complete() string { + return a.busID +} diff --git a/images/virtualization-dra/internal/usbip/interfaces.go b/images/virtualization-dra/internal/usbip/interfaces.go new file mode 100644 index 0000000000..17062585d6 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/interfaces.go @@ -0,0 +1,80 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usbip + +type Interface interface { + ServerInterface + ClientInterface +} + +type ServerInterface interface { + USBBinder +} + +type ClientInterface interface { + USBAttacher +} + +type USBBinder interface { + Bind(busID string) error + Unbind(busID string) error + IsBound(busID string) (bool, error) +} + +type USBAttacher interface { + Attach(host, busID string, port int) error + Detach(port int) error + GetUsedPorts() ([]int, error) +} + +type serverImpl struct { + USBBinder +} + +func NewServer(binder USBBinder) ServerInterface { + return &serverImpl{USBBinder: binder} +} + +type clientImpl struct { + USBAttacher +} + +func NewClient(attacher USBAttacher) ClientInterface { + return &clientImpl{USBAttacher: attacher} +} + +type interfaceImpl struct { + ServerInterface + ClientInterface +} + +func NewInterface(server ServerInterface, client ClientInterface) Interface { + return &interfaceImpl{ + ServerInterface: server, + ClientInterface: client, + } +} + +func New() Interface { + binder := NewUSBBinder() + attacher := NewUSBAttacher() + + server := NewServer(binder) + client := NewClient(attacher) + + return NewInterface(server, client) +} diff --git a/images/virtualization-dra/internal/usbip/protocol/common.go b/images/virtualization-dra/internal/usbip/protocol/common.go new file mode 100644 index 0000000000..c1e3c5f714 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/protocol/common.go @@ -0,0 +1,145 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protocol + +import ( + "encoding/binary" + "fmt" + "io" +) + +type USBVersion uint16 + +const ( + Version USBVersion = 0x0111 +) + +type Op uint16 + +// Common header for all the kinds of PDUs. +const ( + OpRequest Op = 0x80 << 8 + OpReply Op = 0x00 << 8 +) + +// Dummy Code +const ( + OpUnspec Op = 0x00 + OpReqUnspec Op = OpRequest | OpUnspec + OpRepUnspec Op = OpReply | OpUnspec +) + +// Retrieve USB device information. (still not used) +const ( + OpDevInfo Op = 0x02 + OpReqDevInfo Op = OpRequest | OpDevInfo + OpRepDevInfo Op = OpReply | OpDevInfo +) + +// Import a remote USB device. +const ( + OpImport Op = 0x03 + OpReqImport Op = OpRequest | OpImport + OpRepImport Op = OpReply | OpImport +) + +// Negotiate IPSec encryption key. (still not used) +const ( + OpCrypkey Op = 0x04 + OpReqCrypkey Op = OpRequest | OpCrypkey + OpRepCrypkey Op = OpReply | OpCrypkey +) + +// Retrieve the list of exported USB devices. +const ( + OpDevList Op = 0x05 + OpReqDevList Op = OpRequest | OpDevList + OpRepDevList Op = OpReply | OpDevList +) + +// Export a USB device to a remote host. +const ( + OpExport Op = 0x06 + OpReqExport Op = OpRequest | OpExport + OpRepExport Op = OpReply | OpExport +) + +// un-Export a USB device from a remote host. +const ( + OpUnexport Op = 0x07 + OpReqUnexport Op = OpRequest | OpUnexport + OpRepUnexport Op = OpReply | OpUnexport +) + +type OpStatus uint32 + +const ( + OpStatusOk OpStatus = 0x00 + OpStatusNA OpStatus = 0x01 + OpStatusDevBusy OpStatus = 0x02 + OpStatusDevErr OpStatus = 0x03 + OpStatusNoDev OpStatus = 0x04 + OpStatusError OpStatus = 0x05 +) + +type DeviceStatus uint32 + +const ( + DeviceStatusAvailable DeviceStatus = iota + 0x01 + DeviceStatusUsed + DeviceStatusError + VDeviceStatusNull + VDeviceStatusNotAssigned + VDeviceStatusUsed + VDeviceStatusError +) + +func NewOpCommon(code Op, status OpStatus) *OpCommon { + return &OpCommon{ + Version: Version, + Code: code, + Status: status, + } +} + +type OpCommon struct { + Version USBVersion + Code Op + Status OpStatus +} + +func (op *OpCommon) Decode(r io.Reader) error { + buf := make([]byte, 8) + _, err := io.ReadFull(r, buf) + if err != nil { + return fmt.Errorf("failed to read OpCommon: %w", err) + } + + op.Version = USBVersion(binary.BigEndian.Uint16(buf[0:2])) + op.Code = Op(binary.BigEndian.Uint16(buf[2:4])) + op.Status = OpStatus(binary.BigEndian.Uint32(buf[4:8])) + return nil +} + +func (op *OpCommon) Encode(w io.Writer) error { + buf := make([]byte, 8) + binary.BigEndian.PutUint16(buf[0:2], uint16(op.Version)) + binary.BigEndian.PutUint16(buf[2:4], uint16(op.Code)) + binary.BigEndian.PutUint32(buf[4:8], uint32(op.Status)) + _, err := w.Write(buf) + return err +} diff --git a/images/virtualization-dra/internal/usbip/protocol/convert.go b/images/virtualization-dra/internal/usbip/protocol/convert.go new file mode 100644 index 0000000000..21d3b5ad43 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/protocol/convert.go @@ -0,0 +1,52 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protocol + +import "bytes" + +func ToDevicePath(path string) [256]byte { + var result [256]byte + writeCString(result[:], path) + return result +} + +func ToBusID(busID string) [32]byte { + var result [32]byte + writeCString(result[:], busID) + return result +} + +func fromCString(buf []byte) string { + newBytes := buf[:] + if ib := bytes.IndexByte(newBytes, 0); ib != -1 { + newBytes = newBytes[:ib] + } + return string(newBytes) +} + +func writeCString(dst []byte, s string) { + for i := range dst { + dst[i] = 0 + } + + n := len(s) + if n >= len(dst) { + n = len(dst) - 1 + } + + copy(dst[:n], s) +} diff --git a/images/virtualization-dra/internal/usbip/protocol/device_list.go b/images/virtualization-dra/internal/usbip/protocol/device_list.go new file mode 100644 index 0000000000..2f25a56fff --- /dev/null +++ b/images/virtualization-dra/internal/usbip/protocol/device_list.go @@ -0,0 +1,249 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protocol + +import ( + "encoding/binary" + "fmt" + "io" +) + +func NewDeviceList(status OpStatus, devices []USBDeviceInfo) *DeviceList { + return &DeviceList{ + OpCommon: OpCommon{ + Version: Version, + Code: OpReqDevList, + Status: status, + }, + Ndev: uint32(len(devices)), + Devices: devices, + } +} + +type DeviceList struct { + OpCommon + + Ndev uint32 + Devices []USBDeviceInfo +} + +func (d *DeviceList) Encode(w io.Writer) error { + if err := d.OpCommon.Encode(w); err != nil { + return fmt.Errorf("failed to encode OpCommon: %w", err) + } + + buf := make([]byte, 4) + binary.BigEndian.PutUint32(buf[0:4], d.Ndev) + + if _, err := w.Write(buf); err != nil { + return fmt.Errorf("failed to write Ndev to writer: %w", err) + } + + for _, dev := range d.Devices { + if err := dev.Encode(w); err != nil { + return fmt.Errorf("failed to encode USBDeviceInfo: %w", err) + } + } + + return nil +} + +const ( + sysfsPathMax = 256 + sysfsBusIdMax = 32 +) + +type USBDeviceInfo struct { + USBDevice + Interfaces []USBDeviceInterface +} + +func (d *USBDeviceInfo) Decode(r io.Reader) error { + if err := d.USBDevice.Decode(r); err != nil { + return fmt.Errorf("unable to decode USBDevice: %w", err) + } + + d.Interfaces = make([]USBDeviceInterface, d.BNumInterfaces) + for i := 0; i < int(d.BNumInterfaces); i++ { + if err := d.Interfaces[i].Decode(r); err != nil { + return fmt.Errorf("unable to decode USBDeviceInterface: %w", err) + } + } + + return nil +} + +func (d *USBDeviceInfo) Encode(w io.Writer) error { + if err := d.USBDevice.Encode(w); err != nil { + return fmt.Errorf("unable to encode USBDevice: %w", err) + } + + for _, iface := range d.Interfaces { + if err := iface.Encode(w); err != nil { + return fmt.Errorf("unable to encode USBDeviceInterface: %w", err) + } + } + + return nil +} + +type USBDevice struct { + Path [sysfsPathMax]byte + BusID [sysfsBusIdMax]byte + + Busnum uint32 + Devnum uint32 + Speed uint32 + + IDVendor uint16 + IDProduct uint16 + BcdDevice uint16 + + BDeviceClass uint8 + BDeviceSubClass uint8 + BDeviceProtocol uint8 + BConfigurationValue uint8 + BNumConfigurations uint8 + BNumInterfaces uint8 +} + +func (u *USBDevice) GetPath() string { + return fromCString(u.Path[:]) +} + +func (u *USBDevice) GetBusID() string { + return fromCString(u.BusID[:]) +} + +func (u *USBDevice) Decode(r io.Reader) error { + buf := make([]byte, sysfsPathMax+sysfsBusIdMax+12+6+6) + _, err := io.ReadFull(r, buf) + if err != nil { + return fmt.Errorf("failed to read USBDevice from reader: %w", err) + } + + copy(u.Path[:], buf[0:sysfsPathMax]) + copy(u.BusID[:], buf[sysfsPathMax:sysfsPathMax+sysfsBusIdMax]) + + pass := sysfsPathMax + sysfsBusIdMax + + u.Busnum = binary.BigEndian.Uint32(buf[pass : pass+4]) + pass += 4 + u.Devnum = binary.BigEndian.Uint32(buf[pass : pass+4]) + pass += 4 + u.Speed = binary.BigEndian.Uint32(buf[pass : pass+4]) + pass += 4 + + u.IDVendor = binary.BigEndian.Uint16(buf[pass : pass+2]) + pass += 2 + u.IDProduct = binary.BigEndian.Uint16(buf[pass : pass+2]) + pass += 2 + u.BcdDevice = binary.BigEndian.Uint16(buf[pass : pass+2]) + pass += 2 + + u.BDeviceClass = buf[pass] + pass += 1 + u.BDeviceSubClass = buf[pass] + pass += 1 + u.BDeviceProtocol = buf[pass] + pass += 1 + u.BConfigurationValue = buf[pass] + pass += 1 + u.BNumConfigurations = buf[pass] + pass += 1 + u.BNumInterfaces = buf[pass] + + return nil +} + +func (u *USBDevice) Encode(w io.Writer) error { + buf := make([]byte, sysfsPathMax+sysfsBusIdMax+12+6+6) + + copy(buf[0:sysfsPathMax], u.Path[:]) + copy(buf[sysfsPathMax:sysfsPathMax+sysfsBusIdMax], u.BusID[:]) + + pass := sysfsPathMax + sysfsBusIdMax + + binary.BigEndian.PutUint32(buf[pass:pass+4], u.Busnum) + pass += 4 + binary.BigEndian.PutUint32(buf[pass:pass+4], u.Devnum) + pass += 4 + binary.BigEndian.PutUint32(buf[pass:pass+4], u.Speed) + pass += 4 + + binary.BigEndian.PutUint16(buf[pass:pass+2], u.IDVendor) + pass += 2 + binary.BigEndian.PutUint16(buf[pass:pass+2], u.IDProduct) + pass += 2 + binary.BigEndian.PutUint16(buf[pass:pass+2], u.BcdDevice) + pass += 2 + + buf[pass] = u.BDeviceClass + pass += 1 + buf[pass] = u.BDeviceSubClass + pass += 1 + buf[pass] = u.BDeviceProtocol + pass += 1 + buf[pass] = u.BConfigurationValue + pass += 1 + buf[pass] = u.BNumConfigurations + pass += 1 + buf[pass] = u.BNumInterfaces + + _, err := w.Write(buf) + if err != nil { + return fmt.Errorf("failed to write USBDevice to writer: %w", err) + } + return nil +} + +type USBDeviceInterface struct { + BInterfaceClass uint8 + BInterfaceSubClass uint8 + BInterfaceProtocol uint8 + padding uint8 +} + +func (u *USBDeviceInterface) Decode(r io.Reader) error { + buf := make([]byte, 4) + _, err := io.ReadFull(r, buf) + if err != nil { + return fmt.Errorf("failed to read USBDeviceInterface from reader: %w", err) + } + + u.BInterfaceClass = buf[0] + u.BInterfaceSubClass = buf[1] + u.BInterfaceProtocol = buf[2] + u.padding = buf[3] + + return nil +} + +func (u *USBDeviceInterface) Encode(w io.Writer) error { + buf := make([]byte, 4) + + buf[0] = u.BInterfaceClass + buf[1] = u.BInterfaceSubClass + buf[2] = u.BInterfaceProtocol + buf[3] = u.padding + + _, err := w.Write(buf) + if err != nil { + return fmt.Errorf("failed to write USBDeviceInterface to writer: %w", err) + } + return nil +} diff --git a/images/virtualization-dra/internal/usbip/protocol/import.go b/images/virtualization-dra/internal/usbip/protocol/import.go new file mode 100644 index 0000000000..783e659936 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/protocol/import.go @@ -0,0 +1,88 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package protocol + +import ( + "fmt" + "io" +) + +func NewImportRequest(busID string) *ImportRequest { + return &ImportRequest{ + busID: ToBusID(busID), + } +} + +type ImportRequest struct { + busID [sysfsBusIdMax]byte +} + +func (i *ImportRequest) BusID() string { + return fromCString(i.busID[:]) +} + +func (i *ImportRequest) Encode(w io.Writer) error { + _, err := w.Write(i.busID[:]) + return err +} + +func (i *ImportRequest) Decode(r io.Reader) error { + buf := make([]byte, sysfsBusIdMax) + _, err := io.ReadFull(r, buf) + if err != nil { + return fmt.Errorf("failed to read ImportRequest from reader: %w", err) + } + + copy(i.busID[:], buf) + return nil +} + +type ImportReply struct { + OpCommon + USBDevice +} + +func NewImportReply(status OpStatus, device USBDevice) *ImportReply { + return &ImportReply{ + OpCommon: OpCommon{ + Version: Version, + Code: OpRepImport, + Status: status, + }, + USBDevice: device, + } +} + +func (i *ImportReply) Encode(w io.Writer) error { + if err := i.OpCommon.Encode(w); err != nil { + return fmt.Errorf("failed to encode OpCommon: %w", err) + } + if err := i.USBDevice.Encode(w); err != nil { + return fmt.Errorf("failed to encode USBDevice: %w", err) + } + return nil +} + +func (i *ImportReply) Decode(r io.Reader) error { + if err := i.OpCommon.Decode(r); err != nil { + return fmt.Errorf("failed to decode OpCommon: %w", err) + } + if err := i.USBDevice.Decode(r); err != nil { + return fmt.Errorf("failed to decode USBDevice: %w", err) + } + return nil +} diff --git a/images/virtualization-dra/internal/usbip/sysfs.go b/images/virtualization-dra/internal/usbip/sysfs.go new file mode 100644 index 0000000000..cb41335791 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/sysfs.go @@ -0,0 +1,79 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usbip + +import ( + "fmt" + "os" +) + +type sysfsAttr interface { + Complete() string +} + +func writeSysfsAttr(attrPath string, value sysfsAttr) error { + f, err := os.OpenFile(attrPath, os.O_WRONLY, 0644) + if err != nil { + return err + } + defer f.Close() + + _, err = f.WriteString(value.Complete()) + return err +} + +const ( + bindAttrPathTmpl = "/sys/bus/usb/drivers/%s/bind" + unbindAttrPathTmpl = "/sys/bus/usb/drivers/%s/unbind" + rebindAttrPathTmpl = "/sys/bus/usb/drivers/%s/rebind" + matchBusIDAttrPathTmpl = "/sys/bus/usb/drivers/%s/match_busid" + + usbDevicesTmpl = "/sys/bus/usb/devices/%s" + + usbipStatusPathTmpl = "/sys/bus/usb/devices/%s/usbip_status" + usbipSockFdPathTmpl = "/sys/bus/usb/devices/%s/usbip_sockfd" + + usbipHostDriverName = "usbip-host" +) + +func getUSBDevicePath(busID string) string { + return fmt.Sprintf(usbDevicesTmpl, busID) +} + +func bindAttrPath(driver string) string { + return fmt.Sprintf(bindAttrPathTmpl, driver) +} + +func unbindAttrPath(driver string) string { + return fmt.Sprintf(unbindAttrPathTmpl, driver) +} + +func rebindAttrPath(driver string) string { + return fmt.Sprintf(rebindAttrPathTmpl, driver) +} + +func matchBusIDAttrPath(driver string) string { + return fmt.Sprintf(matchBusIDAttrPathTmpl, driver) +} + +func usbipStatusPath(busID string) string { + return fmt.Sprintf(usbipStatusPathTmpl, busID) +} + +func usbipSockFdPath(busID string) string { + return fmt.Sprintf(usbipSockFdPathTmpl, busID) +} diff --git a/images/virtualization-dra/internal/usbip/usbip.go b/images/virtualization-dra/internal/usbip/usbip.go new file mode 100644 index 0000000000..959d25ae5f --- /dev/null +++ b/images/virtualization-dra/internal/usbip/usbip.go @@ -0,0 +1,17 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usbip diff --git a/images/virtualization-dra/internal/usbip/usbipd.go b/images/virtualization-dra/internal/usbip/usbipd.go new file mode 100644 index 0000000000..055fbd13a2 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/usbipd.go @@ -0,0 +1,442 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usbip + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "log/slog" + "net" + "os" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/deckhouse/virtualization-dra/internal/usbip/protocol" + "github.com/deckhouse/virtualization-dra/pkg/usb" +) + +const ( + defaultMaxTCPConnection = 100 + defaultGracefulShutdownTimeout = 30 * time.Second +) + +func makeOptions(opts ...Option) *options { + options := &options{ + maxTCPConnection: defaultMaxTCPConnection, + gracefulShutdownTimeout: defaultGracefulShutdownTimeout, + } + + for _, opt := range opts { + opt(options) + } + + return options +} + +type options struct { + tlsConfig *tls.Config + gracefulShutdownTimeout time.Duration + maxTCPConnection int +} + +type Option func(*options) + +func WithTLSConfig(tlsConfig *tls.Config) Option { + return func(o *options) { + o.tlsConfig = tlsConfig + } +} +func WithGracefulShutdownTimeout(timeout time.Duration) Option { + return func(o *options) { + o.gracefulShutdownTimeout = timeout + } +} + +func WithMaxTCPConnection(maxTCPConnection int) Option { + return func(o *options) { + o.maxTCPConnection = maxTCPConnection + } +} + +func NewUSBIPD(port int, monitor *usb.Monitor, opts ...Option) *USBIPD { + options := makeOptions(opts...) + return &USBIPD{ + addr: ":" + strconv.Itoa(port), + tlsConfig: options.tlsConfig, + gracefulShutdownTimeout: options.gracefulShutdownTimeout, + logger: slog.Default().With(slog.String("component", "usbipd")), + maxTCPConnection: options.maxTCPConnection, + monitor: monitor, + } + +} + +type USBIPD struct { + addr string + tlsConfig *tls.Config + gracefulShutdownTimeout time.Duration + logger *slog.Logger + maxTCPConnection int + + listener net.Listener + connWg sync.WaitGroup + connCount atomic.Int64 + quit chan struct{} + + monitor *usb.Monitor +} + +func (u *USBIPD) Start(ctx context.Context) error { + if err := u.setup(); err != nil { + return err + } + + u.connWg.Add(1) + go u.run(ctx) + + return nil +} + +func (u *USBIPD) Run(ctx context.Context) error { + if err := u.setup(); err != nil { + return err + } + + u.connWg.Add(1) + u.run(ctx) + + return nil +} + +func (u *USBIPD) setup() (err error) { + if u.tlsConfig != nil { + u.listener, err = tls.Listen("tcp", u.addr, u.tlsConfig) + if err != nil { + return err + } + } else { + u.listener, err = net.Listen("tcp", u.addr) + if err != nil { + return err + } + } + return nil +} + +func (u *USBIPD) run(ctx context.Context) { + var connCount atomic.Int64 + defer u.connWg.Done() + for { + conn, err := u.listener.Accept() + // Error occurred when + // 1. Connection error + // 2. The listener is closed (quit channel is closed) + if err != nil { + select { + case <-u.quit: + return + default: + u.logger.Error("unsable to accept request", slog.String("address", u.addr), slog.Any("err", err)) + } + } else { + // Check if TCP connection reached the limit specified in given config + count := connCount.Load() + if count+1 > int64(u.maxTCPConnection) { + u.logger.Error("maximum TCP connection reached, drop the connection", slog.Int64("count", count)) + conn.Close() + continue + } + + // TCP connection handler + u.connWg.Add(1) + connCount.Add(1) + go func() { + defer connCount.Add(-1) + defer u.connWg.Done() + defer conn.Close() + + u.logger.Info("new connection established", slog.String("addr", conn.RemoteAddr().String())) + keepConn, err := u.handleConnection(conn) + if err != nil { + if !errors.Is(err, io.EOF) { + u.logger.Error("failed to handle connection", slog.Any("err", err), slog.String("addr", conn.RemoteAddr().String())) + } else { + u.logger.Info("connection EOF", slog.String("addr", conn.RemoteAddr().String())) + } + } + if keepConn { + // don't handle and read from the socket. other work doing a kernel module + <-ctx.Done() + } + u.logger.Info("connection closed", slog.String("addr", conn.RemoteAddr().String())) + }() + } + } +} + +// https://docs.kernel.org/usb/usbip_protocol.html +// https://github.com/torvalds/linux/blob/9448598b22c50c8a5bb77a9103e2d49f134c9578/tools/usb/usbip/src/usbipd.c#L251 +func (u *USBIPD) handleConnection(conn net.Conn) (bool, error) { + opCommon := &protocol.OpCommon{} + if err := opCommon.Decode(conn); err != nil { + return false, fmt.Errorf("failed to decode OpCommon: %w", err) + } + + if opCommon.Version != protocol.Version { + return false, fmt.Errorf("unsupported USBIP version: %d", opCommon.Version) + } + + if opCommon.Status != protocol.OpStatusOk { + return false, fmt.Errorf("request failed: %d", opCommon.Status) + } + + switch opCommon.Code { + case protocol.OpReqDevList: + if err := u.handleDeviceList(conn); err != nil { + return false, fmt.Errorf("failed to handle OpReqDevList: %w", err) + } + case protocol.OpReqImport: + if err := u.handleImportRequest(conn); err != nil { + return false, fmt.Errorf("failed to handle OpReqImport: %w", err) + } + return true, nil + case protocol.OpReqDevInfo, protocol.OpReqCrypkey: + // nothing to do + default: + return false, fmt.Errorf("unsupported OpCommon.Code: %d", opCommon.Code) + } + + return false, nil +} + +// https://github.com/torvalds/linux/blob/9448598b22c50c8a5bb77a9103e2d49f134c9578/tools/usb/usbip/src/usbipd.c#L229 +func (u *USBIPD) handleDeviceList(conn net.Conn) error { + info := u.getUSBDeviceInfo() + if len(info) == 0 { + slog.Info("no USB devices found") + } + devList := protocol.NewDeviceList(protocol.OpStatusOk, info) + return devList.Encode(conn) +} + +// https://github.com/torvalds/linux/blob/9448598b22c50c8a5bb77a9103e2d49f134c9578/tools/usb/usbip/src/usbipd.c#L91 +func (u *USBIPD) handleImportRequest(conn net.Conn) error { + importReq := &protocol.ImportRequest{} + if err := importReq.Decode(conn); err != nil { + return fmt.Errorf("failed to decode ImportRequest: %w", err) + } + + busID := importReq.BusID() + log := u.logger.With(slog.String("busID", busID)) + log.Info("import request") + + devices := u.monitor.GetDevices() + + var bindDevice *usb.Device + for _, device := range devices { + if device.BusID == busID { + log.Info("found device for export") + bindDevice = &device + break + } + } + + status := protocol.OpStatusOk + + if bindDevice != nil { + // should set TCP_NODELAY for usbip + u.setNotDelay(conn) + + status = u.exportDevice(conn, bindDevice) + if status != protocol.OpStatusOk { + log.Error("failed to export device") + } + + } else { + // not found + status = protocol.OpStatusNoDev + } + + u.logger.Info("export device", slog.Any("device", bindDevice)) + + usbDevice := toUSBDeviceInfo(bindDevice).USBDevice + reply := protocol.NewImportReply(status, usbDevice) + + return reply.Encode(conn) +} + +// https://github.com/torvalds/linux/blob/9448598b22c50c8a5bb77a9103e2d49f134c9578/tools/usb/usbip/libsrc/usbip_host_common.c#L212 +func (u *USBIPD) exportDevice(conn net.Conn, device *usb.Device) protocol.OpStatus { + + log := u.logger.With(slog.String("busID", device.BusID)) + log.Info("export request") + + usbIpStatus, err := u.getUSBIPStatus(device) + if err != nil { + log.Error("failed to get USBIP status", slog.Any("error", err)) + return protocol.OpStatusError + } + + if usbIpStatus != protocol.DeviceStatusAvailable { + log.Info("USBIP status is not available") + switch usbIpStatus { + case protocol.DeviceStatusError: + log.Debug("USBIP status is error") + return protocol.OpStatusDevErr + case protocol.DeviceStatusUsed: + log.Debug("USBIP status is used") + return protocol.OpStatusDevBusy + default: + log.Debug("USBIP status unknown") + return protocol.OpStatusNA + } + } + + syscallConn, ok := conn.(syscall.Conn) + if !ok { + log.Error("conn does not implement syscall.Conn") + return protocol.OpStatusNA + } + + var sockFd int + rawConn, err := syscallConn.SyscallConn() + if err != nil { + log.Error("failed to get raw connection", slog.Any("error", err)) + return protocol.OpStatusNA + } + err = rawConn.Control(func(fd uintptr) { + sockFd = int(fd) + }) + if err != nil { + log.Error("failed to get socket fd", slog.Any("error", err)) + return protocol.OpStatusNA + } + + err = writeSysfsAttr(usbipSockFdPath(device.BusID), sockFdAttr{sockFd: sockFd}) + if err != nil { + log.Error("failed to write usbip_sockfd", slog.Any("error", err)) + return protocol.OpStatusNA + } + + log.Info("Connect") + + return protocol.OpStatusOk +} + +type sockFdAttr struct { + sockFd int +} + +func (a sockFdAttr) Complete() string { + return fmt.Sprintf("%d\n", a.sockFd) +} + +func (u *USBIPD) getUSBIPStatus(device *usb.Device) (protocol.DeviceStatus, error) { + statusPath := usbipStatusPath(device.BusID) + + data, err := os.ReadFile(statusPath) + if err != nil { + return 0, fmt.Errorf("failed to read %s: %w", statusPath, err) + } + + statusStr := strings.TrimSpace(string(data)) + + value, err := strconv.ParseUint(statusStr, 10, 32) + if err != nil { + return 0, fmt.Errorf("invalid status value %q: %w", statusStr, err) + } + + status := protocol.DeviceStatus(value) + + return status, nil +} + +func (u *USBIPD) setNotDelay(conn net.Conn) { + tcpConn, ok := conn.(*net.TCPConn) + if ok { + err := tcpConn.SetNoDelay(true) + if err != nil { + u.logger.Error("failed to set TCP_NODELAY", slog.String("error", err.Error())) + } + return + } + u.logger.Error("failed to cast connection to TCPConn") +} + +// TODO: check already used devices +func (u *USBIPD) getUSBDeviceInfo() []protocol.USBDeviceInfo { + devices := u.monitor.GetDevices() + + var bindDevices []protocol.USBDeviceInfo + + for _, device := range devices { + if device.Driver == usbipHostDriverName { + bindDevice := toUSBDeviceInfo(&device) + bindDevices = append(bindDevices, bindDevice) + } + } + + return bindDevices +} + +func toUSBDeviceInfo(device *usb.Device) protocol.USBDeviceInfo { + if device == nil { + return protocol.USBDeviceInfo{} + } + return protocol.USBDeviceInfo{ + USBDevice: protocol.USBDevice{ + Path: protocol.ToDevicePath(device.DevicePath), + BusID: protocol.ToBusID(device.BusID), + Busnum: device.Bus, + Devnum: device.DeviceNumber, + Speed: toSpeed(device.Speed), + IDVendor: device.VendorID, + IDProduct: device.ProductID, + BcdDevice: device.BCD, + BDeviceClass: device.BDeviceClass, + BDeviceSubClass: device.BDeviceSubClass, + BDeviceProtocol: device.BDeviceProtocol, + BConfigurationValue: device.BConfigurationValue, + BNumConfigurations: device.BNumConfigurations, + BNumInterfaces: device.BNumInterfaces, + }, + Interfaces: toInterfaces(device.Interfaces), + } +} + +func toInterfaces(interfaces []usb.DeviceInterface) []protocol.USBDeviceInterface { + result := make([]protocol.USBDeviceInterface, len(interfaces)) + for i, iface := range interfaces { + result[i] = protocol.USBDeviceInterface{ + BInterfaceClass: iface.BInterfaceClass, + BInterfaceSubClass: iface.BInterfaceSubClass, + BInterfaceProtocol: iface.BInterfaceProtocol, + } + } + return result +} + +func toSpeed(speed uint32) uint32 { + return uint32(usb.ResolveDeviceSpeed(speed)) +} diff --git a/images/virtualization-dra/internal/usbip/usbipd_config.go b/images/virtualization-dra/internal/usbip/usbipd_config.go new file mode 100644 index 0000000000..49ddadcea6 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/usbipd_config.go @@ -0,0 +1,164 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usbip + +import ( + "crypto/tls" + "crypto/x509" + "flag" + "fmt" + "os" + "time" + + "github.com/deckhouse/virtualization-dra/pkg/usb" +) + +type ClientAuthType tls.ClientAuthType + +func (c *ClientAuthType) String() string { + cc := tls.ClientAuthType(*c) + return cc.String() +} + +func (c *ClientAuthType) Set(s string) error { + switch s { + case "NoClientCert": + *c = ClientAuthType(tls.NoClientCert) + case "RequestClientCert": + *c = ClientAuthType(tls.RequestClientCert) + case "RequireAnyClientCert": + *c = ClientAuthType(tls.RequireAnyClientCert) + case "VerifyClientCertIfGiven": + *c = ClientAuthType(tls.VerifyClientCertIfGiven) + case "RequireAndVerifyClientCert": + *c = ClientAuthType(tls.RequireAndVerifyClientCert) + default: + return fmt.Errorf("invalid client auth type: %s", s) + } + return nil +} + +type USBIPDConfig struct { + ServerCertificateFile string + ServerKeyFile string + + RootCAFile string + + ClientCAFile string + ClientAuthType *ClientAuthType + clientAuthType int + InsecureSkipVerify bool + + Port int + GracefulShutdownTimeout time.Duration + + Monitor *usb.Monitor +} + +func (c *USBIPDConfig) AddFlags(fs *flag.FlagSet) { + fs.IntVar(&c.Port, "usbipd-port", 0, "USBIPD port") + fs.StringVar(&c.ServerCertificateFile, "usbipd-server-certificate-file", "", "USBIPD server certificate file") + fs.StringVar(&c.ServerKeyFile, "usbipd-server-key-file", "", "USBIPD server key file") + fs.StringVar(&c.RootCAFile, "usbipd-root-ca-file", "", "USBIPD root CA file") + fs.StringVar(&c.ClientCAFile, "usbipd-client-ca-file", "", "USBIPD client CA file") + fs.Var(c.ClientAuthType, "usbipd-client-auth-type", "USBIPD client auth type") + fs.BoolVar(&c.InsecureSkipVerify, "usbipd-insecure-skip-verify", false, "USBIPD insecure skip verify") + fs.DurationVar(&c.GracefulShutdownTimeout, "usbipd-graceful-shutdown-timeout", 0, "USBIPD graceful shutdown timeout") +} + +func (c *USBIPDConfig) Validate() error { + if c.Port == 0 { + return fmt.Errorf("port is required") + } + + if c.ServerCertificateFile != "" && c.ServerKeyFile == "" { + return fmt.Errorf("server key file is required if server certificate file is provided") + } + + if c.ServerCertificateFile == "" && c.ServerKeyFile != "" { + return fmt.Errorf("server certificate file is required if server key file is provided") + } + + if c.Monitor == nil { + return fmt.Errorf("monitor is required") + } + + return nil +} + +func (c *USBIPDConfig) Complete() (*USBIPD, error) { + var opts []Option + if c.GracefulShutdownTimeout != 0 { + opts = append(opts, WithGracefulShutdownTimeout(c.GracefulShutdownTimeout)) + } + + var serverCertificate *tls.Certificate + if c.ServerCertificateFile != "" && c.ServerKeyFile != "" { + certificate, err := tls.LoadX509KeyPair(c.ServerCertificateFile, c.ServerKeyFile) + if err != nil { + return nil, err + } + serverCertificate = &certificate + } + + rootCACertPool, err := loadCAPoolFromFile(c.RootCAFile) + if err != nil { + return nil, err + } + + clientCACertPool, err := loadCAPoolFromFile(c.ClientCAFile) + if err != nil { + return nil, err + } + + if serverCertificate != nil || rootCACertPool != nil || clientCACertPool != nil { + tlsConfig := &tls.Config{ + RootCAs: rootCACertPool, + ClientCAs: clientCACertPool, + InsecureSkipVerify: c.InsecureSkipVerify, + } + if serverCertificate != nil { + tlsConfig.Certificates = []tls.Certificate{*serverCertificate} + } + if c.ClientAuthType != nil { + tlsConfig.ClientAuth = tls.ClientAuthType(*c.ClientAuthType) + } + + opts = append(opts, WithTLSConfig(tlsConfig)) + } + + return NewUSBIPD(c.Port, c.Monitor, opts...), nil + +} + +func loadCAPoolFromFile(file string) (*x509.CertPool, error) { + if file == "" { + return nil, nil + } + + caCertPool := x509.NewCertPool() + caCertPEMBlock, err := os.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("failed to read CA certificate: %w", err) + } + + if !caCertPool.AppendCertsFromPEM(caCertPEMBlock) { + return nil, fmt.Errorf("failed to parse CA certificate") + } + + return caCertPool, nil +} diff --git a/images/virtualization-dra/internal/usbip/vhci.go b/images/virtualization-dra/internal/usbip/vhci.go new file mode 100644 index 0000000000..f6439f1825 --- /dev/null +++ b/images/virtualization-dra/internal/usbip/vhci.go @@ -0,0 +1,213 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usbip + +import ( + "bytes" + "fmt" + "os" + "strconv" + "strings" +) + +const ( + vhciStatePath = "/var/run/vhci_hcd" + platformPath = "/sys/devices/platform" + usbipVhciHcdNPortsPath = "/sys/devices/platform/vhci_hcd.0/nports" + + vhciHcdAttach = "/sys/devices/platform/vhci_hcd.0/attach" + vhciHcdDetach = "/sys/devices/platform/vhci_hcd.0/detach" + vhciHcdStatus = "/sys/devices/platform/vhci_hcd.0/status" + secondaryVhciHcdStatusTmpl = "/sys/devices/platform/vhci_hcd.%d/status.%d" + + vhciStatePortTmpl = "/var/run/vhci_hcd/port%d" +) + +func vhciStatePortPath(port int) string { + return fmt.Sprintf(vhciStatePortTmpl, port) +} + +func secondaryVhciHcdStatusPath(count int) string { + return fmt.Sprintf(secondaryVhciHcdStatusTmpl, count, count) +} + +type vhciDriver struct { + nports int + ncontrollers int + idevs []importDevice +} + +type importDevice struct { + hub hubSpeed + port, status, devID, busnum, devnum int +} + +type hubSpeed int + +const ( + hubSpeedHigh hubSpeed = iota + hubSpeedSuper +) + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/libsrc/vhci_driver.c#L243 +func newVhciDriver() (*vhciDriver, error) { + nports, err := getNPorts() + if err != nil { + return nil, err + } + ncontrollers, err := getNControllers() + if err != nil { + return nil, err + } + + driver := &vhciDriver{ + nports: nports, + ncontrollers: ncontrollers, + } + + err = driver.refreshImportDeviceList() + if err != nil { + return nil, fmt.Errorf("failed to refresh import device list: %w", err) + } + + return driver, nil +} + +func getNPorts() (int, error) { + data, err := os.ReadFile(usbipVhciHcdNPortsPath) + if err != nil { + return -1, err + } + + nports, err := strconv.Atoi(strings.TrimSpace(string(data))) + if err != nil { + return -1, err + } + + return nports, nil +} + +func getNControllers() (int, error) { + entries, err := os.ReadDir(platformPath) + if err != nil { + return -1, err + } + count := 0 + for _, entry := range entries { + if entry.IsDir() && strings.HasPrefix(entry.Name(), "vhci_hcd") { + count++ + } + } + + return count, nil +} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/libsrc/vhci_driver.c#L111 +func (d *vhciDriver) refreshImportDeviceList() error { + + status := vhciHcdStatus + + for i := 0; i < d.ncontrollers; i++ { + if i > 0 { + status = secondaryVhciHcdStatusPath(i) + } + + attrStatus, err := os.ReadFile(status) + if err != nil { + return fmt.Errorf("failed to read %s: %w", status, err) + } + + err = d.parseStatus(attrStatus) + if err != nil { + return fmt.Errorf("failed to parse attr status %s: %w", status, err) + } + } + + return nil +} + +// https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/libsrc/vhci_driver.c#L40 +func (d *vhciDriver) parseStatus(statusBytes []byte) error { + lines := strings.Split(string(statusBytes), "\n") + + // hub port sta spd dev sockfd local_busid + // hs 0000 004 000 00000000 000000 0-0 + // hs 0001 004 000 00000000 000000 0-0 + // hs 0002 004 000 00000000 000000 0-0 + + head := true + for _, line := range lines { + if head { + // skip header + head = false + continue + } + + if strings.TrimSpace(line) == "" { + continue + } + + var ( + hub string + port, status, speed, devID, sockFd int + localBusID string + ) + + buf := bytes.NewBufferString(line) + _, err := fmt.Fscanf(buf, "%2s %d %d %d %x %d %31s", &hub, &port, &status, &speed, &devID, &sockFd, &localBusID) + if err != nil { + return fmt.Errorf("failed to parse status: %w", err) + } + + if len(d.idevs) <= port { + idevs := make([]importDevice, port+1) + for i, idev := range d.idevs { + idevs[i] = idev + } + d.idevs = idevs + } + + busnum, devnum := getBusNumDevNum(devID) + + idev := &d.idevs[port] + + idev.port = port + idev.status = status + idev.devID = devID + idev.busnum = busnum + idev.devnum = devnum + + if hub == "hs" { + idev.hub = hubSpeedHigh + } else if hub == "ss" { + idev.hub = hubSpeedSuper + } + } + + return nil +} + +func getDevId(busnum, devnum uint32) int { + return int((busnum << 16) | devnum) +} + +func getBusNumDevNum(devID int) (int, int) { + busnum := devID >> 16 + devnum := devID & 0x0000ffff + + return busnum, devnum +} diff --git a/images/virtualization-dra/pkg/modprobe/modprobe.go b/images/virtualization-dra/pkg/modprobe/modprobe.go new file mode 100644 index 0000000000..838285a926 --- /dev/null +++ b/images/virtualization-dra/pkg/modprobe/modprobe.go @@ -0,0 +1,58 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package modprobe + +import ( + "fmt" + "os" + "path/filepath" + + "golang.org/x/sys/unix" +) + +func LoadModules(modules []string) error { + var uts unix.Utsname + if err := unix.Uname(&uts); err != nil { + return fmt.Errorf("uname: %w", err) + } + + kernel := unix.ByteSliceToString(uts.Release[:]) + base := filepath.Join("/lib/modules", kernel) + + for _, m := range modules { + path := filepath.Join(base, m) + if err := loadModule(path); err != nil { + return fmt.Errorf("load module %s: %w", path, err) + } + } + + return nil +} + +func loadModule(path string) error { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("open %s: %w", path, err) + } + defer f.Close() + + if err = unix.FinitModule(int(f.Fd()), "", 0); err != nil { + return fmt.Errorf("finit_module %s: %w", path, err) + } + + return nil +} diff --git a/images/virtualization-dra/pkg/usb/discovery.go b/images/virtualization-dra/pkg/usb/discovery.go new file mode 100644 index 0000000000..f3a7a19b4e --- /dev/null +++ b/images/virtualization-dra/pkg/usb/discovery.go @@ -0,0 +1,87 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" +) + +func DiscoverPluggedUSBDevices(pathToUSBDevices string) (map[string]*Device, error) { + devices := make(map[string]*Device) + + err := filepath.Walk(pathToUSBDevices, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if !isUsbPath(path) { + return nil + } + + // Get device information + device, err := LoadDevice(path) + if err != nil { + return err + } + if err = device.Validate(); err != nil { + slog.Error("failed to validate device, skip...", slog.Any("device", device), slog.String("error", err.Error())) + return nil + } + + devices[path] = &device + return nil + }) + + if err != nil { + return nil, fmt.Errorf("failed when walking usb devices tree: %w", err) + } + + return devices, nil +} + +func isUsbPath(path string) bool { + // Ignore named usb controllers + if strings.HasPrefix(filepath.Base(path), "usb") { + return false + } + // We are interested in actual USB devices information that + // contains idVendor and idProduct. We can skip all others. + for _, file := range requiredFiles { + if _, err := os.Stat(filepath.Join(path, file)); err != nil { + return false + } + } + + return true +} + +var requiredFiles = []string{ + "idVendor", + "idProduct", + "uevent", + "serial", + "manufacturer", + "product", + "bConfigurationValue", + "bNumConfigurations", + "bNumInterfaces", + "speed", +} diff --git a/images/virtualization-dra/pkg/usb/monitor.go b/images/virtualization-dra/pkg/usb/monitor.go new file mode 100644 index 0000000000..d5b12204e5 --- /dev/null +++ b/images/virtualization-dra/pkg/usb/monitor.go @@ -0,0 +1,180 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + "context" + "fmt" + "log/slog" + "maps" + "slices" + "strings" + "sync" + "time" + + "github.com/fsnotify/fsnotify" +) + +type Monitor struct { + mu sync.RWMutex + devices map[string]*Device + watcher *fsnotify.Watcher + notifiers []Notifier +} + +func NewMonitor(ctx context.Context, resyncPeriod time.Duration) (*Monitor, error) { + devices, err := DiscoverPluggedUSBDevices(PathToUSBDevices) + if err != nil { + return nil, err + } + if devices == nil { + devices = make(map[string]*Device) + } + + // TODO: recursive watcher + watcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + if err = watcher.Add(PathToUSBDevices); err != nil { + _ = watcher.Close() + return nil, fmt.Errorf("failed to add USB devices path to fsnotify watcher: %w", err) + + } + + monitor := &Monitor{ + devices: devices, + watcher: watcher, + } + + go func() { + monitor.run(ctx, resyncPeriod) + }() + + return monitor, nil +} + +func (m *Monitor) run(ctx context.Context, resyncPeriod time.Duration) { + for { + select { + case <-ctx.Done(): + _ = m.watcher.Close() + return + case event := <-m.watcher.Events: + switch event.Op { + case fsnotify.Create, fsnotify.Write: + if err := m.handleUpdate(event); err != nil { + slog.Error("failed to handle update", slog.String("error", err.Error())) + } + case fsnotify.Remove: + m.handleRemove(event) + default: + continue + } + case err := <-m.watcher.Errors: + slog.Error("error watching USB devices", slog.String("error", err.Error())) + + case <-time.After(resyncPeriod): + devices, err := DiscoverPluggedUSBDevices(PathToUSBDevices) + if err != nil { + slog.Error("failed to discover USB devices", slog.String("error", err.Error())) + continue + } + if devices == nil { + devices = make(map[string]*Device) + } + m.mu.Lock() + if !maps.Equal(m.devices, devices) { + m.devices = devices + m.notify() + } + m.mu.Unlock() + } + } +} + +func (m *Monitor) handleUpdate(event fsnotify.Event) error { + path := event.Name + if !isUsbPath(path) { + return nil + } + + // Get device information + device, err := LoadDevice(path) + if err != nil { + return err + } + if err = device.Validate(); err != nil { + slog.Error("failed to validate device, skip...", slog.Any("device", device), slog.String("error", err.Error())) + return nil + } + + m.mu.Lock() + defer m.mu.Unlock() + + oldDevice, ok := m.devices[path] + if !ok || !device.Equal(oldDevice) { + m.devices[path] = &device + m.notify() + } + + return nil +} + +func (m *Monitor) handleRemove(event fsnotify.Event) { + path := event.Name + + m.mu.Lock() + defer m.mu.Unlock() + + if _, ok := m.devices[path]; ok { + delete(m.devices, path) + m.notify() + } +} + +func (m *Monitor) GetDevices() []Device { + m.mu.RLock() + devices := make([]Device, 0, len(m.devices)) + for _, device := range m.devices { + devices = append(devices, *device) + } + m.mu.RUnlock() + + slices.SortFunc(devices, func(a, b Device) int { + return strings.Compare(a.DevicePath, b.DevicePath) + }) + + return devices +} + +func (m *Monitor) AddNotifier(notifier Notifier) { + m.mu.Lock() + defer m.mu.Unlock() + + m.notifiers = append(m.notifiers, notifier) +} + +func (m *Monitor) notify() { + for _, notifier := range m.notifiers { + notifier.Notify() + } +} + +type Notifier interface { + Notify() +} diff --git a/images/virtualization-dra/pkg/usb/speed.go b/images/virtualization-dra/pkg/usb/speed.go new file mode 100644 index 0000000000..874d45b72e --- /dev/null +++ b/images/virtualization-dra/pkg/usb/speed.go @@ -0,0 +1,46 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +type DeviceSpeed uint32 + +const ( + DeviceSpeedUnknown DeviceSpeed = iota // enumerating + DeviceSpeedLow // usb 1.1 + DeviceSpeedFull // usb 1.1 + DeviceSpeedHigh // usb 2.0 + DeviceSpeedSuper // usb 3.0 + DeviceSpeedSuperPlus // usb 3.1 +) + +// https://mjmwired.net/kernel/Documentation/ABI/testing/sysfs-bus-usb#502 +func ResolveDeviceSpeed(speed uint32) DeviceSpeed { + switch speed { + case 1: + return DeviceSpeedLow + case 12, 15: + return DeviceSpeedFull + case 480: + return DeviceSpeedHigh + case 5000: + return DeviceSpeedSuper + case 10000, 20000: + return DeviceSpeedSuperPlus + default: + return DeviceSpeedUnknown + } +} diff --git a/images/virtualization-dra/pkg/usb/usb.go b/images/virtualization-dra/pkg/usb/usb.go new file mode 100644 index 0000000000..ae220ea329 --- /dev/null +++ b/images/virtualization-dra/pkg/usb/usb.go @@ -0,0 +1,452 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package usb + +import ( + "bufio" + "fmt" + "log/slog" + "os" + "path/filepath" + "slices" + "strconv" + "strings" +) + +const PathToUSBDevices = "/sys/bus/usb/devices" + +type Device struct { + Path string + BusID string + Manufacturer string + Product string + Serial string + DevicePath string + Driver string + IsHub bool + VendorID uint16 + ProductID uint16 + BCD uint16 + Bus uint32 + DeviceNumber uint32 + Speed uint32 + Major uint64 + Minor uint64 + BDeviceClass uint8 + BDeviceSubClass uint8 + BDeviceProtocol uint8 + BConfigurationValue uint8 + BNumConfigurations uint8 + BNumInterfaces uint8 + Interfaces []DeviceInterface +} + +type DeviceInterface struct { + BInterfaceClass uint8 + BInterfaceSubClass uint8 + BInterfaceProtocol uint8 +} + +func (d *Device) Equal(other *Device) bool { + return d.Path == other.Path && + d.BusID == other.BusID && + d.Manufacturer == other.Manufacturer && + d.Product == other.Product && + d.Serial == other.Serial && + d.DevicePath == other.DevicePath && + d.Driver == other.Driver && + d.IsHub == other.IsHub && + d.VendorID == other.VendorID && + d.ProductID == other.ProductID && + d.BCD == other.BCD && + d.Bus == other.Bus && + d.DeviceNumber == other.DeviceNumber && + d.Major == other.Major && + d.Minor == other.Minor && + d.BDeviceClass == other.BDeviceClass && + d.BDeviceSubClass == other.BDeviceSubClass && + d.BDeviceProtocol == other.BDeviceProtocol && + d.BConfigurationValue == other.BConfigurationValue && + d.BNumConfigurations == other.BNumConfigurations && + d.BNumInterfaces == other.BNumInterfaces && + slices.Equal(d.Interfaces, other.Interfaces) +} + +func (d *Device) Validate() error { + if d.VendorID == 0 { + return fmt.Errorf("VendorID is required") + } + if d.ProductID == 0 { + return fmt.Errorf("ProductID is required") + } + if d.Bus == 0 { + return fmt.Errorf("Bus is required") + } + if d.DeviceNumber == 0 { + return fmt.Errorf("DeviceNumber is required") + } + if d.DevicePath == "" { + return fmt.Errorf("DevicePath is required") + } + if d.Major == 0 { + return fmt.Errorf("Major is required") + } + if d.Minor == 0 { + return fmt.Errorf("Minor is required") + } + return nil +} + +func LoadDevice(path string) (device Device, err error) { + if !strings.HasPrefix(path, PathToUSBDevices) { + return device, fmt.Errorf("path %s is not a usb device", path) + } + + device.Path = path + device.BusID = filepath.Base(path) + + if err = parseSysUeventFile(path, &device); err != nil { + return + } + if err = parseSerial(path, &device); err != nil { + return + } + if err = parseManufacturer(path, &device); err != nil { + return + } + if err = parseProduct(path, &device); err != nil { + return + } + if err = parseBConfigurationValue(path, &device); err != nil { + return + } + if err = parseBNumConfigurations(path, &device); err != nil { + return + } + if err = parseBNumInterfaces(path, &device); err != nil { + return + } + if err = parseSpeed(path, &device); err != nil { + return + } + if err = parseSysUeventInterfaces(path, &device); err != nil { + return + } + + return +} + +func parseSysUeventFile(path string, device *Device) error { + // Example uevent file: + // MAJOR=189 + // MINOR=257 + // DEVNAME=bus/usb/003/002 + // DEVTYPE=usb_device + // DRIVER=usb + // PRODUCT=e39/f100/35d + // TYPE=0/0/0 + // BUSNUM=003 + // DEVNUM=002 + file, err := os.Open(filepath.Join(path, "uevent")) + if err != nil { + return fmt.Errorf("unable to open the file %s: %w", path, err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + values := strings.Split(line, "=") + if len(values) != 2 { + slog.Info("Skipping %s due not being key=value", slog.String("line", line)) + continue + } + switch values[0] { + case "MAJOR": + val, err := strconv.ParseUint(values[1], 10, 32) + if err != nil { + return fmt.Errorf("failed to parse MAJOR: %s", values[1]) + } + device.Major = val + case "MINOR": + val, err := strconv.ParseUint(values[1], 10, 32) + if err != nil { + return fmt.Errorf("failed to parse MINOR: %s", values[1]) + } + device.Minor = val + case "DEVNAME": + device.DevicePath = filepath.Join("/dev", values[1]) + case "DRIVER": + device.Driver = values[1] + case "PRODUCT": + products := strings.Split(values[1], "/") + if len(products) != 3 { + slog.Error("Failed to parse PRODUCT", slog.String("value", values[1]), slog.Any("err", err)) + return fmt.Errorf("failed to parse PRODUCT: %s", values[1]) + } + + val, err := strconv.ParseUint(products[0], 16, 32) + if err != nil { + return fmt.Errorf("failed to parse PRODUCT: %s", values[1]) + } + device.VendorID = uint16(val) + + val, err = strconv.ParseUint(products[1], 16, 32) + if err != nil { + return fmt.Errorf("failed to parse PRODUCT: %s", values[1]) + } + device.ProductID = uint16(val) + + val, err = strconv.ParseUint(products[2], 16, 32) + if err != nil { + return fmt.Errorf("failed to parse PRODUCT: %s", values[1]) + } + device.BCD = uint16(val) + case "TYPE": + types := strings.Split(values[1], "/") + if len(types) != 3 { + return fmt.Errorf("failed to parse TYPE: %s", values[1]) + } + val, err := strconv.ParseUint(types[0], 10, 8) + if err != nil { + return fmt.Errorf("failed to parse TYPE: %s", values[1]) + } + device.BDeviceClass = uint8(val) + device.IsHub = device.BDeviceClass == 9 // 09 = USB Hub class + + val, err = strconv.ParseUint(types[1], 10, 8) + if err != nil { + return fmt.Errorf("failed to parse TYPE: %s", values[1]) + } + device.BDeviceSubClass = uint8(val) + + val, err = strconv.ParseUint(types[2], 10, 8) + if err != nil { + return fmt.Errorf("failed to parse TYPE: %s", values[1]) + } + device.BDeviceProtocol = uint8(val) + case "BUSNUM": + val, err := strconv.ParseUint(values[1], 10, 32) + if err != nil { + return fmt.Errorf("failed to parse BUSNUM: %s", values[1]) + } + device.Bus = uint32(val) + case "DEVNUM": + val, err := strconv.ParseUint(values[1], 10, 32) + if err != nil { + return fmt.Errorf("failed to parse DEVNUM: %s", values[1]) + } + device.DeviceNumber = uint32(val) + default: + slog.Info("Skipping unhandled line", slog.String("line", line)) + } + } + return nil +} + +func parseSerial(path string, device *Device) error { + serial, err := parseStringValue(path, "serial") + if err != nil { + return err + } + device.Serial = serial + return nil +} + +func parseManufacturer(path string, device *Device) error { + manufacturer, err := parseStringValue(path, "manufacturer") + if err != nil { + return err + } + device.Manufacturer = manufacturer + return nil +} + +func parseProduct(path string, device *Device) error { + product, err := parseStringValue(path, "product") + if err != nil { + return err + } + device.Product = product + return nil +} + +func parseBConfigurationValue(path string, device *Device) error { + val, err := parseUintValue(path, "bConfigurationValue", 8, true) + if err != nil { + return err + } + device.BConfigurationValue = uint8(val) + return nil +} + +func parseBNumConfigurations(path string, device *Device) error { + val, err := parseUintValue(path, "bNumConfigurations", 8, false) + if err != nil { + return err + } + device.BNumConfigurations = uint8(val) + return nil +} + +func parseBNumInterfaces(path string, device *Device) error { + val, err := parseUintValue(path, "bNumInterfaces", 8, true) + if err != nil { + return err + } + device.BNumInterfaces = uint8(val) + return nil +} + +func parseSpeed(path string, device *Device) error { + val, err := parseUintValue(path, "speed", 32, false) + if err != nil { + return err + } + device.Speed = uint32(val) + return nil +} + +func parseSysUeventInterfaces(path string, device *Device) error { + // 3-2.1.1:1.0 + // | | | + // │ | |- bInterfaceNumber + // | |--- bConfigurationValue + // |----------- busID usb_device + + // path - /sys/bus/usb/devices/3-2.1.1 + // uevent path - /sys/bus/usb/devices/3-2.1.1:1.0/uevent + + if device.BConfigurationValue == 0 || device.BNumInterfaces == 0 { + device.Interfaces = nil + return nil + } + + var deviceInterfaces []DeviceInterface + + parent := filepath.Dir(path) + entries, err := os.ReadDir(parent) + if err != nil { + return fmt.Errorf("unable to read the directory %s: %w", path, err) + } + + prefix := fmt.Sprintf("%s:%d.", device.BusID, device.BConfigurationValue) + for _, entry := range entries { + if !entry.IsDir() { + continue + } + if !strings.HasPrefix(entry.Name(), prefix) { + continue + } + interfaceNumberStr := strings.TrimPrefix(entry.Name(), prefix) + _, err := strconv.Atoi(interfaceNumberStr) + if err != nil { + // not a valid interface number + continue + } + + ueventPath := filepath.Join(path, entry.Name(), "uevent") + file, err := os.Open(ueventPath) + if err != nil { + return fmt.Errorf("unable to open the file %s: %w", ueventPath, err) + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + values := strings.Split(line, "=") + if len(values) != 2 { + slog.Info("Skipping %s due not being key=value", slog.String("line", line)) + continue + } + switch values[0] { + case "INTERFACE": + deviceInterface := DeviceInterface{} + + interfaces := strings.Split(values[1], "/") + if len(interfaces) != 3 { + return fmt.Errorf("failed to parse INTERFACE: %s", values[1]) + } + val, err := strconv.ParseUint(interfaces[0], 10, 8) + if err != nil { + return fmt.Errorf("failed to parse INTERFACE: %s", values[1]) + } + deviceInterface.BInterfaceClass = uint8(val) + + val, err = strconv.ParseUint(interfaces[1], 10, 8) + if err != nil { + return fmt.Errorf("failed to parse INTERFACE: %s", values[1]) + } + deviceInterface.BInterfaceSubClass = uint8(val) + + val, err = strconv.ParseUint(interfaces[2], 10, 8) + if err != nil { + return fmt.Errorf("failed to parse INTERFACE: %s", values[1]) + } + deviceInterface.BInterfaceProtocol = uint8(val) + + deviceInterfaces = append(deviceInterfaces, deviceInterface) + + break + } + } + } + + device.Interfaces = deviceInterfaces + + return nil +} + +func parseStringValue(path, valueName string) (string, error) { + valuePath := filepath.Join(path, valueName) + data, err := os.ReadFile(valuePath) + if err != nil { + return "", fmt.Errorf("failed to read %s: %w", valuePath, err) + } + + value := strings.TrimSpace(string(data)) + if value == "" { + return "", fmt.Errorf("invalid %s: empty", valueName) + } + + return value, nil +} + +func parseUintValue(path, valueName string, bitSize int, ignoreNotExist bool) (uint64, error) { + valuePath := filepath.Join(path, valueName) + data, err := os.ReadFile(valuePath) + if err != nil { + return 0, fmt.Errorf("failed to read %s: %w", path, err) + } + + value := strings.TrimSpace(string(data)) + if value == "" && ignoreNotExist { + return 0, nil + } + + val, err := strconv.ParseUint(value, 10, bitSize) + if err != nil { + return 0, fmt.Errorf("failed to parse %s: %w", valueName, err) + } + + if val == 0 { + return 0, fmt.Errorf("invalid %s: 0", valueName) + } + + return val, nil +} diff --git a/images/virtualization-dra/test/pod-with-template-3.yaml b/images/virtualization-dra/test/pod-with-template-3.yaml new file mode 100644 index 0000000000..0c4c603333 --- /dev/null +++ b/images/virtualization-dra/test/pod-with-template-3.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: test-pod-with-usb-template-3 + namespace: usb +spec: + containers: + - name: test-container + image: nicolaka/netshoot:latest + command: ["sleep", "3600"] + resources: + claims: + - name: usb-device + resourceClaims: + - name: usb-device + resourceClaimTemplateName: usb-product-0951-vendor-0104-template diff --git a/images/virtualization-dra/test/resourceclaim-template-2.yaml b/images/virtualization-dra/test/resourceclaim-template-2.yaml new file mode 100644 index 0000000000..e51ab2ab84 --- /dev/null +++ b/images/virtualization-dra/test/resourceclaim-template-2.yaml @@ -0,0 +1,17 @@ +apiVersion: resource.k8s.io/v1beta1 +kind: ResourceClaimTemplate +metadata: + name: usb-product-0951-vendor-0104-template +spec: + spec: + devices: + requests: + - name: req-0 + allocationMode: "ExactCount" + count: 1 + deviceClassName: usb-devices.virtualization.deckhouse.io + selectors: + - cel: + expression: |- + device.attributes["virtualization-dra"].productID == "0104" && + device.attributes["virtualization-dra"].vendorID == "0951" diff --git a/images/virtualization-dra/werf.inc.yaml b/images/virtualization-dra/werf.inc.yaml index 919a0adcbb..f3549fa388 100644 --- a/images/virtualization-dra/werf.inc.yaml +++ b/images/virtualization-dra/werf.inc.yaml @@ -1,7 +1,7 @@ --- image: {{ .ModuleNamePrefix }}{{ .ImageName }}-builder final: false -fromImage: {{ eq $.SVACE_ENABLED "false" | ternary "builder/golang-bookworm-1.24" "builder/golang-alt-svace-1.24.9" }} +fromImage: {{ eq $.SVACE_ENABLED "false" | ternary "builder/golang-bookworm-1.25" "builder/golang-alt-svace-1.25.4" }} git: - add: {{ .ModuleDir }}/images/{{ .ImageName }} to: /src/images/virtualization-dra @@ -39,3 +39,12 @@ shell: {{- include "image-build.build" (set $ "BuildCommand" `go build -ldflags="-s -w" -v -o /out/virtualization-dra-plugin ./cmd/virtualization-dra-plugin`) | nindent 4 }} {{- end }} + - | + echo "Build virtualization-dra-usb-gateway binary" + {{- $_ := set $ "ProjectName" (list $.ImageName "virtualization-dra-usb-gateway" | join "/") }} + + {{- if eq $.DEBUG_COMPONENT "delve/virtualization-dra-usb-gateway" }} + go build -v -o /out/virtualization-dra-usb-gateway ./cmd/usb-gateway + {{- else }} + {{- include "image-build.build" (set $ "BuildCommand" `go build -ldflags="-s -w" -v -o /out/virtualization-dra-usb-gateway ./cmd/usb-gateway`) | nindent 4 }} + {{- end }} diff --git a/templates/virtualization-dra-usb-gateway/_helper.tpl b/templates/virtualization-dra-usb-gateway/_helper.tpl new file mode 100644 index 0000000000..26caf8ac46 --- /dev/null +++ b/templates/virtualization-dra-usb-gateway/_helper.tpl @@ -0,0 +1,5 @@ +{{- define "virtualization-dra-usb-gateway.isEnabled" -}} +{{- if eq (include "hasValidModuleConfig" .) "true" -}} +true +{{- end -}} +{{- end -}} diff --git a/templates/virtualization-dra-usb-gateway/daemonset.yaml b/templates/virtualization-dra-usb-gateway/daemonset.yaml new file mode 100644 index 0000000000..4c8c6a79d2 --- /dev/null +++ b/templates/virtualization-dra-usb-gateway/daemonset.yaml @@ -0,0 +1,126 @@ +{{- $priorityClassName := include "priorityClassName" . }} +{{- $delve := (include "delve" . | fromYaml) -}} +{{- define "virtualization-dra-usb-gateway_resources" }} +cpu: 10m +memory: 25Mi +{{- end }} + + +{{- if eq (include "virtualization-dra-usb-gateway.isEnabled" .) "true"}} + +{{- if (.Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} +--- +apiVersion: autoscaling.k8s.io/v1 +kind: VerticalPodAutoscaler +metadata: + name: virtualization-dra-usb-gateway + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra-usb-gateway" "workload-resource-policy.deckhouse.io" "every-node")) | nindent 2 }} +spec: + targetRef: + apiVersion: "apps/v1" + kind: DaemonSet + name: virtualization-dra-usb-gateway + updatePolicy: + updateMode: "Auto" + resourcePolicy: + containerPolicies: + - containerName: virtualization-dra-usb-gateway + minAllowed: + {{- include "virtualization-dra-usb-gateway_resources" . | nindent 8 }} + maxAllowed: + cpu: 20m + memory: 25Mi +{{- end }} + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: virtualization-dra-usb-gateway + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra-usb-gateway")) | nindent 2 }} +spec: + selector: + matchLabels: + app: virtualization-dra-usb-gateway + template: + metadata: + labels: + app: virtualization-dra-usb-gateway + spec: + {{- include "helm_lib_priority_class" (tuple . $priorityClassName) | nindent 6 }} + {{- include "helm_lib_tolerations" (tuple . "any-node") | nindent 6 }} + {{- include "helm_lib_module_pod_security_context_run_as_user_root" . | nindent 6 }} + imagePullSecrets: + - name: virtualization-module-registry + serviceAccountName: virtualization-dra-usb-gateway + dnsPolicy: ClusterFirstWithHostNet + nodeSelector: + kubernetes.io/os: linux + initContainers: + - name: virtualization-dra-usb-gateway-init + image: {{ include "helm_lib_module_image" (list . "virtualizationDraUsbGateway") }} + imagePullPolicy: "IfNotPresent" + args: ["init"] + securityContext: + privileged: false + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + add: + - SYS_MODULE + mounts: + - name: lib-modules + mountPath: /lib/modules + containers: + - name: virtualization-dra-usb-gateway + {{- include "helm_lib_module_container_security_context_privileged_read_only_root_filesystem" . | nindent 10 }} + image: {{ include "helm_lib_module_image" (list . "virtualizationDraUsbGateway") }} + imagePullPolicy: "IfNotPresent" + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + {{- if eq (include "moduleLogLevel" .) "debug" }} + - name: VERBOSITY + value: "10" + {{- end }} + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + resources: + requests: + {{- include "helm_lib_module_ephemeral_storage_only_logs" . | nindent 14 }} + {{- if not ( .Values.global.enabledModules | has "vertical-pod-autoscaler-crd") }} + {{- include "virtualization-dra-usb-gateway_resources" . | nindent 14 }} + {{- end }} + ports: + - containerPort: 51515 + name: health + protocol: TCP + {{- include "delvePorts" (list $delve "delve/virtualization-dra-usb-gateway") | nindent 12 }} + {{- if ne "delve/virtualization-dra-usb-gateway" ($delve | dig "debug" "component" "") }} + # TODO: add readinessProbe and livenessProbe + #readinessProbe: + #livenessProbe: + {{- end }} + volumeMounts: + - name: sys + mountPath: /sys + - name: var-run + mountPath: /var/run + volumes: + - name: sys + hostPath: + path: /sys + - name: var-run + hostPath: + path: /var/run + - name: lib-modules + hostPath: + path: /lib/modules +{{- end }} diff --git a/templates/virtualization-dra-usb-gateway/rbac-for-us.yaml b/templates/virtualization-dra-usb-gateway/rbac-for-us.yaml new file mode 100644 index 0000000000..8b2b2097f4 --- /dev/null +++ b/templates/virtualization-dra-usb-gateway/rbac-for-us.yaml @@ -0,0 +1,34 @@ +{{- if eq (include "virtualization-dra-usb-gateway.isEnabled" .) "true"}} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: virtualization-dra-usb-gateway + namespace: d8-{{ .Chart.Name }} + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra-usb-gateway")) | nindent 2 }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:virtualization-dra-usb-gateway + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra-usb-gateway")) | nindent 2 }} +rules: + # TODO: fix me + - apiGroups: ["*"] + resources: ["*"] + verbs: ["*"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: d8:{{ .Chart.Name }}:virtualization-dra-usb-gateway + {{- include "helm_lib_module_labels" (list . (dict "app" "virtualization-dra-usb-gateway")) | nindent 2 }} +subjects: + - kind: ServiceAccount + name: virtualization-dra-usb-gateway + namespace: d8-{{ .Chart.Name }} +roleRef: + kind: ClusterRole + name: d8:{{ .Chart.Name }}:virtualization-dra-usb-gateway + apiGroup: rbac.authorization.k8s.io +{{- end }} From e0cef851862769b09f8d84ac606cb8640842f3bd Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Mon, 29 Dec 2025 13:09:56 +0300 Subject: [PATCH 02/37] add Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/Taskfile.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/images/virtualization-dra/Taskfile.yaml b/images/virtualization-dra/Taskfile.yaml index 145a70c862..c0c37e7377 100644 --- a/images/virtualization-dra/Taskfile.yaml +++ b/images/virtualization-dra/Taskfile.yaml @@ -58,7 +58,6 @@ tasks: - | golangci-lint run - build:go-usbip: desc: "Build go-usbip binary" cmds: @@ -68,4 +67,3 @@ tasks: desc: "Generate API code" cmds: - hack/update-codegen.sh - From ed014f3bb60ef721698ee8a9e46741fb59c1b99c Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 12:57:26 +0300 Subject: [PATCH 03/37] fix templates Signed-off-by: Yaroslav Borbat --- templates/virtualization-dra-usb-gateway/daemonset.yaml | 2 +- tools/kubeconform/fixtures/module-values.yaml | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/templates/virtualization-dra-usb-gateway/daemonset.yaml b/templates/virtualization-dra-usb-gateway/daemonset.yaml index 4c8c6a79d2..49e7d0e72e 100644 --- a/templates/virtualization-dra-usb-gateway/daemonset.yaml +++ b/templates/virtualization-dra-usb-gateway/daemonset.yaml @@ -71,7 +71,7 @@ spec: - ALL add: - SYS_MODULE - mounts: + volumeMounts: - name: lib-modules mountPath: /lib/modules containers: diff --git a/tools/kubeconform/fixtures/module-values.yaml b/tools/kubeconform/fixtures/module-values.yaml index c5588355ac..1763d1f2cd 100644 --- a/tools/kubeconform/fixtures/module-values.yaml +++ b/tools/kubeconform/fixtures/module-values.yaml @@ -331,7 +331,8 @@ global: virtualizationApi: sha256:0000000000000000000000000000000000000000000000000000000000000000 virtualizationController: sha256:0000000000000000000000000000000000000000000000000000000000000000 vmRouteForge: sha256:0000000000000000000000000000000000000000000000000000000000000000 - virtualizationDraPlugin: sha256:0000000000000000000000000000000000000000000000000000000000000000w + virtualizationDraPlugin: sha256:0000000000000000000000000000000000000000000000000000000000000000 + virtualizationDraUsbGateway: sha256:0000000000000000000000000000000000000000000000000000000000000000 registry: CA: "" address: some-registry.io From 01c21fccb9d954fae7bfd21c5bff18a282e4d737 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 13:33:12 +0300 Subject: [PATCH 04/37] fix json patch Signed-off-by: Yaroslav Borbat --- .../virtualization-dra/internal/usb-gateway/labeler/labeler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go b/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go index 2e36e3fdfc..6f8af80b21 100644 --- a/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go +++ b/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go @@ -58,7 +58,7 @@ func (l *genericLabeler) Label(ctx context.Context, name, namespace string, newL return err } - patch := []byte(fmt.Sprintf("[{'op': 'replace', 'path': '/metadata/labels', 'value': %s}]", value)) + patch := []byte(fmt.Sprintf(`[{"op": "replace", "path": "/metadata/labels", "value": %s}]`, string(value))) _, err = l.client.Resource(l.gvr).Namespace(namespace).Patch(ctx, name, types.JSONPatchType, patch, metav1.PatchOptions{}) return err From 190430ce5f162f6b0add7fab85e6a41462fb1c2c Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 13:37:49 +0300 Subject: [PATCH 05/37] fix load modules Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/pkg/modprobe/modprobe.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/images/virtualization-dra/pkg/modprobe/modprobe.go b/images/virtualization-dra/pkg/modprobe/modprobe.go index 838285a926..30b61127ed 100644 --- a/images/virtualization-dra/pkg/modprobe/modprobe.go +++ b/images/virtualization-dra/pkg/modprobe/modprobe.go @@ -17,7 +17,9 @@ limitations under the License. package modprobe import ( + "errors" "fmt" + "log/slog" "os" "path/filepath" @@ -51,8 +53,14 @@ func loadModule(path string) error { defer f.Close() if err = unix.FinitModule(int(f.Fd()), "", 0); err != nil { + if errors.Is(err, unix.EEXIST) { + slog.Info("Module already loaded", slog.String("path", path)) + return nil + } return fmt.Errorf("finit_module %s: %w", path, err) } + slog.Info("Module loaded", slog.String("path", path)) + return nil } From fbd1e03b87691a315215aafa01fadcc694979bab Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 13:49:36 +0300 Subject: [PATCH 06/37] fix map nil panic Signed-off-by: Yaroslav Borbat --- .../internal/usb-gateway/informer/informer.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/images/virtualization-dra/internal/usb-gateway/informer/informer.go b/images/virtualization-dra/internal/usb-gateway/informer/informer.go index 3d32df4fb6..ea703ba826 100644 --- a/images/virtualization-dra/internal/usb-gateway/informer/informer.go +++ b/images/virtualization-dra/internal/usb-gateway/informer/informer.go @@ -44,9 +44,10 @@ func NewFactory(clientSet *kubernetes.Clientset, resync *time.Duration) *Factory } return &Factory{ - clientSet: clientSet, - defaultResync: defaultResync, - informers: make(map[string]cache.SharedIndexInformer), + clientSet: clientSet, + defaultResync: defaultResync, + informers: make(map[string]cache.SharedIndexInformer), + startedInformers: make(map[string]struct{}), } } From ecbe61cd084ab58ff43ddbcbbc1e480df366059e Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 14:19:20 +0300 Subject: [PATCH 07/37] fix loading modules for fresh kernels Signed-off-by: Yaroslav Borbat --- .../pkg/modprobe/modprobe.go | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/images/virtualization-dra/pkg/modprobe/modprobe.go b/images/virtualization-dra/pkg/modprobe/modprobe.go index 30b61127ed..c4867c1e0d 100644 --- a/images/virtualization-dra/pkg/modprobe/modprobe.go +++ b/images/virtualization-dra/pkg/modprobe/modprobe.go @@ -22,6 +22,7 @@ import ( "log/slog" "os" "path/filepath" + "strings" "golang.org/x/sys/unix" ) @@ -36,7 +37,23 @@ func LoadModules(modules []string) error { base := filepath.Join("/lib/modules", kernel) for _, m := range modules { - path := filepath.Join(base, m) + + var path string + if strings.HasSuffix(m, ".zst") { + path = filepath.Join(base, m) + } else { + pathZst := filepath.Join(base, m+".zst") + pathKo := filepath.Join(base, m) + + if _, err := os.Stat(pathZst); err == nil { + path = pathZst + } else if _, err := os.Stat(pathKo); err == nil { + path = pathKo + } else { + return fmt.Errorf("module file not found: %s or %s", pathKo, pathZst) + } + } + if err := loadModule(path); err != nil { return fmt.Errorf("load module %s: %w", path, err) } From 1e8400966f13f67cb645364875861fff3a881fa4 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 14:46:11 +0300 Subject: [PATCH 08/37] fix loading modules for fresh kernels Signed-off-by: Yaroslav Borbat --- .../cmd/usb-gateway/app/init.go | 18 ++++- .../pkg/modprobe/modprobe.go | 66 ++++++++++--------- 2 files changed, 51 insertions(+), 33 deletions(-) diff --git a/images/virtualization-dra/cmd/usb-gateway/app/init.go b/images/virtualization-dra/cmd/usb-gateway/app/init.go index a435e880dd..ebba3886ff 100644 --- a/images/virtualization-dra/cmd/usb-gateway/app/init.go +++ b/images/virtualization-dra/cmd/usb-gateway/app/init.go @@ -18,6 +18,7 @@ package app import ( "fmt" + "path/filepath" "github.com/spf13/cobra" @@ -39,12 +40,23 @@ func NewInitCommand() *cobra.Command { type initOptions struct{} func (o *initOptions) Run(_ *cobra.Command, _ []string) error { + kernelRelease, err := modprobe.KernelRelease() + if err != nil { + return fmt.Errorf("failed to get kernel release: %w", err) + } + modules := []string{ - "kernel/drivers/usb/usbip/usbip-core.ko", - "kernel/drivers/usb/usbip/vhci-hcd.ko", + filepath.Join("/lib/modules", kernelRelease, "kernel/drivers/usb/usbip/usbip-core.ko"), + filepath.Join("/lib/modules", kernelRelease, "kernel/drivers/usb/usbip/vhci-hcd.ko"), + } + + if modprobe.KernelSupportsZst(kernelRelease) { + for i := range modules { + modules[i] += ".zst" + } } - if err := modprobe.LoadModules(modules); err != nil { + if err := modprobe.LoadModules(modules...); err != nil { return fmt.Errorf("failed to load modules: %w", err) } diff --git a/images/virtualization-dra/pkg/modprobe/modprobe.go b/images/virtualization-dra/pkg/modprobe/modprobe.go index c4867c1e0d..67eb61ffb9 100644 --- a/images/virtualization-dra/pkg/modprobe/modprobe.go +++ b/images/virtualization-dra/pkg/modprobe/modprobe.go @@ -21,41 +21,16 @@ import ( "fmt" "log/slog" "os" - "path/filepath" + "strconv" "strings" "golang.org/x/sys/unix" ) -func LoadModules(modules []string) error { - var uts unix.Utsname - if err := unix.Uname(&uts); err != nil { - return fmt.Errorf("uname: %w", err) - } - - kernel := unix.ByteSliceToString(uts.Release[:]) - base := filepath.Join("/lib/modules", kernel) - - for _, m := range modules { - - var path string - if strings.HasSuffix(m, ".zst") { - path = filepath.Join(base, m) - } else { - pathZst := filepath.Join(base, m+".zst") - pathKo := filepath.Join(base, m) - - if _, err := os.Stat(pathZst); err == nil { - path = pathZst - } else if _, err := os.Stat(pathKo); err == nil { - path = pathKo - } else { - return fmt.Errorf("module file not found: %s or %s", pathKo, pathZst) - } - } - - if err := loadModule(path); err != nil { - return fmt.Errorf("load module %s: %w", path, err) +func LoadModules(modules ...string) error { + for _, module := range modules { + if err := loadModule(module); err != nil { + return fmt.Errorf("load module %s: %w", module, err) } } @@ -81,3 +56,34 @@ func loadModule(path string) error { return nil } + +func KernelRelease() (string, error) { + var uts unix.Utsname + if err := unix.Uname(&uts); err != nil { + return "", fmt.Errorf("uname: %w", err) + } + return unix.ByteSliceToString(uts.Release[:]), nil + +} + +func KernelSupportsZst(release string) bool { + parts := strings.Split(release, ".") + if len(parts) < 2 { + return false + } + + major, err1 := strconv.Atoi(parts[0]) + minor, err2 := strconv.Atoi(parts[1]) + if err1 != nil || err2 != nil { + return false + } + + // ZST is supported since 5.16 + if major > 5 { + return true + } + if major == 5 && minor >= 16 { + return true + } + return false +} From 07864fc469547cebbe1ee146dd5a2a14cb895942 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 15:40:25 +0300 Subject: [PATCH 09/37] fix loading modules for fresh kernels Signed-off-by: Yaroslav Borbat --- .../pkg/modprobe/modprobe.go | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/images/virtualization-dra/pkg/modprobe/modprobe.go b/images/virtualization-dra/pkg/modprobe/modprobe.go index 67eb61ffb9..62f3893434 100644 --- a/images/virtualization-dra/pkg/modprobe/modprobe.go +++ b/images/virtualization-dra/pkg/modprobe/modprobe.go @@ -19,11 +19,14 @@ package modprobe import ( "errors" "fmt" + "io" "log/slog" "os" + "path/filepath" "strconv" "strings" + "github.com/klauspost/compress/zstd" "golang.org/x/sys/unix" ) @@ -38,6 +41,15 @@ func LoadModules(modules ...string) error { } func loadModule(path string) error { + if strings.HasSuffix(path, ".zst") { + uncompressedPath, err := uncompressModuleToTmp(path) + if err != nil { + return fmt.Errorf("uncompress module %s: %w", path, err) + } + defer os.Remove(uncompressedPath) + path = uncompressedPath + } + f, err := os.Open(path) if err != nil { return fmt.Errorf("open %s: %w", path, err) @@ -57,6 +69,33 @@ func loadModule(path string) error { return nil } +func uncompressModuleToTmp(path string) (string, error) { + pattern := filepath.Base(path) + "-*" + uncompress, err := os.CreateTemp("", pattern) + if err != nil { + return "", err + } + defer uncompress.Close() + + in, err := os.Open(path) + if err != nil { + return "", err + } + defer in.Close() + + decoder, err := zstd.NewReader(in) + if err != nil { + return "", err + } + defer decoder.Close() + + if _, err := io.Copy(uncompress, decoder); err != nil { + return "", err + } + + return uncompress.Name(), nil +} + func KernelRelease() (string, error) { var uts unix.Utsname if err := unix.Uname(&uts); err != nil { From cb37f84f270f10e35dea85a9312017a9bc21ac2c Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 15:41:00 +0300 Subject: [PATCH 10/37] go tidy Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/images/virtualization-dra/go.mod b/images/virtualization-dra/go.mod index 9bac9000d9..705cf01feb 100644 --- a/images/virtualization-dra/go.mod +++ b/images/virtualization-dra/go.mod @@ -13,6 +13,7 @@ require ( github.com/fsnotify/fsnotify v1.5.1 github.com/go-logr/logr v1.4.3 github.com/godbus/dbus/v5 v5.2.0 + github.com/klauspost/compress v1.18.0 github.com/onsi/ginkgo/v2 v2.27.2 github.com/onsi/gomega v1.38.2 github.com/spf13/cobra v1.10.1 From 696ff19230b805824ef89088f46e880a80a15cf8 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 15:56:42 +0300 Subject: [PATCH 11/37] fix pointer resourceSlice Signed-off-by: Yaroslav Borbat --- .../internal/usb-gateway/controller/resourceclaim/controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go index 85ad158e98..bf6c216a4c 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -520,7 +520,7 @@ func (c *Controller) getVirtualizationDraResourceSlices() ([]resourcev1beta1.Res } var slices []resourcev1beta1.ResourceSlice for _, obj := range slicesObj { - slice, ok := obj.(resourcev1beta1.ResourceSlice) + slice, ok := obj.(*resourcev1beta1.ResourceSlice) if !ok { return nil, fmt.Errorf("unexpected type of resource slice: %T", obj) } From d40f7e883841bd185ee979e07fb31027874ac812 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 16:20:15 +0300 Subject: [PATCH 12/37] enable USBGateway featureGate Signed-off-by: Yaroslav Borbat --- templates/virtualization-dra/daemonset.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/templates/virtualization-dra/daemonset.yaml b/templates/virtualization-dra/daemonset.yaml index 47c1e7543f..8300a6b418 100644 --- a/templates/virtualization-dra/daemonset.yaml +++ b/templates/virtualization-dra/daemonset.yaml @@ -63,6 +63,8 @@ spec: {{- include "helm_lib_module_container_security_context_privileged_read_only_root_filesystem" . | nindent 10 }} image: {{ include "helm_lib_module_image" (list . "virtualizationDraPlugin") }} imagePullPolicy: "IfNotPresent" + args: + - --feature-gates=USBGateway=true env: - name: NODE_NAME valueFrom: From e395de6ff64fcb84cf1cdced22ec52ffff149302 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 16:43:27 +0300 Subject: [PATCH 13/37] fix dra feature-gates flags Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/cmd/usb-gateway/app/app.go | 7 +------ .../cmd/virtualization-dra-plugin/app/app.go | 7 ++++++- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/images/virtualization-dra/cmd/usb-gateway/app/app.go b/images/virtualization-dra/cmd/usb-gateway/app/app.go index 0bd715c0df..4f0a92dcf4 100644 --- a/images/virtualization-dra/cmd/usb-gateway/app/app.go +++ b/images/virtualization-dra/cmd/usb-gateway/app/app.go @@ -29,7 +29,6 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/component-base/cli/flag" - "github.com/deckhouse/virtualization-dra/internal/featuregates" "github.com/deckhouse/virtualization-dra/internal/usb-gateway/controller/resourceclaim" "github.com/deckhouse/virtualization-dra/internal/usb-gateway/informer" "github.com/deckhouse/virtualization-dra/internal/usb-gateway/prepare" @@ -70,8 +69,7 @@ func NewUSBGatewayCommand() *cobra.Command { func newUsbOptions() *usbOptions { return &usbOptions{ - Logging: &logger.Options{}, - featureGates: featuregates.AddFlags, + Logging: &logger.Options{}, } } @@ -82,7 +80,6 @@ type usbOptions struct { USBIPPort int USBResyncPeriod time.Duration Logging *logger.Options - featureGates featuregates.AddFlagsFunc } func (o *usbOptions) NamedFlags() (fs flag.NamedFlagSets) { @@ -95,8 +92,6 @@ func (o *usbOptions) NamedFlags() (fs flag.NamedFlagSets) { o.Logging.AddFlags(fs.FlagSet("logging")) - o.featureGates(fs.FlagSet("feature-gates")) - return fs } diff --git a/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go b/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go index 088c76c204..a7a69110b1 100644 --- a/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go +++ b/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go @@ -29,6 +29,7 @@ import ( "k8s.io/component-base/cli/flag" "github.com/deckhouse/virtualization-dra/internal/cdi" + "github.com/deckhouse/virtualization-dra/internal/featuregates" "github.com/deckhouse/virtualization-dra/internal/plugin" "github.com/deckhouse/virtualization-dra/internal/usb" "github.com/deckhouse/virtualization-dra/pkg/logger" @@ -79,6 +80,7 @@ func newDraOptions() *draOptions { HealthzPort: 51515, USBResyncPeriod: usb.DefaultResyncPeriod, Logging: &logger.Options{}, + featureGates: featuregates.AddFlags, } if healthzPort := os.Getenv("HEALTHZ_PORT"); healthzPort != "" { @@ -101,7 +103,8 @@ type draOptions struct { HealthzPort int USBResyncPeriod time.Duration - Logging *logger.Options + Logging *logger.Options + featureGates featuregates.AddFlagsFunc } func (o *draOptions) NamedFlags() (fs flag.NamedFlagSets) { @@ -117,6 +120,8 @@ func (o *draOptions) NamedFlags() (fs flag.NamedFlagSets) { o.Logging.AddFlags(fs.FlagSet("logging")) + o.featureGates(fs.FlagSet("feature-gates")) + return fs } From 71b9c976fa81fbb9249fa53c886740bb876bdc5f Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 17:32:16 +0300 Subject: [PATCH 14/37] support NodeSelector in ResourceSlice Signed-off-by: Yaroslav Borbat --- .../virtualization-dra/internal/plugin/driver.go | 15 ++++++++++----- images/virtualization-dra/internal/usb/convert.go | 1 + 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/images/virtualization-dra/internal/plugin/driver.go b/images/virtualization-dra/internal/plugin/driver.go index a8605d29a9..458d37b1ba 100644 --- a/images/virtualization-dra/internal/plugin/driver.go +++ b/images/virtualization-dra/internal/plugin/driver.go @@ -30,8 +30,10 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/dynamic-resource-allocation/kubeletplugin" "k8s.io/dynamic-resource-allocation/resourceslice" + "k8s.io/utils/ptr" "github.com/deckhouse/virtualization-dra/internal/common" + "github.com/deckhouse/virtualization-dra/internal/featuregates" ) const DriverName = common.VirtualizationDraPluginName @@ -182,14 +184,17 @@ func (d *Driver) startPublisher(ctx context.Context) { } func (d *Driver) makeResources(devices []resourceapi.Device) resourceslice.DriverResources { + slice := resourceslice.Slice{ + Devices: devices, + } + if featuregates.Default().USBGatewayEnabled() { + slice.PerDeviceNodeSelection = ptr.To(true) + } + return resourceslice.DriverResources{ Pools: map[string]resourceslice.Pool{ d.nodeName: { - Slices: []resourceslice.Slice{ - { - Devices: devices, - }, - }, + Slices: []resourceslice.Slice{slice}, }, }, } diff --git a/images/virtualization-dra/internal/usb/convert.go b/images/virtualization-dra/internal/usb/convert.go index 8cbeb53f9a..deb52202a5 100644 --- a/images/virtualization-dra/internal/usb/convert.go +++ b/images/virtualization-dra/internal/usb/convert.go @@ -82,6 +82,7 @@ func convertToAPIDevice(usbDevice Device, nodeName string) *resourceapi.Device { } if featuregates.Default().USBGatewayEnabled() { + // TODO: need pr to deckhouse for enable DRAPartitionableDevices feature gate on ApiServer device.NodeSelector = &corev1.NodeSelector{ NodeSelectorTerms: []corev1.NodeSelectorTerm{ { From 3b01c50af148538ccecdf2ae271b04a80de738f6 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 17:42:57 +0300 Subject: [PATCH 15/37] fix nodeSelector operator Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/internal/usb/convert.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virtualization-dra/internal/usb/convert.go b/images/virtualization-dra/internal/usb/convert.go index deb52202a5..542c8b11bf 100644 --- a/images/virtualization-dra/internal/usb/convert.go +++ b/images/virtualization-dra/internal/usb/convert.go @@ -89,7 +89,7 @@ func convertToAPIDevice(usbDevice Device, nodeName string) *resourceapi.Device { MatchExpressions: []corev1.NodeSelectorRequirement{ { Key: common.USBGatewayLabel, - Operator: corev1.NodeSelectorOpExists, + Operator: corev1.NodeSelectorOpIn, Values: []string{"true"}, }, }, From 78b0d9cbf3cfef5a242731c7070a78188da03dd8 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 18:05:01 +0300 Subject: [PATCH 16/37] fix poolname Signed-off-by: Yaroslav Borbat --- images/virtualization-dra-plugin/debug/dlv.Dockerfile | 5 +++-- images/virtualization-dra/internal/plugin/driver.go | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/images/virtualization-dra-plugin/debug/dlv.Dockerfile b/images/virtualization-dra-plugin/debug/dlv.Dockerfile index 9e85ba1d99..5ee5803b5f 100644 --- a/images/virtualization-dra-plugin/debug/dlv.Dockerfile +++ b/images/virtualization-dra-plugin/debug/dlv.Dockerfile @@ -1,4 +1,4 @@ -FROM golang:1.24.7-bookworm@sha256:2c5f7a0c252a17cf6aa30ddee15caa0f485ee29410a6ea64cddb62eea2b07bdf AS builder +FROM golang:1.25-bookworm@sha256:019c22232e57fda8ded2b10a8f201989e839f3d3f962d4931375069bbb927e03 AS builder ARG TARGETOS ARG TARGETARCH @@ -13,13 +13,14 @@ RUN go mod download COPY ./images/virtualization-dra/cmd /app/images/virtualization-dra/cmd COPY ./images/virtualization-dra/internal /app/images/virtualization-dra/internal COPY ./images/virtualization-dra/pkg /app/images/virtualization-dra/pkg +COPY ./images/virtualization-dra/api /app/images/virtualization-dra/api ENV GO111MODULE=on ENV GOOS=${TARGETOS:-linux} ENV GOARCH=${TARGETARCH:-amd64} ENV CGO_ENABLED=0 -RUN go build -tags EE -gcflags "all=-N -l" -a -o virtualization-dra-plugin ./cmd/virtualization-dra-plugin +RUN go build -gcflags "all=-N -l" -a -o virtualization-dra-plugin ./cmd/virtualization-dra-plugin/main.go FROM busybox:1.36.1-glibc diff --git a/images/virtualization-dra/internal/plugin/driver.go b/images/virtualization-dra/internal/plugin/driver.go index 458d37b1ba..1bc5773c8c 100644 --- a/images/virtualization-dra/internal/plugin/driver.go +++ b/images/virtualization-dra/internal/plugin/driver.go @@ -187,13 +187,16 @@ func (d *Driver) makeResources(devices []resourceapi.Device) resourceslice.Drive slice := resourceslice.Slice{ Devices: devices, } + poolName := d.nodeName + if featuregates.Default().USBGatewayEnabled() { slice.PerDeviceNodeSelection = ptr.To(true) + poolName = "global" } return resourceslice.DriverResources{ Pools: map[string]resourceslice.Pool{ - d.nodeName: { + poolName: { Slices: []resourceslice.Slice{slice}, }, }, From afa92710f9b45933a77a04ef791bd76af3841af2 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Tue, 13 Jan 2026 20:59:08 +0300 Subject: [PATCH 17/37] add new publisher Signed-off-by: Yaroslav Borbat --- .../internal/plugin/driver.go | 34 ++----- .../internal/plugin/interfaces.go | 3 +- .../internal/plugin/publish.go | 98 +++++++++++++++++++ .../virtualization-dra/internal/usb/store.go | 29 +++++- 4 files changed, 132 insertions(+), 32 deletions(-) create mode 100644 images/virtualization-dra/internal/plugin/publish.go diff --git a/images/virtualization-dra/internal/plugin/driver.go b/images/virtualization-dra/internal/plugin/driver.go index 1bc5773c8c..d7e58a14c7 100644 --- a/images/virtualization-dra/internal/plugin/driver.go +++ b/images/virtualization-dra/internal/plugin/driver.go @@ -29,11 +29,8 @@ import ( utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/client-go/kubernetes" "k8s.io/dynamic-resource-allocation/kubeletplugin" - "k8s.io/dynamic-resource-allocation/resourceslice" - "k8s.io/utils/ptr" "github.com/deckhouse/virtualization-dra/internal/common" - "github.com/deckhouse/virtualization-dra/internal/featuregates" ) const DriverName = common.VirtualizationDraPluginName @@ -54,6 +51,7 @@ type Driver struct { allocator Allocator log *slog.Logger + publisher resourcePublisher helper *kubeletplugin.Helper pluginCtx context.Context pluginCancel context.CancelCauseFunc @@ -64,6 +62,8 @@ func (d *Driver) Start(ctx context.Context) error { d.pluginCtx = ctx d.pluginCancel = cancel + d.publisher = newNonOwnerPublisher(ctx, d.kubeClient, d.HandleError) + log.Info("Starting dra plugin") helper, err := kubeletplugin.Start( ctx, @@ -92,6 +92,7 @@ func (d *Driver) Wait() { } func (d *Driver) Shutdown() { + d.publisher.Stop() if d.helper != nil { d.log.Info("Stopping dra plugin") d.helper.Stop() @@ -171,10 +172,9 @@ func (d *Driver) startPublisher(ctx context.Context) { select { case <-ctx.Done(): return - case devices := <-ch: - d.log.Info("Publishing devices", slog.Any("devices", devices)) - resources := d.makeResources(devices) - err := d.helper.PublishResources(ctx, resources) + case resources := <-ch: + d.log.Info("Publishing devices", slog.Any("resources", resources)) + err := d.publisher.PublishResources(ctx, resources) if err != nil { d.log.Error("Failed to publish devices", slog.Any("err", err)) } @@ -182,23 +182,3 @@ func (d *Driver) startPublisher(ctx context.Context) { } }() } - -func (d *Driver) makeResources(devices []resourceapi.Device) resourceslice.DriverResources { - slice := resourceslice.Slice{ - Devices: devices, - } - poolName := d.nodeName - - if featuregates.Default().USBGatewayEnabled() { - slice.PerDeviceNodeSelection = ptr.To(true) - poolName = "global" - } - - return resourceslice.DriverResources{ - Pools: map[string]resourceslice.Pool{ - poolName: { - Slices: []resourceslice.Slice{slice}, - }, - }, - } -} diff --git a/images/virtualization-dra/internal/plugin/interfaces.go b/images/virtualization-dra/internal/plugin/interfaces.go index 4ef9d1bf9d..3f757fc3a5 100644 --- a/images/virtualization-dra/internal/plugin/interfaces.go +++ b/images/virtualization-dra/internal/plugin/interfaces.go @@ -22,11 +22,12 @@ import ( "github.com/containerd/nri/pkg/api" resourceapi "k8s.io/api/resource/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/dynamic-resource-allocation/resourceslice" drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1beta1" ) type Allocator interface { - UpdateChannel() chan []resourceapi.Device + UpdateChannel() chan resourceslice.DriverResources Prepare(ctx context.Context, claim *resourceapi.ResourceClaim) ([]*drapbv1.Device, error) Unprepare(ctx context.Context, claimUID types.UID) error Synchronize(ctx context.Context, pods []*api.PodSandbox, containers []*api.Container) ([]*api.ContainerUpdate, error) diff --git a/images/virtualization-dra/internal/plugin/publish.go b/images/virtualization-dra/internal/plugin/publish.go new file mode 100644 index 0000000000..ecb338b75d --- /dev/null +++ b/images/virtualization-dra/internal/plugin/publish.go @@ -0,0 +1,98 @@ +package plugin + +import ( + "context" + "errors" + "fmt" + "sync" + + "k8s.io/client-go/kubernetes" + "k8s.io/dynamic-resource-allocation/kubeletplugin" + "k8s.io/dynamic-resource-allocation/resourceslice" + "k8s.io/klog/v2" +) + +type resourcePublisher interface { + PublishResources(ctx context.Context, resources resourceslice.DriverResources) error + Stop() +} +type errorHandler func(ctx context.Context, err error, msg string) + +func newNonOwnerPublisher(ctx context.Context, kubeClient kubernetes.Interface, errorHandler errorHandler) resourcePublisher { + ctx, cancel := context.WithCancelCause(ctx) + return &nonOwnerPublisher{ + driverName: DriverName, + kubeClient: kubeClient, + errorHandler: errorHandler, + backgroundCtx: ctx, + cancel: cancel, + } +} + +type nonOwnerPublisher struct { + driverName string + kubeClient kubernetes.Interface + backgroundCtx context.Context + cancel func(cause error) + errorHandler errorHandler + + mutex sync.Mutex + resourceSliceController *resourceslice.Controller +} + +func (p *nonOwnerPublisher) PublishResources(_ context.Context, resources resourceslice.DriverResources) error { + p.mutex.Lock() + defer p.mutex.Unlock() + + driverResources := &resourceslice.DriverResources{ + Pools: resources.Pools, + } + + if p.resourceSliceController == nil { + // Start publishing the information. The controller is using + // our background context, not the one passed into this + // function, and thus is connected to the lifecycle of the + // plugin. + controllerCtx := p.backgroundCtx + controllerLogger := klog.FromContext(controllerCtx) + controllerLogger = klog.LoggerWithName(controllerLogger, "ResourceSlice controller") + controllerCtx = klog.NewContext(controllerCtx, controllerLogger) + var err error + if p.resourceSliceController, err = resourceslice.StartController(controllerCtx, + resourceslice.Options{ + DriverName: p.driverName, + KubeClient: p.kubeClient, + Resources: driverResources, + ErrorHandler: func(ctx context.Context, err error, msg string) { + // ResourceSlice publishing errors like dropped fields or + // invalid spec are not going to get resolved by retrying, + // but neither is restarting the process going to help + // -> all errors are recoverable. + p.errorHandler(ctx, recoverableError{error: err}, msg) + }, + }); err != nil { + return fmt.Errorf("start ResourceSlice controller: %w", err) + } + } else { + // Inform running controller about new information. + p.resourceSliceController.Update(driverResources) + } + + return nil +} + +func (p *nonOwnerPublisher) Stop() { + if p == nil { + return + } + p.cancel(errors.New("nonOwnerPublisher was stopped")) +} + +type recoverableError struct { + error +} + +var _ error = recoverableError{} + +func (err recoverableError) Is(other error) bool { return other == kubeletplugin.ErrRecoverable } +func (err recoverableError) Unwrap() error { return err.error } diff --git a/images/virtualization-dra/internal/usb/store.go b/images/virtualization-dra/internal/usb/store.go index aa658170d5..7190f29200 100644 --- a/images/virtualization-dra/internal/usb/store.go +++ b/images/virtualization-dra/internal/usb/store.go @@ -29,6 +29,7 @@ import ( resourceapi "k8s.io/api/resource/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + "k8s.io/dynamic-resource-allocation/resourceslice" drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1beta1" "k8s.io/utils/ptr" cdiapi "tags.cncf.io/container-device-interface/pkg/cdi" @@ -57,7 +58,7 @@ func NewAllocationStore(nodeName, devicesPath string, resyncPeriod time.Duration resyncPeriod: resyncPeriod, cdi: cdiManager, log: log.With(slog.String("component", "usb-allocation-store")), - updateChannel: make(chan []resourceapi.Device, 2), + updateChannel: make(chan resourceslice.DriverResources, 2), discoverPluggedUSBDevices: NewDeviceSet(), allocatableDevices: make(map[string]resourceapi.Device), allocatedDevices: set.New[string](), @@ -85,7 +86,7 @@ type AllocationStore struct { monitor *monitor - updateChannel chan []resourceapi.Device + updateChannel chan resourceslice.DriverResources mu sync.RWMutex discoverPluggedUSBDevices *DeviceSet @@ -119,7 +120,7 @@ func (s *AllocationStore) sync() error { s.allocatableDevices = allocatableDevicesByName - s.updateChannel <- allocatableDevices + s.updateChannel <- s.makeResources(allocatableDevices) return nil } @@ -159,7 +160,7 @@ func (s *AllocationStore) Start(ctx context.Context) error { return nil } -func (s *AllocationStore) UpdateChannel() chan []resourceapi.Device { +func (s *AllocationStore) UpdateChannel() chan resourceslice.DriverResources { return s.updateChannel } @@ -417,3 +418,23 @@ func parseDraEnvToClaimAllocations(envs []string) (map[types.UID][]string, error return result, nil } + +func (s *AllocationStore) makeResources(devices []resourceapi.Device) resourceslice.DriverResources { + slice := resourceslice.Slice{ + Devices: devices, + } + poolName := s.nodeName + + if featuregates.Default().USBGatewayEnabled() { + slice.PerDeviceNodeSelection = ptr.To(true) + poolName = "usb-gateway" + } + + return resourceslice.DriverResources{ + Pools: map[string]resourceslice.Pool{ + poolName: { + Slices: []resourceslice.Slice{slice}, + }, + }, + } +} From 3327c702269ff22673d634490b1f94e9ceacd304 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Wed, 14 Jan 2026 11:39:56 +0300 Subject: [PATCH 18/37] fix usb gateway status Signed-off-by: Yaroslav Borbat --- Taskfile.yaml | 36 ++++++- .../debug/dlv.Dockerfile | 32 +++++++ images/virtualization-dra/api/sheme.go | 36 +++++++ images/virtualization-dra/api/types.go | 36 ++++++- .../cmd/usb-gateway/app/init.go | 1 + .../controller/resourceclaim/controller.go | 94 ++++++++++++++----- .../internal/usb/convert.go | 30 +++--- .../virtualization-dra/internal/usb/store.go | 16 +++- .../internal/usbip/binder.go | 64 ++++++------- 9 files changed, 267 insertions(+), 78 deletions(-) create mode 100644 images/virtualization-dra-usb-gateway/debug/dlv.Dockerfile create mode 100644 images/virtualization-dra/api/sheme.go diff --git a/Taskfile.yaml b/Taskfile.yaml index 6703c2ad6e..ea1475e669 100644 --- a/Taskfile.yaml +++ b/Taskfile.yaml @@ -314,5 +314,39 @@ tasks: } } }' - kubectl -n d8-virtualization port-forward deploy/virtualization-dra 2345:2345 + kubectl -n d8-virtualization port-forward pod/ 2345:2345 + EOF + + dlv:virtualization-dra-usb-gateway:build: + desc: "Build image virtualization-dra-usb-gateway with dlv" + cmds: + - docker build --build-arg BRANCH=$BRANCH -f ./images/virtualization-dra-usb-gateway/debug/dlv.Dockerfile -t "{{ .DLV_IMAGE }}" --platform linux/amd64 . + + dlv:virtualization-dra-usb-gateway:build-push: + desc: "Build and Push image virtualization-dra-usb-gateway with dlv" + cmds: + - task: dlv:virtualization-dra-usb-gateway:build + - docker push "{{ .DLV_IMAGE }}" + - task: dlv:virtualization-dra-usb-gateway:print + + dlv:virtualization-dra-usb-gateway:print: + desc: "Print commands for debug" + env: + IMAGE: "{{ .DLV_IMAGE }}" + cmd: | + cat < 2345:2345 EOF diff --git a/images/virtualization-dra-usb-gateway/debug/dlv.Dockerfile b/images/virtualization-dra-usb-gateway/debug/dlv.Dockerfile new file mode 100644 index 0000000000..99c8469601 --- /dev/null +++ b/images/virtualization-dra-usb-gateway/debug/dlv.Dockerfile @@ -0,0 +1,32 @@ +FROM golang:1.25-bookworm@sha256:019c22232e57fda8ded2b10a8f201989e839f3d3f962d4931375069bbb927e03 AS builder +ARG TARGETOS +ARG TARGETARCH + +WORKDIR /app/images/virtualization-dra +RUN go install github.com/go-delve/delve/cmd/dlv@latest + +COPY ./images/virtualization-dra/go.mod /app/images/virtualization-dra/ +COPY ./images/virtualization-dra/go.sum /app/images/virtualization-dra/ + +RUN go mod download + +COPY ./images/virtualization-dra/cmd /app/images/virtualization-dra/cmd +COPY ./images/virtualization-dra/internal /app/images/virtualization-dra/internal +COPY ./images/virtualization-dra/pkg /app/images/virtualization-dra/pkg +COPY ./images/virtualization-dra/api /app/images/virtualization-dra/api + +ENV GO111MODULE=on +ENV GOOS=${TARGETOS:-linux} +ENV GOARCH=${TARGETARCH:-amd64} +ENV CGO_ENABLED=0 + +RUN go build -gcflags "all=-N -l" -a -o virtualization-dra-usb-gateway ./cmd/usb-gateway/main.go + +FROM busybox:1.36.1-glibc + +WORKDIR /app +COPY --from=builder /go/bin/dlv /app/dlv +COPY --from=builder /app/images/virtualization-dra/virtualization-dra-usb-gateway /app/virtualization-dra-usb-gateway +USER 65532:65532 + +ENTRYPOINT ["./dlv", "--listen=:2345", "--headless=true", "--continue", "--log=true", "--log-output=debugger,debuglineerr,gdbwire,lldbout,rpc", "--accept-multiclient", "--api-version=2", "exec", "./virtualization-dra-usb-gateway", "--"] diff --git a/images/virtualization-dra/api/sheme.go b/images/virtualization-dra/api/sheme.go new file mode 100644 index 0000000000..2a6eeb290f --- /dev/null +++ b/images/virtualization-dra/api/sheme.go @@ -0,0 +1,36 @@ +package api + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" +) + +const Version = "v1alpha2" +const Group = "usb-gateway.virtualization.deckhouse.io" + +var SchemeGroupVersion = schema.GroupVersion{Group: Group, Version: Version} + +var ( + Scheme = runtime.NewScheme() + Codecs = serializer.NewCodecFactory(Scheme) +) + +var ( + SchemeBuilder = runtime.NewSchemeBuilder(addKnownTypes) + AddToScheme = SchemeBuilder.AddToScheme +) + +func addKnownTypes(scheme *runtime.Scheme) error { + scheme.AddKnownTypes(SchemeGroupVersion, + &USBGatewayStatus{}, + ) + return nil +} + +func init() { + metav1.AddToGroupVersion(Scheme, SchemeGroupVersion) + utilruntime.Must(AddToScheme(Scheme)) +} diff --git a/images/virtualization-dra/api/types.go b/images/virtualization-dra/api/types.go index abdadcbaa6..8f09b29f99 100644 --- a/images/virtualization-dra/api/types.go +++ b/images/virtualization-dra/api/types.go @@ -17,10 +17,14 @@ limitations under the License. package api import ( + "fmt" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" ) +const USBGatewayStatusKind = "USBGatewayStatus" + // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object type USBGatewayStatus struct { @@ -36,13 +40,35 @@ type USBGatewayStatus struct { Attached bool `json:"attached"` } -func FromData(data *runtime.RawExtension) *USBGatewayStatus { +func FromData(data *runtime.RawExtension) (*USBGatewayStatus, error) { if data == nil { - return nil + return nil, nil + } + + obj, err := runtime.Decode(Codecs.UniversalDecoder(SchemeGroupVersion), data.Raw) + if err != nil { + return nil, fmt.Errorf("failed to decode USBGatewayStatus: %w", err) } - status, ok := data.Object.(*USBGatewayStatus) + status, ok := obj.(*USBGatewayStatus) if !ok { - return nil + return nil, fmt.Errorf("failed to decode USBGatewayStatus: unexpected object type: %T", obj) } - return status + + return status, nil +} + +func ToData(status *USBGatewayStatus) (*runtime.RawExtension, error) { + if status == nil { + return nil, nil + } + + raw, err := runtime.Encode(Codecs.LegacyCodec(SchemeGroupVersion), status) + if err != nil { + return nil, fmt.Errorf("failed to encode USBGatewayStatus: %w", err) + } + + return &runtime.RawExtension{ + Raw: raw, + Object: status, + }, nil } diff --git a/images/virtualization-dra/cmd/usb-gateway/app/init.go b/images/virtualization-dra/cmd/usb-gateway/app/init.go index ebba3886ff..83511a71dc 100644 --- a/images/virtualization-dra/cmd/usb-gateway/app/init.go +++ b/images/virtualization-dra/cmd/usb-gateway/app/init.go @@ -47,6 +47,7 @@ func (o *initOptions) Run(_ *cobra.Command, _ []string) error { modules := []string{ filepath.Join("/lib/modules", kernelRelease, "kernel/drivers/usb/usbip/usbip-core.ko"), + filepath.Join("/lib/modules", kernelRelease, "kernel/drivers/usb/usbip/usbip-host.ko"), filepath.Join("/lib/modules", kernelRelease, "kernel/drivers/usb/usbip/vhci-hcd.ko"), } diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go index bf6c216a4c..9b57c957bd 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -28,7 +28,6 @@ import ( resourcev1beta1 "k8s.io/api/resource/v1beta1" k8serrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" @@ -246,7 +245,11 @@ func (c *Controller) sync(key string) error { // TODO: handle detach too func (c *Controller) handleDelete(rc *resourcev1beta1.ResourceClaim) error { - if c.allUnBound(rc) { + unbound, err := c.allUnBound(rc) + if err != nil { + return err + } + if unbound { return c.removeFinalizer(rc) } @@ -265,7 +268,10 @@ func (c *Controller) handleDelete(rc *resourcev1beta1.ResourceClaim) error { for i := range rc.Status.Devices { allocatedDeviceStatus := &rc.Status.Devices[i] - usbGatewayStatus := vdraapi.FromData(allocatedDeviceStatus.Data) + usbGatewayStatus, err := vdraapi.FromData(allocatedDeviceStatus.Data) + if err != nil { + return err + } if usbGatewayStatus == nil { continue } @@ -292,7 +298,10 @@ func (c *Controller) handleDelete(rc *resourcev1beta1.ResourceClaim) error { } usbGatewayStatus.Bound = false - allocatedDeviceStatus.Data.Object = usbGatewayStatus + allocatedDeviceStatus.Data, err = vdraapi.ToData(usbGatewayStatus) + if err != nil { + return err + } shouldUpdate = true } @@ -306,18 +315,21 @@ func (c *Controller) handleDelete(rc *resourcev1beta1.ResourceClaim) error { return nil } -func (c *Controller) allUnBound(rc *resourcev1beta1.ResourceClaim) bool { +func (c *Controller) allUnBound(rc *resourcev1beta1.ResourceClaim) (bool, error) { for _, deviceStatus := range rc.Status.Devices { - usbGatewayStatus := vdraapi.FromData(deviceStatus.Data) + usbGatewayStatus, err := vdraapi.FromData(deviceStatus.Data) + if err != nil { + return false, err + } if usbGatewayStatus == nil { continue } if usbGatewayStatus.Bound { - return false + return false, nil } } - return true + return true, nil } func (c *Controller) addFinalizer(rc *resourcev1beta1.ResourceClaim) error { @@ -366,18 +378,49 @@ func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocatio index, ok := indexAllocDevice[device.Name] if !ok { - continue + var ( + driver string + pool string + ) + if rc.Status.Allocation != nil { + for _, result := range rc.Status.Allocation.Devices.Results { + if result.Device == device.Name { + driver = result.Driver + pool = result.Pool + } + } + } + if driver == "" || pool == "" { + return fmt.Errorf("device %s is not allocated, driver or pool is empty", device.Name) + } + + rc.Status.Devices = append(rc.Status.Devices, resourcev1beta1.AllocatedDeviceStatus{ + Driver: driver, + Pool: pool, + Device: device.Name, + }) } allocDeviceStatus := &rc.Status.Devices[index] - usbGatewayStatus := vdraapi.FromData(allocDeviceStatus.Data) + usbGatewayStatus, err := vdraapi.FromData(allocDeviceStatus.Data) + if err != nil { + return err + } targetIPAlreadySet := usbGatewayStatus != nil && usbGatewayStatus.TargetIP != "" - if targetIPAlreadySet { + targetIPWrong := usbGatewayStatus != nil && usbGatewayStatus.TargetIP != c.podIP.String() + + if targetIPAlreadySet && !targetIPWrong { continue } - usbGatewayStatus = &vdraapi.USBGatewayStatus{} + + usbGatewayStatus = &vdraapi.USBGatewayStatus{ + TypeMeta: metav1.TypeMeta{ + APIVersion: vdraapi.SchemeGroupVersion.String(), + Kind: vdraapi.USBGatewayStatusKind, + }, + } busID := "" if attr, ok := device.Basic.Attributes["busID"]; ok && attr.StringValue != nil { @@ -401,10 +444,12 @@ func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocatio usbGatewayStatus.TargetPort = c.usbipPort usbGatewayStatus.Bound = true - if allocDeviceStatus.Data == nil { - allocDeviceStatus.Data = &runtime.RawExtension{} + data, err := vdraapi.ToData(usbGatewayStatus) + if err != nil { + return err } - allocDeviceStatus.Data.Object = usbGatewayStatus + + allocDeviceStatus.Data = data shouldUpdate = true } @@ -439,7 +484,10 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca continue } allocDeviceStatus := &rc.Status.Devices[index] - usbGatewayStatus := vdraapi.FromData(allocDeviceStatus.Data) + usbGatewayStatus, err := vdraapi.FromData(allocDeviceStatus.Data) + if err != nil { + return err + } if usbGatewayStatus == nil { continue } @@ -456,16 +504,16 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca continue } - err := c.usbIP.Attach(usbGatewayStatus.TargetIP, busID, usbGatewayStatus.TargetPort) + err = c.usbIP.Attach(usbGatewayStatus.TargetIP, busID, usbGatewayStatus.TargetPort) if err != nil { return fmt.Errorf("failed to attach usb: %w", err) } usbGatewayStatus.Attached = true - if allocDeviceStatus.Data == nil { - allocDeviceStatus.Data = &runtime.RawExtension{} + allocDeviceStatus.Data, err = vdraapi.ToData(usbGatewayStatus) + if err != nil { + return err } - allocDeviceStatus.Data.Object = usbGatewayStatus shouldUpdate = true } @@ -573,10 +621,14 @@ func (c *Controller) getAllocationDevices(rc *resourcev1beta1.ResourceClaim) ([] } // now, driver virtualization-dra supports only usb, but we can add more devices later // so we need to check if the device is usb - if strings.HasPrefix(status.Device, "usb") { + if !strings.HasPrefix(status.Device, "usb") { continue } + if _, exists := allocResultsByPool[status.Pool]; !exists { + allocResultsByPool[status.Pool] = make(map[string]resourcev1beta1.DeviceRequestAllocationResult) + } + allocResultsByPool[status.Pool][status.Device] = status } diff --git a/images/virtualization-dra/internal/usb/convert.go b/images/virtualization-dra/internal/usb/convert.go index 542c8b11bf..500136d3a2 100644 --- a/images/virtualization-dra/internal/usb/convert.go +++ b/images/virtualization-dra/internal/usb/convert.go @@ -83,19 +83,7 @@ func convertToAPIDevice(usbDevice Device, nodeName string) *resourceapi.Device { if featuregates.Default().USBGatewayEnabled() { // TODO: need pr to deckhouse for enable DRAPartitionableDevices feature gate on ApiServer - device.NodeSelector = &corev1.NodeSelector{ - NodeSelectorTerms: []corev1.NodeSelectorTerm{ - { - MatchExpressions: []corev1.NodeSelectorRequirement{ - { - Key: common.USBGatewayLabel, - Operator: corev1.NodeSelectorOpIn, - Values: []string{"true"}, - }, - }, - }, - }, - } + //device.NodeSelector = getNodeSelector() // TODO: add support for multiple allocations // device.AllowMultipleAllocations = ptr.To(true) } else { @@ -104,3 +92,19 @@ func convertToAPIDevice(usbDevice Device, nodeName string) *resourceapi.Device { return device } + +func getNodeSelector() *corev1.NodeSelector { + return &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: common.USBGatewayLabel, + Operator: corev1.NodeSelectorOpIn, + Values: []string{"true"}, + }, + }, + }, + }, + } +} diff --git a/images/virtualization-dra/internal/usb/store.go b/images/virtualization-dra/internal/usb/store.go index 7190f29200..0e6fefa704 100644 --- a/images/virtualization-dra/internal/usb/store.go +++ b/images/virtualization-dra/internal/usb/store.go @@ -420,21 +420,27 @@ func parseDraEnvToClaimAllocations(envs []string) (map[types.UID][]string, error } func (s *AllocationStore) makeResources(devices []resourceapi.Device) resourceslice.DriverResources { + //time.Sleep(20 * time.Second) slice := resourceslice.Slice{ Devices: devices, } poolName := s.nodeName if featuregates.Default().USBGatewayEnabled() { - slice.PerDeviceNodeSelection = ptr.To(true) - poolName = "usb-gateway" + //slice.PerDeviceNodeSelection = ptr.To(true) + } + + pool := resourceslice.Pool{ + Slices: []resourceslice.Slice{slice}, + } + + if featuregates.Default().USBGatewayEnabled() { + pool.NodeSelector = getNodeSelector() } return resourceslice.DriverResources{ Pools: map[string]resourceslice.Pool{ - poolName: { - Slices: []resourceslice.Slice{slice}, - }, + poolName: pool, }, } } diff --git a/images/virtualization-dra/internal/usbip/binder.go b/images/virtualization-dra/internal/usbip/binder.go index 226eabc6a8..027a54e242 100644 --- a/images/virtualization-dra/internal/usbip/binder.go +++ b/images/virtualization-dra/internal/usbip/binder.go @@ -54,7 +54,8 @@ func (b *usbBinder) Bind(busID string) error { return fmt.Errorf("failed to bind usb device: %w: %w", err, b.modifyMatchBusID(busID, false)) } - return b.storeBind(busID, true) + return nil + // return b.storeBind(busID, true) } // Unbind unbinds the USB device from the USBIP server. @@ -65,7 +66,7 @@ func (b *usbBinder) Unbind(busID string) error { return fmt.Errorf("device with bus ID %s does not exist: %w", busID, err) } - if devInfo.Driver != usbipHostDriverName { + if b.isBound(devInfo) { return fmt.Errorf("device %s is not bound to %s driver", devInfo.BusID, usbipHostDriverName) } @@ -83,11 +84,16 @@ func (b *usbBinder) Unbind(busID string) error { return fmt.Errorf("failed to rebind usb device %s: %w", busID, err) } - return b.storeBind(busID, false) + return nil + // return b.storeBind(busID, false) } func (b *usbBinder) IsBound(busID string) (bool, error) { - return b.isBound(busID) + devInfo, err := b.getUSBDeviceInfo(busID) + if err != nil { + return false, fmt.Errorf("device with bus ID %s does not exist: %w", busID, err) + } + return b.isBound(devInfo), nil } type usbDeviceInfo struct { @@ -132,37 +138,29 @@ func (b *usbBinder) getUSBDeviceInfo(busID string) (*usbDeviceInfo, error) { return info, nil } -func (b *usbBinder) storeBind(busID string, bind bool) error { - bound, err := b.isBound(busID) - if err != nil { - return err - } - if bound == bind { - return nil - } - path := bindPath(busID) - if bind { - _, err = os.Create(path) - return err - } - return os.Remove(path) -} +//func (b *usbBinder) storeBind(busID string, bind bool) error { +// bound, err := b.isBound(busID) +// if err != nil { +// return err +// } +// if bound == bind { +// return nil +// } +// path := bindPath(busID) +// if bind { +// _, err = os.Create(path) +// return err +// } +// return os.Remove(path) +//} -func (b *usbBinder) isBound(busID string) (bool, error) { - path := bindPath(busID) - _, err := os.Stat(path) - if err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, err - } - return true, nil +func (b *usbBinder) isBound(devInfo *usbDeviceInfo) bool { + return devInfo.Driver == usbipHostDriverName } -func bindPath(busID string) string { - return filepath.Join(getUSBDevicePath(busID), "usbip_bound") -} +//func bindPath(busID string) string { +// return filepath.Join(getUSBDevicePath(busID), "usbip_bound") +//} func (b *usbBinder) unbindOther(devInfo *usbDeviceInfo) error { if devInfo.IsHub { @@ -174,7 +172,7 @@ func (b *usbBinder) unbindOther(devInfo *usbDeviceInfo) error { return nil } - if devInfo.Driver == usbipHostDriverName { + if b.isBound(devInfo) { return fmt.Errorf("device %s is already bound to %s", devInfo.BusID, usbipHostDriverName) } From 718c8fc40cad495257b8b15e55ed79fabff8a625 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Wed, 14 Jan 2026 15:49:25 +0300 Subject: [PATCH 19/37] fix dra usbgateway devices Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/api/types.go | 15 +- .../cmd/go-usbip/app/app.go | 1 + .../cmd/go-usbip/app/attach.go | 3 +- .../cmd/go-usbip/app/info.go | 46 +++++ .../controller/resourceclaim/controller.go | 28 ++- .../internal/usb/discovery.go | 38 +++- .../virtualization-dra/internal/usb/store.go | 170 +++++++++++------- .../internal/usbip/attacher.go | 35 +++- .../internal/usbip/interfaces.go | 12 +- .../virtualization-dra/internal/usbip/vhci.go | 2 + 10 files changed, 260 insertions(+), 90 deletions(-) create mode 100644 images/virtualization-dra/cmd/go-usbip/app/info.go diff --git a/images/virtualization-dra/api/types.go b/images/virtualization-dra/api/types.go index 8f09b29f99..7a729f0dd5 100644 --- a/images/virtualization-dra/api/types.go +++ b/images/virtualization-dra/api/types.go @@ -30,14 +30,13 @@ const USBGatewayStatusKind = "USBGatewayStatus" type USBGatewayStatus struct { metav1.TypeMeta `json:",inline"` - BusNum int `json:"busNum"` - DeviceNum int `json:"deviceNum"` - DevicePath string `json:"devicePath"` - - TargetIP string `json:"targetIP"` - TargetPort int `json:"targetPort"` - Bound bool `json:"bound"` - Attached bool `json:"attached"` + BusID string `json:"busID"` + + RemoteIP string `json:"remoteIP"` + RemotePort int `json:"remotePort"` + + Bound bool `json:"bound"` + Attached bool `json:"attached"` } func FromData(data *runtime.RawExtension) (*USBGatewayStatus, error) { diff --git a/images/virtualization-dra/cmd/go-usbip/app/app.go b/images/virtualization-dra/cmd/go-usbip/app/app.go index fbb1fe327f..e4801ed497 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/app.go +++ b/images/virtualization-dra/cmd/go-usbip/app/app.go @@ -45,6 +45,7 @@ func NewUSBIPCommand() *cobra.Command { NewAttachCommand(), NewDetachCommand(), NewUsedPortsCommand(), + NewUsedInfoCommand(), ) return cmd diff --git a/images/virtualization-dra/cmd/go-usbip/app/attach.go b/images/virtualization-dra/cmd/go-usbip/app/attach.go index 818f983a00..1e73b92519 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/attach.go +++ b/images/virtualization-dra/cmd/go-usbip/app/attach.go @@ -55,5 +55,6 @@ func (o *attachOptions) AddFlags(fs *pflag.FlagSet) { func (o *attachOptions) Run(_ *cobra.Command, args []string) error { host := args[0] busID := args[1] - return usbip.NewUSBAttacher().Attach(host, busID, o.port) + _, err := usbip.NewUSBAttacher().Attach(host, busID, o.port) + return err } diff --git a/images/virtualization-dra/cmd/go-usbip/app/info.go b/images/virtualization-dra/cmd/go-usbip/app/info.go new file mode 100644 index 0000000000..c386ca1ee5 --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/info.go @@ -0,0 +1,46 @@ +package app + +import ( + "encoding/json" + "fmt" + + "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +func NewUsedInfoCommand() *cobra.Command { + o := &usedInfoOptions{} + cmd := &cobra.Command{ + Use: "info", + Short: "Get used info", + Example: o.Usage(), + RunE: o.Run, + } + + return cmd +} + +type usedInfoOptions struct{} + +func (o *usedInfoOptions) Usage() string { + return ` # Get used info + $ go-usbip info +` +} + +func (o *usedInfoOptions) Run(cmd *cobra.Command, _ []string) error { + infos, err := usbip.NewUSBAttacher().GetUsedInfo() + if err != nil { + return err + } + + bytes, err := json.Marshal(infos) + if err != nil { + return fmt.Errorf("failed to marshal json: %w", err) + } + + cmd.Println(string(bytes)) + + return nil +} diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go index 9b57c957bd..a00f4d6587 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -408,8 +408,8 @@ func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocatio return err } - targetIPAlreadySet := usbGatewayStatus != nil && usbGatewayStatus.TargetIP != "" - targetIPWrong := usbGatewayStatus != nil && usbGatewayStatus.TargetIP != c.podIP.String() + targetIPAlreadySet := usbGatewayStatus != nil && usbGatewayStatus.RemoteIP != "" + targetIPWrong := usbGatewayStatus != nil && usbGatewayStatus.RemoteIP != c.podIP.String() if targetIPAlreadySet && !targetIPWrong { continue @@ -440,8 +440,8 @@ func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocatio } } - usbGatewayStatus.TargetIP = c.podIP.String() - usbGatewayStatus.TargetPort = c.usbipPort + usbGatewayStatus.RemoteIP = c.podIP.String() + usbGatewayStatus.RemotePort = c.usbipPort usbGatewayStatus.Bound = true data, err := vdraapi.ToData(usbGatewayStatus) @@ -504,12 +504,30 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca continue } - err = c.usbIP.Attach(usbGatewayStatus.TargetIP, busID, usbGatewayStatus.TargetPort) + rhport, err := c.usbIP.Attach(usbGatewayStatus.RemoteIP, busID, usbGatewayStatus.RemotePort) if err != nil { return fmt.Errorf("failed to attach usb: %w", err) } + infos, err := c.usbIP.GetUsedInfo() + if err != nil { + return fmt.Errorf("failed to get used info: %w", err) + } + + var usedInfo *usbip.UsedInfo + for _, info := range infos { + if info.Port == rhport { + usedInfo = &info + break + } + } + if usedInfo == nil { + return fmt.Errorf("failed to find used info for port %d", rhport) + } + usbGatewayStatus.Attached = true + usbGatewayStatus.BusID = usedInfo.LocalBusID + allocDeviceStatus.Data, err = vdraapi.ToData(usbGatewayStatus) if err != nil { return err diff --git a/images/virtualization-dra/internal/usb/discovery.go b/images/virtualization-dra/internal/usb/discovery.go index 28b72771d3..6d787d97a0 100644 --- a/images/virtualization-dra/internal/usb/discovery.go +++ b/images/virtualization-dra/internal/usb/discovery.go @@ -17,20 +17,50 @@ limitations under the License. package usb import ( + "github.com/deckhouse/virtualization-dra/internal/featuregates" + "github.com/deckhouse/virtualization-dra/internal/usbip" "github.com/deckhouse/virtualization-dra/pkg/usb" ) const PathToUSBDevices = usb.PathToUSBDevices -func discoverPluggedUSBDevices(pathToUSBDevices string) (*DeviceSet, error) { +func newDiscoverer() discoverer { + return discoverer{ + getter: usbip.NewUSBAttacher(), + } +} + +type discoverer struct { + getter usbip.USBInfoGetter +} + +func (d *discoverer) DiscoveryPluggedUSBDevices(pathToUSBDevices string) (*DeviceSet, *DeviceSet, error) { devices, err := usb.DiscoverPluggedUSBDevices(pathToUSBDevices) if err != nil { - return nil, err + return nil, nil, err } + + busIdMaps := make(map[string]struct{}) + if featuregates.Default().USBGatewayEnabled() { + infos, err := d.getter.GetUsedInfo() + if err != nil { + return nil, nil, err + } + for _, info := range infos { + busIdMaps[info.LocalBusID] = struct{}{} + } + } + usbDeviceSet := NewDeviceSet() + usbipDeviceSet := NewDeviceSet() + for _, device := range devices { - usbDeviceSet.Add(toDevice(device)) + if _, ok := busIdMaps[device.BusID]; ok { + usbipDeviceSet.Add(toDevice(device)) + } else { + usbDeviceSet.Add(toDevice(device)) + } } - return usbDeviceSet, nil + return usbDeviceSet, usbipDeviceSet, nil } diff --git a/images/virtualization-dra/internal/usb/store.go b/images/virtualization-dra/internal/usb/store.go index 0e6fefa704..7d9217a882 100644 --- a/images/virtualization-dra/internal/usb/store.go +++ b/images/virtualization-dra/internal/usb/store.go @@ -20,14 +20,12 @@ import ( "context" "fmt" "log/slog" - "strconv" "strings" "sync" "time" "github.com/containerd/nri/pkg/api" resourceapi "k8s.io/api/resource/v1" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/dynamic-resource-allocation/resourceslice" drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1beta1" @@ -36,10 +34,8 @@ import ( cdispec "tags.cncf.io/container-device-interface/specs-go" vdraapi "github.com/deckhouse/virtualization-dra/api" - "github.com/deckhouse/virtualization-dra/internal/common" - "github.com/deckhouse/virtualization-dra/internal/featuregates" - "github.com/deckhouse/virtualization-dra/internal/cdi" + "github.com/deckhouse/virtualization-dra/internal/featuregates" "github.com/deckhouse/virtualization-dra/pkg/set" ) @@ -63,6 +59,7 @@ func NewAllocationStore(nodeName, devicesPath string, resyncPeriod time.Duration allocatableDevices: make(map[string]resourceapi.Device), allocatedDevices: set.New[string](), resourceClaimAllocations: make(map[types.UID][]string), + discoverer: newDiscoverer(), } monitor := newUSBMonitor(monitorCallback{ @@ -89,20 +86,26 @@ type AllocationStore struct { updateChannel chan resourceslice.DriverResources mu sync.RWMutex - discoverPluggedUSBDevices *DeviceSet - allocatableDevices map[string]resourceapi.Device - allocatedDevices *set.Set[string] - resourceClaimAllocations map[types.UID][]string + discoverer discoverer + + discoverPluggedUSBDevices *DeviceSet + discoverUsbIpPluggedUSBDevices *DeviceSet + allocatableDevices map[string]resourceapi.Device + + allocatedDevices *set.Set[string] + resourceClaimAllocations map[types.UID][]string } func (s *AllocationStore) sync() error { s.mu.Lock() defer s.mu.Unlock() - discoverPluggedUSBDevices, err := discoverPluggedUSBDevices(s.devicesPath) + discoverPluggedUSBDevices, discoverUsbIpPluggedUSBDevices, err := s.discoverer.DiscoveryPluggedUSBDevices(s.devicesPath) if err != nil { return err } + s.discoverUsbIpPluggedUSBDevices = discoverUsbIpPluggedUSBDevices + if discoverPluggedUSBDevices.Equal(s.discoverPluggedUSBDevices) { return nil } @@ -175,11 +178,10 @@ func (s *AllocationStore) Prepare(_ context.Context, claim *resourceapi.Resource claimUID := string(claim.UID) preparedDevices := make(cdi.PreparedDevices, len(claim.Status.Allocation.Devices.Results)) + usbGatewayEnabled := featuregates.Default().USBGatewayEnabled() + for i, result := range claim.Status.Allocation.Devices.Results { - usbDevice, exists := s.allocatableDevices[result.Device] - if !exists { - return nil, fmt.Errorf("requested device is not allocatable: %v", result.Device) - } + // TODO: unnecessary? // kubernetes check allocatable devices // Warning FailedScheduling 8s default-scheduler 0/3 nodes are available: @@ -190,9 +192,45 @@ func (s *AllocationStore) Prepare(_ context.Context, claim *resourceapi.Resource return nil, fmt.Errorf("device %v is already allocated", result.Device) } - containerEditsOptions, err := newContainerEditsOptions(&usbDevice, claim) - if err != nil { - return nil, err + isUSBGatewayRequest := s.isUSBGatewayRequest(&result) + + if !usbGatewayEnabled && isUSBGatewayRequest { + return nil, fmt.Errorf("claim %s/%s has usb gateway request but usb gateway is disabled", claim.Namespace, claim.Name) + } + + var containerEditsOptions containerEditsOptions + + if isUSBGatewayRequest { + + usbGatewayStatus, err := s.getUsbGatewayStatus(claim, result.Device) + if err != nil { + return nil, err + } + + if !usbGatewayStatus.Attached { + return nil, fmt.Errorf("claim %s/%s has usb gateway request but usb gateway is not attached", claim.Namespace, claim.Name) + } + + usbDevice := s.getUsbGatewayUsbDevice(usbGatewayStatus.BusID) + if usbDevice == nil { + return nil, fmt.Errorf("usb device %s is not found", usbGatewayStatus.BusID) + } + + containerEditsOptions = newContainerEditsOptionsForUSBGateway(result.Device, usbDevice) + + } else { + + usbDevice, exists := s.allocatableDevices[result.Device] + if !exists { + return nil, fmt.Errorf("requested device is not allocatable: %v", result.Device) + } + + opts, err := newContainerEditsOptions(&usbDevice) + if err != nil { + return nil, err + } + containerEditsOptions = opts + } edits, err := s.makeContainerEdits(claimUID, containerEditsOptions) @@ -225,58 +263,61 @@ func (s *AllocationStore) Prepare(_ context.Context, claim *resourceapi.Resource return devices, nil } -func newContainerEditsOptions(device *resourceapi.Device, claim *resourceapi.ResourceClaim) (containerEditsOptions, error) { - opts := containerEditsOptions{ - Name: device.Name, +func (s *AllocationStore) getUsbGatewayStatus(claim *resourceapi.ResourceClaim, deviceName string) (*vdraapi.USBGatewayStatus, error) { + for _, allocDeviceStatus := range claim.Status.Devices { + if allocDeviceStatus.Device == deviceName { + return vdraapi.FromData(allocDeviceStatus.Data) + } } + return nil, fmt.Errorf("device %s is not allocated", deviceName) +} - if featuregates.Default().USBGatewayEnabled() && isUSBGateway(claim) { - var data *runtime.RawExtension - for _, deviceStatus := range claim.Status.Devices { - if deviceStatus.Device == device.Name { - data = deviceStatus.Data - break - } - } - if data == nil { - return opts, fmt.Errorf("device status data is not found") +func (s *AllocationStore) getUsbGatewayUsbDevice(busID string) *Device { + for _, device := range s.discoverUsbIpPluggedUSBDevices.Slice() { + if device.BusID == busID { + return &device } + } + return nil +} - usbGatewayStatus, ok := data.Object.(*vdraapi.USBGatewayStatus) - if !ok { - return opts, fmt.Errorf("device status data is not a USBGatewayStatus") - } - if usbGatewayStatus == nil { - return opts, fmt.Errorf("device status data Object is nil") - } +func newContainerEditsOptionsForUSBGateway(deviceName string, usbDevice *Device) containerEditsOptions { + return containerEditsOptions{ + Name: deviceName, + DevicePath: usbDevice.DevicePath, + DeviceNum: usbDevice.DeviceNumber.String(), + Bus: usbDevice.Bus.String(), + Major: int64(usbDevice.Major), + Minor: int64(usbDevice.Minor), + } +} - opts.Bus = strconv.Itoa(usbGatewayStatus.BusNum) - opts.DeviceNum = strconv.Itoa(usbGatewayStatus.DeviceNum) - opts.DevicePath = usbGatewayStatus.DevicePath +func newContainerEditsOptions(device *resourceapi.Device) (containerEditsOptions, error) { + opts := containerEditsOptions{ + Name: device.Name, + } - } else { - if attr, ok := device.Attributes["devicePath"]; ok { - if val := attr.StringValue; val != nil { - opts.DevicePath = *val - } else { - return opts, fmt.Errorf("devicePath attribute is not exist") - } + if attr, ok := device.Attributes["devicePath"]; ok { + if val := attr.StringValue; val != nil { + opts.DevicePath = *val + } else { + return opts, fmt.Errorf("devicePath attribute is not exist") } + } - if attr, ok := device.Attributes["deviceNumber"]; ok { - if val := attr.StringValue; val != nil { - opts.DeviceNum = *val - } else { - return opts, fmt.Errorf("deviceNum attribute is not exist") - } + if attr, ok := device.Attributes["deviceNumber"]; ok { + if val := attr.StringValue; val != nil { + opts.DeviceNum = *val + } else { + return opts, fmt.Errorf("deviceNum attribute is not exist") } + } - if attr, ok := device.Attributes["bus"]; ok { - if val := attr.StringValue; val != nil { - opts.Bus = *val - } else { - return opts, fmt.Errorf("bus attribute is not exist") - } + if attr, ok := device.Attributes["bus"]; ok { + if val := attr.StringValue; val != nil { + opts.Bus = *val + } else { + return opts, fmt.Errorf("bus attribute is not exist") } } @@ -299,8 +340,10 @@ func newContainerEditsOptions(device *resourceapi.Device, claim *resourceapi.Res return opts, nil } -func isUSBGateway(claim *resourceapi.ResourceClaim) bool { - return claim.Annotations[common.USBGatewayAnnotation] == "true" +func (s *AllocationStore) isUSBGatewayRequest(result *resourceapi.DeviceRequestAllocationResult) bool { + // virtualization-dra creates slices with pool name by node name + // if pool not equal our node name, it is usb gateway request + return result.Pool != s.nodeName } type containerEditsOptions struct { @@ -420,16 +463,11 @@ func parseDraEnvToClaimAllocations(envs []string) (map[types.UID][]string, error } func (s *AllocationStore) makeResources(devices []resourceapi.Device) resourceslice.DriverResources { - //time.Sleep(20 * time.Second) slice := resourceslice.Slice{ Devices: devices, } poolName := s.nodeName - if featuregates.Default().USBGatewayEnabled() { - //slice.PerDeviceNodeSelection = ptr.To(true) - } - pool := resourceslice.Pool{ Slices: []resourceslice.Slice{slice}, } diff --git a/images/virtualization-dra/internal/usbip/attacher.go b/images/virtualization-dra/internal/usbip/attacher.go index c23f24d1c9..0bec243d73 100644 --- a/images/virtualization-dra/internal/usbip/attacher.go +++ b/images/virtualization-dra/internal/usbip/attacher.go @@ -35,23 +35,23 @@ func NewUSBAttacher() USBAttacher { type usbAttacher struct{} // https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_attach.c#L174 -func (a usbAttacher) Attach(host, busID string, port int) error { +func (a usbAttacher) Attach(host, busID string, port int) (int, error) { conn, err := a.usbipNetTCPConnect(host, fmt.Sprintf("%d", port)) if err != nil { - return fmt.Errorf("failed to connect to usbipd: %w", err) + return -1, fmt.Errorf("failed to connect to usbipd: %w", err) } rhport, err := a.queryImportDevice(conn, busID) if err != nil { - return fmt.Errorf("failed to query import device: %w", err) + return -1, fmt.Errorf("failed to query import device: %w", err) } err = a.recordConnection(host, strconv.Itoa(port), busID, rhport) if err != nil { - return fmt.Errorf("failed to record connection: %w", err) + return -1, fmt.Errorf("failed to record connection: %w", err) } - return nil + return rhport, nil } // https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_detach.c#L32 @@ -121,6 +121,31 @@ func (a usbAttacher) GetUsedPorts() ([]int, error) { return ports, nil } +func (a usbAttacher) GetUsedInfo() ([]UsedInfo, error) { + driver, err := newVhciDriver() + if err != nil { + return nil, fmt.Errorf("failed to get vhci driver: %w", err) + } + + var usedInfos []UsedInfo + + for i := 0; i < driver.nports; i++ { + idev := &driver.idevs[i] + + vstatus := protocol.DeviceStatus(idev.status) + if vstatus == protocol.VDeviceStatusUsed { + usedInfos = append(usedInfos, UsedInfo{ + Port: idev.port, + Busnum: idev.busnum, + Devnum: idev.devnum, + LocalBusID: idev.localBusID, + }) + } + } + + return usedInfos, nil +} + // https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_network.c#L261 func (a usbAttacher) usbipNetTCPConnect(host, port string) (*net.TCPConn, error) { tcpAddr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(host, port)) diff --git a/images/virtualization-dra/internal/usbip/interfaces.go b/images/virtualization-dra/internal/usbip/interfaces.go index 17062585d6..48ef560530 100644 --- a/images/virtualization-dra/internal/usbip/interfaces.go +++ b/images/virtualization-dra/internal/usbip/interfaces.go @@ -36,9 +36,19 @@ type USBBinder interface { } type USBAttacher interface { - Attach(host, busID string, port int) error + Attach(host, busID string, port int) (int, error) Detach(port int) error + USBInfoGetter +} + +type USBInfoGetter interface { GetUsedPorts() ([]int, error) + GetUsedInfo() ([]UsedInfo, error) +} + +type UsedInfo struct { + Port, Busnum, Devnum int + LocalBusID string } type serverImpl struct { diff --git a/images/virtualization-dra/internal/usbip/vhci.go b/images/virtualization-dra/internal/usbip/vhci.go index f6439f1825..5945bb6b41 100644 --- a/images/virtualization-dra/internal/usbip/vhci.go +++ b/images/virtualization-dra/internal/usbip/vhci.go @@ -54,6 +54,7 @@ type vhciDriver struct { type importDevice struct { hub hubSpeed port, status, devID, busnum, devnum int + localBusID string } type hubSpeed int @@ -190,6 +191,7 @@ func (d *vhciDriver) parseStatus(statusBytes []byte) error { idev.devID = devID idev.busnum = busnum idev.devnum = devnum + idev.localBusID = localBusID if hub == "hs" { idev.hub = hubSpeedHigh From 3f607485bffdf813fb563d5b089d44db8039abdb Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Wed, 14 Jan 2026 22:13:17 +0300 Subject: [PATCH 20/37] improve attach Signed-off-by: Yaroslav Borbat --- .../mount-points.yaml | 3 +- .../controller/resourceclaim/controller.go | 105 ++++++++--- .../controller/resourceclaim/record.go | 167 ++++++++++++++++++ .../internal/usbip/attacher.go | 2 +- .../internal/usbip/protocol/common.go | 19 ++ .../internal/usbip/usbipd.go | 4 +- .../daemonset.yaml | 4 + 7 files changed, 279 insertions(+), 25 deletions(-) create mode 100644 images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go diff --git a/images/virtualization-dra-usb-gateway/mount-points.yaml b/images/virtualization-dra-usb-gateway/mount-points.yaml index 448596c34b..dfccfafd0c 100644 --- a/images/virtualization-dra-usb-gateway/mount-points.yaml +++ b/images/virtualization-dra-usb-gateway/mount-points.yaml @@ -1,4 +1,5 @@ # A list of pre-created mount points for containerd strict mode. -dirs: [] +dirs: + - /var/run/usb-gateway diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go index a00f4d6587..3a9e118046 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -68,6 +68,7 @@ type Controller struct { queue workqueue.TypedRateLimitingInterface[string] log *slog.Logger hasSynced cache.InformerSynced + recordManager *recordManager } func NewController(nodeName string, podIP net.IP, usbipPort int, client kubernetes.Interface, resourceClaimInformer, resourceSliceInformer, nodeInformer, podInformer cache.SharedIndexInformer, usbIP usbip.Interface) (*Controller, error) { @@ -76,6 +77,11 @@ func NewController(nodeName string, podIP net.IP, usbipPort int, client kubernet workqueue.TypedRateLimitingQueueConfig[string]{Name: controllerName}, ) + recordManager, err := newRecordManager(DefaultRecordStateDir, usbIP) + if err != nil { + return nil, err + } + c := &Controller{ nodeName: nodeName, podIP: podIP, @@ -88,9 +94,10 @@ func NewController(nodeName string, podIP net.IP, usbipPort int, client kubernet usbIP: usbIP, queue: queue, log: slog.With(slog.String("controller", controllerName)), + recordManager: recordManager, } - _, err := resourceClaimInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + _, err = resourceClaimInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ AddFunc: c.addResourceClaim, UpdateFunc: c.updateResourceClaim, DeleteFunc: c.deleteResourceClaim, @@ -217,12 +224,13 @@ func (c *Controller) sync(key string) error { return nil } + onMyNode := c.podOnMyNode(pod) + myAllocationDevices, otherAllocationDevices, err := c.getAllocationDevices(rc) if err != nil { return err } - onMyNode := c.podOnMyNode(pod) shouldShare := !onMyNode && len(myAllocationDevices) > 0 shouldAttach := onMyNode && len(otherAllocationDevices) > 0 @@ -504,30 +512,15 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca continue } - rhport, err := c.usbIP.Attach(usbGatewayStatus.RemoteIP, busID, usbGatewayStatus.RemotePort) - if err != nil { - return fmt.Errorf("failed to attach usb: %w", err) - } - - infos, err := c.usbIP.GetUsedInfo() + err = c.recordManager.Refresh() if err != nil { - return fmt.Errorf("failed to get used info: %w", err) + return fmt.Errorf("failed to Refresh record: %w", err) } - var usedInfo *usbip.UsedInfo - for _, info := range infos { - if info.Port == rhport { - usedInfo = &info - break - } - } - if usedInfo == nil { - return fmt.Errorf("failed to find used info for port %d", rhport) + if err = c.attach(busID, usbGatewayStatus); err != nil { + return fmt.Errorf("failed to attach usb: %w", err) } - usbGatewayStatus.Attached = true - usbGatewayStatus.BusID = usedInfo.LocalBusID - allocDeviceStatus.Data, err = vdraapi.ToData(usbGatewayStatus) if err != nil { return err @@ -545,6 +538,76 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca return nil } +func (c *Controller) attach(busID string, usbGatewayStatus *vdraapi.USBGatewayStatus) error { + entries := c.recordManager.GetEntries() + for _, entry := range entries { + if entry.BusID == busID { + if entry.RemoteBusID != usbGatewayStatus.RemoteIP || entry.RemotePort != usbGatewayStatus.RemotePort { + if err := c.detach(entry.Port); err != nil { + return err + } + } + return nil + } + } + + var attachErr error + var rhport int + + defer func() { + if attachErr != nil { + if err := c.detach(rhport); err != nil { + c.log.Error("failed to detach usb", slog.String("error", err.Error()), slog.Int("port", rhport)) + } + } + }() + + rhport, attachErr = c.usbIP.Attach(usbGatewayStatus.RemoteIP, busID, usbGatewayStatus.RemotePort) + if attachErr != nil { + return fmt.Errorf("failed to attach usb: %w", attachErr) + } + + infos, err := c.usbIP.GetUsedInfo() + if err != nil { + return fmt.Errorf("failed to get used info: %w", err) + } + + var usedInfo *usbip.UsedInfo + for _, info := range infos { + if info.Port == rhport { + usedInfo = &info + break + } + } + if usedInfo == nil { + return fmt.Errorf("failed to find used info for port %d", rhport) + } + + entry := Entry{ + Port: rhport, + RemotePort: usbGatewayStatus.RemotePort, + RemoteIP: usbGatewayStatus.RemoteIP, + RemoteBusID: busID, + BusID: usedInfo.LocalBusID, + } + + if err = c.recordManager.AddEntry(entry); err != nil { + return fmt.Errorf("failed to add entry: %w", err) + } + + usbGatewayStatus.Attached = true + usbGatewayStatus.BusID = usedInfo.LocalBusID + + return nil +} + +func (c *Controller) detach(port int) error { + if err := c.usbIP.Detach(port); err != nil { + return fmt.Errorf("failed to detach usb: %w", err) + } + return nil +} + func (c *Controller) getResourceClaim(key string) (*resourcev1beta1.ResourceClaim, error) { obj, exists, err := c.resourceClaimIndexer.GetByKey(key) if err != nil && !k8serrors.IsNotFound(err) { diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go new file mode 100644 index 0000000000..732e006c83 --- /dev/null +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go @@ -0,0 +1,167 @@ +package resourceclaim + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "slices" + "sync" + + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +const DefaultRecordStateDir = "/var/run/usb-gateway" + +type record struct { + Entries []Entry `json:"entries,omitempty"` +} + +type Entry struct { + Port int `json:"port"` + RemotePort int `json:"remotePort" json:"remotePort"` + RemoteIP string `json:"remoteIP" json:"remoteIP"` + RemoteBusID string `json:"remoteBusID" json:"remoteBusID"` + BusID string `json:"busID" json:"busID"` +} + +func (e Entry) Validate() error { + if e.Port <= 0 { + return fmt.Errorf("port is required") + } + if e.RemotePort <= 0 { + return fmt.Errorf("remotePort is required") + } + if e.RemoteIP == "" { + return fmt.Errorf("remoteIP is required") + } + if e.RemoteBusID == "" { + return fmt.Errorf("remoteBusID is required") + } + if e.BusID == "" { + return fmt.Errorf("busID is required") + } + return nil +} + +type recordManager struct { + recordFile string + getter usbip.USBInfoGetter + + mu sync.RWMutex + record record +} + +func newRecordManager(stateDir string, getter usbip.USBInfoGetter) (*recordManager, error) { + err := os.MkdirAll(stateDir, 0700) + if err != nil { + return nil, err + } + + recordFile := filepath.Join(stateDir, "record.json") + if _, err = os.Stat(recordFile); err != nil { + if os.IsNotExist(err) { + f, err := os.Create(recordFile) + if err != nil { + return nil, err + } + _ = f.Close() + } else { + return nil, err + } + } + + r := recordManager{ + recordFile: recordFile, + getter: getter, + } + + if err = r.Refresh(); err != nil { + return nil, fmt.Errorf("failed to Refresh record: %w", err) + } + + return &r, nil +} + +func (r *recordManager) Refresh() error { + r.mu.Lock() + defer r.mu.Unlock() + + infos, err := r.getter.GetUsedInfo() + if err != nil { + return err + } + + byBusId := make(map[string]*usbip.UsedInfo, len(infos)) + for _, info := range infos { + byBusId[info.LocalBusID] = &info + } + + b, err := os.ReadFile(r.recordFile) + if err != nil { + return err + } + + record := record{} + if err = json.Unmarshal(b, &record); err != nil { + return err + } + + // keep only real entries + var newEntries []Entry + for _, e := range record.Entries { + if _, ok := byBusId[e.RemoteBusID]; ok { + newEntries = append(newEntries, e) + } + } + record.Entries = newEntries + + r.record = record + + return nil +} + +func (r *recordManager) GetEntries() []Entry { + r.mu.RLock() + defer r.mu.RUnlock() + + return slices.Clone(r.record.Entries) +} + +func (r *recordManager) AddEntry(e Entry) error { + if err := e.Validate(); err != nil { + return err + } + + r.mu.Lock() + defer r.mu.Unlock() + + for _, entry := range r.record.Entries { + if entry.RemoteBusID == e.RemoteBusID { + return fmt.Errorf("entry with RemoteBusID %s already exists", e.RemoteBusID) + } + if entry.BusID == e.BusID { + return fmt.Errorf("entry with BusID %s already exists", e.BusID) + } + if entry.Port == e.Port { + return fmt.Errorf("entry with Port %d already exists", e.Port) + } + } + + newEntries := slices.Clone(r.record.Entries) + newEntries = append(newEntries, e) + + record := record{Entries: newEntries} + + b, err := json.Marshal(record) + if err != nil { + return err + } + + if err = os.WriteFile(r.recordFile, b, 0600); err != nil { + return err + } + + r.record = record + return nil +} diff --git a/images/virtualization-dra/internal/usbip/attacher.go b/images/virtualization-dra/internal/usbip/attacher.go index 0bec243d73..19b043314a 100644 --- a/images/virtualization-dra/internal/usbip/attacher.go +++ b/images/virtualization-dra/internal/usbip/attacher.go @@ -194,7 +194,7 @@ func (a usbAttacher) queryImportDevice(conn *net.TCPConn, busID string) (int, er } if importReply.Status != protocol.OpStatusOk { - return -1, fmt.Errorf("reply failed: %d", importReply.Status) + return -1, fmt.Errorf("reply failed: %s", importReply.Status.String()) } if importReply.USBDevice.GetBusID() != busID { diff --git a/images/virtualization-dra/internal/usbip/protocol/common.go b/images/virtualization-dra/internal/usbip/protocol/common.go index c1e3c5f714..76e48cf81e 100644 --- a/images/virtualization-dra/internal/usbip/protocol/common.go +++ b/images/virtualization-dra/internal/usbip/protocol/common.go @@ -96,6 +96,25 @@ const ( OpStatusError OpStatus = 0x05 ) +func (o OpStatus) String() string { + switch o { + case OpStatusOk: + return "OK" + case OpStatusNA: + return "NA" + case OpStatusDevBusy: + return "DevBusy" + case OpStatusDevErr: + return "DevErr" + case OpStatusNoDev: + return "NoDev" + case OpStatusError: + return "Error" + default: + return "Unknown" + } +} + type DeviceStatus uint32 const ( diff --git a/images/virtualization-dra/internal/usbip/usbipd.go b/images/virtualization-dra/internal/usbip/usbipd.go index 055fbd13a2..d538232d1d 100644 --- a/images/virtualization-dra/internal/usbip/usbipd.go +++ b/images/virtualization-dra/internal/usbip/usbipd.go @@ -208,7 +208,7 @@ func (u *USBIPD) handleConnection(conn net.Conn) (bool, error) { } if opCommon.Status != protocol.OpStatusOk { - return false, fmt.Errorf("request failed: %d", opCommon.Status) + return false, fmt.Errorf("request failed: %s", opCommon.Status.String()) } switch opCommon.Code { @@ -270,7 +270,7 @@ func (u *USBIPD) handleImportRequest(conn net.Conn) error { status = u.exportDevice(conn, bindDevice) if status != protocol.OpStatusOk { - log.Error("failed to export device") + log.Error("failed to export device", slog.String("status", status.String())) } } else { diff --git a/templates/virtualization-dra-usb-gateway/daemonset.yaml b/templates/virtualization-dra-usb-gateway/daemonset.yaml index 49e7d0e72e..1dfaaf5cca 100644 --- a/templates/virtualization-dra-usb-gateway/daemonset.yaml +++ b/templates/virtualization-dra-usb-gateway/daemonset.yaml @@ -113,6 +113,8 @@ spec: mountPath: /sys - name: var-run mountPath: /var/run + - name: usb-gateway + mountPath: /var/run/usb-gateway volumes: - name: sys hostPath: @@ -123,4 +125,6 @@ spec: - name: lib-modules hostPath: path: /lib/modules + - name: usb-gateway + emptyDir: {} {{- end }} From 64689434b4f33c64474074db3a749bd483ea6a97 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Wed, 14 Jan 2026 22:24:40 +0300 Subject: [PATCH 21/37] fix store empty record Signed-off-by: Yaroslav Borbat --- .../usb-gateway/controller/resourceclaim/record.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go index 732e006c83..f95bf2206b 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go @@ -61,11 +61,14 @@ func newRecordManager(stateDir string, getter usbip.USBInfoGetter) (*recordManag recordFile := filepath.Join(stateDir, "record.json") if _, err = os.Stat(recordFile); err != nil { if os.IsNotExist(err) { - f, err := os.Create(recordFile) + b, err := json.Marshal(record{}) if err != nil { return nil, err } - _ = f.Close() + err = os.WriteFile(recordFile, b, 0600) + if err == nil { + return nil, err + } } else { return nil, err } From 0088a0ba24a37c3531910026e82651727c41ff6a Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Wed, 14 Jan 2026 22:37:26 +0300 Subject: [PATCH 22/37] fix recordManager panic Signed-off-by: Yaroslav Borbat --- .../internal/usb-gateway/controller/resourceclaim/record.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go index f95bf2206b..e39b5da480 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go @@ -66,7 +66,7 @@ func newRecordManager(stateDir string, getter usbip.USBInfoGetter) (*recordManag return nil, err } err = os.WriteFile(recordFile, b, 0600) - if err == nil { + if err != nil { return nil, err } } else { From 4028a21efe03a207480cd2abfca05191b1c60e8d Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Wed, 14 Jan 2026 23:27:43 +0300 Subject: [PATCH 23/37] refactor attach Signed-off-by: Yaroslav Borbat --- .../cmd/go-usbip/app/app.go | 3 +- .../go-usbip/app/{info.go => attach-info.go} | 20 +++---- .../cmd/go-usbip/app/ports.go | 59 ------------------- .../controller/resourceclaim/controller.go | 14 ++--- .../controller/resourceclaim/record.go | 10 ++-- .../internal/usb/discovery.go | 4 +- .../internal/usbip/attacher.go | 26 +------- .../internal/usbip/binder.go | 20 ------- .../internal/usbip/interfaces.go | 9 ++- 9 files changed, 32 insertions(+), 133 deletions(-) rename images/virtualization-dra/cmd/go-usbip/app/{info.go => attach-info.go} (51%) delete mode 100644 images/virtualization-dra/cmd/go-usbip/app/ports.go diff --git a/images/virtualization-dra/cmd/go-usbip/app/app.go b/images/virtualization-dra/cmd/go-usbip/app/app.go index e4801ed497..85a3043e26 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/app.go +++ b/images/virtualization-dra/cmd/go-usbip/app/app.go @@ -44,8 +44,7 @@ func NewUSBIPCommand() *cobra.Command { NewUnbindCommand(), NewAttachCommand(), NewDetachCommand(), - NewUsedPortsCommand(), - NewUsedInfoCommand(), + NewAttachInfoCommand(), ) return cmd diff --git a/images/virtualization-dra/cmd/go-usbip/app/info.go b/images/virtualization-dra/cmd/go-usbip/app/attach-info.go similarity index 51% rename from images/virtualization-dra/cmd/go-usbip/app/info.go rename to images/virtualization-dra/cmd/go-usbip/app/attach-info.go index c386ca1ee5..c5879f8f2c 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/info.go +++ b/images/virtualization-dra/cmd/go-usbip/app/attach-info.go @@ -9,11 +9,11 @@ import ( "github.com/deckhouse/virtualization-dra/internal/usbip" ) -func NewUsedInfoCommand() *cobra.Command { - o := &usedInfoOptions{} +func NewAttachInfoCommand() *cobra.Command { + o := &attachInfoOptions{} cmd := &cobra.Command{ - Use: "info", - Short: "Get used info", + Use: "attach-info", + Short: "Get attach info", Example: o.Usage(), RunE: o.Run, } @@ -21,16 +21,16 @@ func NewUsedInfoCommand() *cobra.Command { return cmd } -type usedInfoOptions struct{} +type attachInfoOptions struct{} -func (o *usedInfoOptions) Usage() string { - return ` # Get used info - $ go-usbip info +func (o *attachInfoOptions) Usage() string { + return ` # Get attach info + $ go-usbip attach-info ` } -func (o *usedInfoOptions) Run(cmd *cobra.Command, _ []string) error { - infos, err := usbip.NewUSBAttacher().GetUsedInfo() +func (o *attachInfoOptions) Run(cmd *cobra.Command, _ []string) error { + infos, err := usbip.NewUSBAttacher().GetAttachInfo() if err != nil { return err } diff --git a/images/virtualization-dra/cmd/go-usbip/app/ports.go b/images/virtualization-dra/cmd/go-usbip/app/ports.go deleted file mode 100644 index 634e519071..0000000000 --- a/images/virtualization-dra/cmd/go-usbip/app/ports.go +++ /dev/null @@ -1,59 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package app - -import ( - "fmt" - - "github.com/spf13/cobra" - - "github.com/deckhouse/virtualization-dra/internal/usbip" -) - -func NewUsedPortsCommand() *cobra.Command { - o := &usedPortsOptions{} - cmd := &cobra.Command{ - Use: "ports", - Short: "List used ports", - Example: o.Usage(), - RunE: o.Run, - } - - return cmd -} - -type usedPortsOptions struct{} - -func (o *usedPortsOptions) Usage() string { - return ` # List used ports - $ go-usbip ports -` -} - -func (o *usedPortsOptions) Run(cmd *cobra.Command, _ []string) error { - ports, err := usbip.NewUSBAttacher().GetUsedPorts() - if err != nil { - return err - } - - cmd.Println("Used ports:") - for _, port := range ports { - cmd.Println(fmt.Sprintf("- %d", port)) - } - - return nil -} diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go index 3a9e118046..3b9738d727 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -551,28 +551,28 @@ func (c *Controller) attach(busID string, usbGatewayStatus *vdraapi.USBGatewaySt } } - var attachErr error + var err error var rhport int defer func() { - if attachErr != nil { + if err != nil && rhport >= 0 { if err := c.detach(rhport); err != nil { c.log.Error("failed to detach usb", slog.String("error", err.Error()), slog.Int("port", rhport)) } } }() - rhport, attachErr = c.usbIP.Attach(usbGatewayStatus.RemoteIP, busID, usbGatewayStatus.RemotePort) - if attachErr != nil { - return fmt.Errorf("failed to attach usb: %w", attachErr) + rhport, err = c.usbIP.Attach(usbGatewayStatus.RemoteIP, busID, usbGatewayStatus.RemotePort) + if err != nil { + return fmt.Errorf("failed to attach usb: %w", err) } - infos, err := c.usbIP.GetUsedInfo() + infos, err := c.usbIP.GetAttachInfo() if err != nil { return fmt.Errorf("failed to get used info: %w", err) } - var usedInfo *usbip.UsedInfo + var usedInfo *usbip.AttachInfo for _, info := range infos { if info.Port == rhport { usedInfo = &info diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go index e39b5da480..3afdd5fa10 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go @@ -26,7 +26,7 @@ type Entry struct { } func (e Entry) Validate() error { - if e.Port <= 0 { + if e.Port < 0 { return fmt.Errorf("port is required") } if e.RemotePort <= 0 { @@ -46,13 +46,13 @@ func (e Entry) Validate() error { type recordManager struct { recordFile string - getter usbip.USBInfoGetter + getter usbip.AttachInfoGetter mu sync.RWMutex record record } -func newRecordManager(stateDir string, getter usbip.USBInfoGetter) (*recordManager, error) { +func newRecordManager(stateDir string, getter usbip.AttachInfoGetter) (*recordManager, error) { err := os.MkdirAll(stateDir, 0700) if err != nil { return nil, err @@ -90,12 +90,12 @@ func (r *recordManager) Refresh() error { r.mu.Lock() defer r.mu.Unlock() - infos, err := r.getter.GetUsedInfo() + infos, err := r.getter.GetAttachInfo() if err != nil { return err } - byBusId := make(map[string]*usbip.UsedInfo, len(infos)) + byBusId := make(map[string]*usbip.AttachInfo, len(infos)) for _, info := range infos { byBusId[info.LocalBusID] = &info } diff --git a/images/virtualization-dra/internal/usb/discovery.go b/images/virtualization-dra/internal/usb/discovery.go index 6d787d97a0..74d8dca3d2 100644 --- a/images/virtualization-dra/internal/usb/discovery.go +++ b/images/virtualization-dra/internal/usb/discovery.go @@ -31,7 +31,7 @@ func newDiscoverer() discoverer { } type discoverer struct { - getter usbip.USBInfoGetter + getter usbip.AttachInfoGetter } func (d *discoverer) DiscoveryPluggedUSBDevices(pathToUSBDevices string) (*DeviceSet, *DeviceSet, error) { @@ -42,7 +42,7 @@ func (d *discoverer) DiscoveryPluggedUSBDevices(pathToUSBDevices string) (*Devic busIdMaps := make(map[string]struct{}) if featuregates.Default().USBGatewayEnabled() { - infos, err := d.getter.GetUsedInfo() + infos, err := d.getter.GetAttachInfo() if err != nil { return nil, nil, err } diff --git a/images/virtualization-dra/internal/usbip/attacher.go b/images/virtualization-dra/internal/usbip/attacher.go index 19b043314a..9c7df3daf6 100644 --- a/images/virtualization-dra/internal/usbip/attacher.go +++ b/images/virtualization-dra/internal/usbip/attacher.go @@ -101,40 +101,20 @@ func (a usbAttacher) Detach(port int) error { return nil } -func (a usbAttacher) GetUsedPorts() ([]int, error) { +func (a usbAttacher) GetAttachInfo() ([]AttachInfo, error) { driver, err := newVhciDriver() if err != nil { return nil, fmt.Errorf("failed to get vhci driver: %w", err) } - var ports []int + var usedInfos []AttachInfo for i := 0; i < driver.nports; i++ { idev := &driver.idevs[i] vstatus := protocol.DeviceStatus(idev.status) if vstatus == protocol.VDeviceStatusUsed { - ports = append(ports, idev.port) - } - } - - return ports, nil -} - -func (a usbAttacher) GetUsedInfo() ([]UsedInfo, error) { - driver, err := newVhciDriver() - if err != nil { - return nil, fmt.Errorf("failed to get vhci driver: %w", err) - } - - var usedInfos []UsedInfo - - for i := 0; i < driver.nports; i++ { - idev := &driver.idevs[i] - - vstatus := protocol.DeviceStatus(idev.status) - if vstatus == protocol.VDeviceStatusUsed { - usedInfos = append(usedInfos, UsedInfo{ + usedInfos = append(usedInfos, AttachInfo{ Port: idev.port, Busnum: idev.busnum, Devnum: idev.devnum, diff --git a/images/virtualization-dra/internal/usbip/binder.go b/images/virtualization-dra/internal/usbip/binder.go index 027a54e242..38f09d2236 100644 --- a/images/virtualization-dra/internal/usbip/binder.go +++ b/images/virtualization-dra/internal/usbip/binder.go @@ -138,30 +138,10 @@ func (b *usbBinder) getUSBDeviceInfo(busID string) (*usbDeviceInfo, error) { return info, nil } -//func (b *usbBinder) storeBind(busID string, bind bool) error { -// bound, err := b.isBound(busID) -// if err != nil { -// return err -// } -// if bound == bind { -// return nil -// } -// path := bindPath(busID) -// if bind { -// _, err = os.Create(path) -// return err -// } -// return os.Remove(path) -//} - func (b *usbBinder) isBound(devInfo *usbDeviceInfo) bool { return devInfo.Driver == usbipHostDriverName } -//func bindPath(busID string) string { -// return filepath.Join(getUSBDevicePath(busID), "usbip_bound") -//} - func (b *usbBinder) unbindOther(devInfo *usbDeviceInfo) error { if devInfo.IsHub { return fmt.Errorf("skip unbinding of hub %s", devInfo.BusID) diff --git a/images/virtualization-dra/internal/usbip/interfaces.go b/images/virtualization-dra/internal/usbip/interfaces.go index 48ef560530..493529d232 100644 --- a/images/virtualization-dra/internal/usbip/interfaces.go +++ b/images/virtualization-dra/internal/usbip/interfaces.go @@ -38,15 +38,14 @@ type USBBinder interface { type USBAttacher interface { Attach(host, busID string, port int) (int, error) Detach(port int) error - USBInfoGetter + AttachInfoGetter } -type USBInfoGetter interface { - GetUsedPorts() ([]int, error) - GetUsedInfo() ([]UsedInfo, error) +type AttachInfoGetter interface { + GetAttachInfo() ([]AttachInfo, error) } -type UsedInfo struct { +type AttachInfo struct { Port, Busnum, Devnum int LocalBusID string } From e22875a8f46215652d104f926efa049f3558d572 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Thu, 15 Jan 2026 11:54:43 +0300 Subject: [PATCH 24/37] use sets util Signed-off-by: Yaroslav Borbat --- .../virtualization-dra/internal/usb/device.go | 9 +-- .../internal/usb/discovery.go | 6 +- .../virtualization-dra/internal/usb/store.go | 22 +++---- images/virtualization-dra/pkg/set/set.go | 64 ------------------- 4 files changed, 19 insertions(+), 82 deletions(-) delete mode 100644 images/virtualization-dra/pkg/set/set.go diff --git a/images/virtualization-dra/internal/usb/device.go b/images/virtualization-dra/internal/usb/device.go index 0a53c8d303..efd8f5acb0 100644 --- a/images/virtualization-dra/internal/usb/device.go +++ b/images/virtualization-dra/internal/usb/device.go @@ -21,14 +21,15 @@ import ( "strconv" "strings" - "github.com/deckhouse/virtualization-dra/pkg/set" + "k8s.io/apimachinery/pkg/util/sets" + "github.com/deckhouse/virtualization-dra/pkg/usb" ) -type DeviceSet = set.Set[Device] +type DeviceSet = sets.Set[Device] -func NewDeviceSet() *DeviceSet { - return set.New[Device]() +func NewDeviceSet() DeviceSet { + return sets.New[Device]() } type Device struct { diff --git a/images/virtualization-dra/internal/usb/discovery.go b/images/virtualization-dra/internal/usb/discovery.go index 74d8dca3d2..0eb5a59245 100644 --- a/images/virtualization-dra/internal/usb/discovery.go +++ b/images/virtualization-dra/internal/usb/discovery.go @@ -34,7 +34,7 @@ type discoverer struct { getter usbip.AttachInfoGetter } -func (d *discoverer) DiscoveryPluggedUSBDevices(pathToUSBDevices string) (*DeviceSet, *DeviceSet, error) { +func (d *discoverer) DiscoveryPluggedUSBDevices(pathToUSBDevices string) (DeviceSet, DeviceSet, error) { devices, err := usb.DiscoverPluggedUSBDevices(pathToUSBDevices) if err != nil { return nil, nil, err @@ -56,9 +56,9 @@ func (d *discoverer) DiscoveryPluggedUSBDevices(pathToUSBDevices string) (*Devic for _, device := range devices { if _, ok := busIdMaps[device.BusID]; ok { - usbipDeviceSet.Add(toDevice(device)) + usbipDeviceSet.Insert(toDevice(device)) } else { - usbDeviceSet.Add(toDevice(device)) + usbDeviceSet.Insert(toDevice(device)) } } diff --git a/images/virtualization-dra/internal/usb/store.go b/images/virtualization-dra/internal/usb/store.go index 7d9217a882..1d3a55689c 100644 --- a/images/virtualization-dra/internal/usb/store.go +++ b/images/virtualization-dra/internal/usb/store.go @@ -27,6 +27,7 @@ import ( "github.com/containerd/nri/pkg/api" resourceapi "k8s.io/api/resource/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/dynamic-resource-allocation/resourceslice" drapbv1 "k8s.io/kubelet/pkg/apis/dra/v1beta1" "k8s.io/utils/ptr" @@ -36,7 +37,6 @@ import ( vdraapi "github.com/deckhouse/virtualization-dra/api" "github.com/deckhouse/virtualization-dra/internal/cdi" "github.com/deckhouse/virtualization-dra/internal/featuregates" - "github.com/deckhouse/virtualization-dra/pkg/set" ) const DefaultResyncPeriod = 10 * time.Minute @@ -57,7 +57,7 @@ func NewAllocationStore(nodeName, devicesPath string, resyncPeriod time.Duration updateChannel: make(chan resourceslice.DriverResources, 2), discoverPluggedUSBDevices: NewDeviceSet(), allocatableDevices: make(map[string]resourceapi.Device), - allocatedDevices: set.New[string](), + allocatedDevices: sets.New[string](), resourceClaimAllocations: make(map[types.UID][]string), discoverer: newDiscoverer(), } @@ -88,11 +88,11 @@ type AllocationStore struct { discoverer discoverer - discoverPluggedUSBDevices *DeviceSet - discoverUsbIpPluggedUSBDevices *DeviceSet + discoverPluggedUSBDevices DeviceSet + discoverUsbIpPluggedUSBDevices DeviceSet allocatableDevices map[string]resourceapi.Device - allocatedDevices *set.Set[string] + allocatedDevices sets.Set[string] resourceClaimAllocations map[types.UID][]string } @@ -112,7 +112,7 @@ func (s *AllocationStore) sync() error { s.discoverPluggedUSBDevices = discoverPluggedUSBDevices allocatableDevices := make([]resourceapi.Device, discoverPluggedUSBDevices.Len()) - for i, usbDevice := range discoverPluggedUSBDevices.Slice() { + for i, usbDevice := range discoverPluggedUSBDevices.UnsortedList() { allocatableDevices[i] = *convertToAPIDevice(usbDevice, s.nodeName) } @@ -188,7 +188,7 @@ func (s *AllocationStore) Prepare(_ context.Context, claim *resourceapi.Resource // 1 node(s) had tolerated taint {node-role.kubernetes.io/control-plane: }, // 2 cannot allocate all claims. // still not schedulable, preemption: 0/3 nodes are available: 3 Preemption is not helpful for scheduling. - if s.allocatedDevices.Contains(result.Device) { + if s.allocatedDevices.Has(result.Device) { return nil, fmt.Errorf("device %v is already allocated", result.Device) } @@ -256,7 +256,7 @@ func (s *AllocationStore) Prepare(_ context.Context, claim *resourceapi.Resource devices := preparedDevices.GetDevices() for _, device := range devices { - s.allocatedDevices.Add(device.DeviceName) + s.allocatedDevices.Insert(device.DeviceName) s.resourceClaimAllocations[claim.UID] = append(s.resourceClaimAllocations[claim.UID], device.DeviceName) } @@ -273,7 +273,7 @@ func (s *AllocationStore) getUsbGatewayStatus(claim *resourceapi.ResourceClaim, } func (s *AllocationStore) getUsbGatewayUsbDevice(busID string) *Device { - for _, device := range s.discoverUsbIpPluggedUSBDevices.Slice() { + for _, device := range s.discoverUsbIpPluggedUSBDevices.UnsortedList() { if device.BusID == busID { return &device } @@ -397,7 +397,7 @@ func (s *AllocationStore) Unprepare(_ context.Context, claimUID types.UID) error allocatedDevices := s.resourceClaimAllocations[claimUID] for _, device := range allocatedDevices { - s.allocatedDevices.Remove(device) + s.allocatedDevices.Delete(device) } delete(s.resourceClaimAllocations, claimUID) @@ -426,7 +426,7 @@ func (s *AllocationStore) Synchronize(_ context.Context, pods []*api.PodSandbox, for claimUID, deviceNames := range claimUIDDeviceNames { s.resourceClaimAllocations[claimUID] = append(s.resourceClaimAllocations[claimUID], deviceNames...) for _, deviceName := range deviceNames { - s.allocatedDevices.Add(deviceName) + s.allocatedDevices.Insert(deviceName) } } diff --git a/images/virtualization-dra/pkg/set/set.go b/images/virtualization-dra/pkg/set/set.go deleted file mode 100644 index 4b7f4378a5..0000000000 --- a/images/virtualization-dra/pkg/set/set.go +++ /dev/null @@ -1,64 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package set - -type Set[T comparable] struct { - m map[T]struct{} -} - -func New[T comparable]() *Set[T] { - return &Set[T]{ - m: make(map[T]struct{}), - } -} -func (s *Set[T]) Add(v T) { - s.m[v] = struct{}{} -} - -func (s *Set[T]) Remove(v T) { - delete(s.m, v) -} - -func (s *Set[T]) Contains(v T) bool { - _, ok := s.m[v] - return ok -} - -func (s *Set[T]) Len() int { - return len(s.m) -} - -func (s *Set[T]) Slice() []T { - out := make([]T, 0, len(s.m)) - for k := range s.m { - out = append(out, k) - } - return out -} - -func (s *Set[T]) Equal(other *Set[T]) bool { - if s.Len() != other.Len() { - return false - } - - for k := range s.m { - if !other.Contains(k) { - return false - } - } - return true -} From 53b15f78f6a10114fd3f27726603adb177ea5eb2 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Thu, 15 Jan 2026 12:48:27 +0300 Subject: [PATCH 25/37] add bindinfo Signed-off-by: Yaroslav Borbat --- .../cmd/go-usbip/app/app.go | 1 + .../cmd/go-usbip/app/bind-info.go | 46 +++++++++++++++++++ .../internal/usbip/binder.go | 30 ++++++++++++ .../internal/usbip/interfaces.go | 13 ++++++ .../virtualization-dra/pkg/usb/discovery.go | 4 ++ 5 files changed, 94 insertions(+) create mode 100644 images/virtualization-dra/cmd/go-usbip/app/bind-info.go diff --git a/images/virtualization-dra/cmd/go-usbip/app/app.go b/images/virtualization-dra/cmd/go-usbip/app/app.go index 85a3043e26..90710d44a2 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/app.go +++ b/images/virtualization-dra/cmd/go-usbip/app/app.go @@ -45,6 +45,7 @@ func NewUSBIPCommand() *cobra.Command { NewAttachCommand(), NewDetachCommand(), NewAttachInfoCommand(), + NewBindInfoCommand(), ) return cmd diff --git a/images/virtualization-dra/cmd/go-usbip/app/bind-info.go b/images/virtualization-dra/cmd/go-usbip/app/bind-info.go new file mode 100644 index 0000000000..defdcbe703 --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/bind-info.go @@ -0,0 +1,46 @@ +package app + +import ( + "encoding/json" + "fmt" + + "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization-dra/internal/usbip" +) + +func NewBindInfoCommand() *cobra.Command { + o := &bindInfoOptions{} + cmd := &cobra.Command{ + Use: "bind-info", + Short: "Get bind info", + Example: o.Usage(), + RunE: o.Run, + } + + return cmd +} + +type bindInfoOptions struct{} + +func (o *bindInfoOptions) Usage() string { + return ` # Get bind info + $ go-usbip bind-info +` +} + +func (o *bindInfoOptions) Run(cmd *cobra.Command, _ []string) error { + infos, err := usbip.NewUSBBinder().GetBindInfo() + if err != nil { + return err + } + + bytes, err := json.Marshal(infos) + if err != nil { + return fmt.Errorf("failed to marshal json: %w", err) + } + + cmd.Println(string(bytes)) + + return nil +} diff --git a/images/virtualization-dra/internal/usbip/binder.go b/images/virtualization-dra/internal/usbip/binder.go index 38f09d2236..647f1fc6f0 100644 --- a/images/virtualization-dra/internal/usbip/binder.go +++ b/images/virtualization-dra/internal/usbip/binder.go @@ -21,6 +21,8 @@ import ( "os" "path/filepath" "strings" + + "github.com/deckhouse/virtualization-dra/pkg/usb" ) func NewUSBBinder() USBBinder { @@ -96,6 +98,34 @@ func (b *usbBinder) IsBound(busID string) (bool, error) { return b.isBound(devInfo), nil } +func (b *usbBinder) GetBindInfo() ([]BindInfo, error) { + usbDevices, err := usb.DefaultDiscoverPluggedUSBDevices() + if err != nil { + return nil, fmt.Errorf("failed to discover USB devices: %w", err) + } + + var infos []BindInfo + + for _, device := range usbDevices { + devInfo := usbDeviceInfo{ + BusID: device.BusID, + Driver: device.Driver, + DevPath: device.DevicePath, + IsHub: device.IsHub, + } + + infos = append(infos, BindInfo{ + DevicePath: device.DevicePath, + BusID: device.BusID, + Busnum: int(device.Bus), + Devnum: int(device.DeviceNumber), + Bound: b.isBound(&devInfo), + }) + } + + return infos, err +} + type usbDeviceInfo struct { BusID string Driver string diff --git a/images/virtualization-dra/internal/usbip/interfaces.go b/images/virtualization-dra/internal/usbip/interfaces.go index 493529d232..537a5004bb 100644 --- a/images/virtualization-dra/internal/usbip/interfaces.go +++ b/images/virtualization-dra/internal/usbip/interfaces.go @@ -33,6 +33,19 @@ type USBBinder interface { Bind(busID string) error Unbind(busID string) error IsBound(busID string) (bool, error) + BindInfoGetter +} + +type BindInfoGetter interface { + GetBindInfo() ([]BindInfo, error) +} + +type BindInfo struct { + DevicePath string + BusID string + Busnum int + Devnum int + Bound bool } type USBAttacher interface { diff --git a/images/virtualization-dra/pkg/usb/discovery.go b/images/virtualization-dra/pkg/usb/discovery.go index f3a7a19b4e..ebe1e357fd 100644 --- a/images/virtualization-dra/pkg/usb/discovery.go +++ b/images/virtualization-dra/pkg/usb/discovery.go @@ -24,6 +24,10 @@ import ( "strings" ) +func DefaultDiscoverPluggedUSBDevices() (map[string]*Device, error) { + return DiscoverPluggedUSBDevices(PathToUSBDevices) +} + func DiscoverPluggedUSBDevices(pathToUSBDevices string) (map[string]*Device, error) { devices := make(map[string]*Device) From 85b66bb7c6f86c3320f19677fc02750de3083dcc Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Thu, 15 Jan 2026 12:49:08 +0300 Subject: [PATCH 26/37] add license Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/api/sheme.go | 16 ++++++++++++++++ .../cmd/go-usbip/app/attach-info.go | 16 ++++++++++++++++ .../cmd/go-usbip/app/bind-info.go | 16 ++++++++++++++++ .../internal/plugin/publish.go | 16 ++++++++++++++++ .../controller/resourceclaim/record.go | 16 ++++++++++++++++ 5 files changed, 80 insertions(+) diff --git a/images/virtualization-dra/api/sheme.go b/images/virtualization-dra/api/sheme.go index 2a6eeb290f..573f52a64e 100644 --- a/images/virtualization-dra/api/sheme.go +++ b/images/virtualization-dra/api/sheme.go @@ -1,3 +1,19 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package api import ( diff --git a/images/virtualization-dra/cmd/go-usbip/app/attach-info.go b/images/virtualization-dra/cmd/go-usbip/app/attach-info.go index c5879f8f2c..a76bddbc93 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/attach-info.go +++ b/images/virtualization-dra/cmd/go-usbip/app/attach-info.go @@ -1,3 +1,19 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package app import ( diff --git a/images/virtualization-dra/cmd/go-usbip/app/bind-info.go b/images/virtualization-dra/cmd/go-usbip/app/bind-info.go index defdcbe703..5f0814f180 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/bind-info.go +++ b/images/virtualization-dra/cmd/go-usbip/app/bind-info.go @@ -1,3 +1,19 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package app import ( diff --git a/images/virtualization-dra/internal/plugin/publish.go b/images/virtualization-dra/internal/plugin/publish.go index ecb338b75d..d93e2bb189 100644 --- a/images/virtualization-dra/internal/plugin/publish.go +++ b/images/virtualization-dra/internal/plugin/publish.go @@ -1,3 +1,19 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package plugin import ( diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go index 3afdd5fa10..e75262faa0 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go @@ -1,3 +1,19 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package resourceclaim import ( From bd67d7e0932d8beb555f95f8f95ab652b62faa92 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Thu, 15 Jan 2026 12:49:51 +0300 Subject: [PATCH 27/37] fix long Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/cmd/go-usbip/app/app.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/virtualization-dra/cmd/go-usbip/app/app.go b/images/virtualization-dra/cmd/go-usbip/app/app.go index 90710d44a2..9fec58bb34 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/app.go +++ b/images/virtualization-dra/cmd/go-usbip/app/app.go @@ -24,7 +24,7 @@ const long = ` / _' |/ _ \ _____| | | / __| '_ \| | '_ \ | (_| | (_) |_____| |_| \__ \ |_) | | |_) | \__, | \___/ \__,_|___/_.__/|_| .__/ -|___/ |_| +|___/ |_| go-usbip is a implementation of USBIP server and client. ` From c269003089e7a3089e8ac35f3301d3352f5bbfd2 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Thu, 15 Jan 2026 20:46:23 +0300 Subject: [PATCH 28/37] impl detach unbound Signed-off-by: Yaroslav Borbat --- .../controller/resourceclaim/controller.go | 278 ++++++++++++------ .../controller/resourceclaim/record.go | 20 +- 2 files changed, 195 insertions(+), 103 deletions(-) diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go index 3b9738d727..6b225b4728 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -34,6 +34,7 @@ import ( "k8s.io/client-go/kubernetes" "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" + "k8s.io/utils/strings/slices" vdraapi "github.com/deckhouse/virtualization-dra/api" "github.com/deckhouse/virtualization-dra/internal/common" @@ -106,6 +107,13 @@ func NewController(nodeName string, podIP net.IP, usbipPort int, client kubernet return nil, fmt.Errorf("unable to add event handler to resourceclaim informer: %w", err) } + _, err = podInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{ + DeleteFunc: c.deletePod, + }) + if err != nil { + return nil, fmt.Errorf("unable to add event handler to pod informer: %w", err) + } + c.hasSynced = func() bool { return resourceClaimInformer.HasSynced() && nodeInformer.HasSynced() && podInformer.HasSynced() && resourceSliceInformer.HasSynced() } @@ -136,6 +144,18 @@ func (c *Controller) updateResourceClaim(_, newObj interface{}) { } } +func (c *Controller) deletePod(obj interface{}) { + pod, ok := obj.(*corev1.Pod) + if !ok { + return + } + for _, status := range pod.Status.ResourceClaimStatuses { + if status.ResourceClaimName != nil { + c.queueAdd(fmt.Sprintf("%s/%s", pod.Namespace, *status.ResourceClaimName)) + } + } +} + func (c *Controller) enqueueResourceClaim(rc *resourcev1beta1.ResourceClaim) { key, err := keyFunc(rc) if err != nil { @@ -203,28 +223,46 @@ func (c *Controller) sync(key string) error { log := c.log.With("key", key) log.Info("syncing resource claim") - rc, err := c.getResourceClaim(key) + rc, err := c.getMyResourceClaim(key) if err != nil { return err } if rc == nil { return nil } - if !rc.GetDeletionTimestamp().IsZero() { - return c.handleDelete(rc) - } + + resourceClaimDeleting := !rc.GetDeletionTimestamp().IsZero() pod, err := c.getReservedFor(rc) if err != nil { return err } - if pod == nil { + + podExist := pod != nil + + if !podExist && !resourceClaimDeleting { c.log.Info("no reserved pod found for resource claim, re-enqueue after 10s") c.queueAfterAdd(key, time.Second*10) return nil } - onMyNode := c.podOnMyNode(pod) + onMyNode := podExist && c.podOnMyNode(pod) + + if onMyNode && c.podFinished(pod) { + log.Info("Pod finished, detach all usb devices for this pod", + slog.String("podName", pod.Name), + slog.String("podNamespace", pod.Namespace), + ) + return c.handleClientPodFinished(pod) + } + + if resourceClaimDeleting { + log.Info("ResourceClaim is deleting, unbind all usb devices for this resource claim") + return c.handleServerDeleteResourceClaim(rc) + } + + // pod exists here + log = log.With(slog.String("podName", pod.Name), slog.String("podNamespace", pod.Namespace)) myAllocationDevices, otherAllocationDevices, err := c.getAllocationDevices(rc) if err != nil { @@ -242,7 +280,7 @@ func (c *Controller) sync(key string) error { } case shouldAttach: log.Info("attaching usb to my node") - if err = c.handleClient(rc, otherAllocationDevices); err != nil { + if err = c.handleClient(rc, otherAllocationDevices, pod); err != nil { return fmt.Errorf("failed to handle client: %w", err) } @@ -251,76 +289,62 @@ func (c *Controller) sync(key string) error { return nil } -// TODO: handle detach too -func (c *Controller) handleDelete(rc *resourcev1beta1.ResourceClaim) error { - unbound, err := c.allUnBound(rc) - if err != nil { - return err - } - if unbound { - return c.removeFinalizer(rc) - } +func (c *Controller) podFinished(pod *corev1.Pod) bool { + return !pod.GetDeletionTimestamp().IsZero() || pod.Status.Phase == corev1.PodSucceeded || pod.Status.Phase == corev1.PodFailed +} - myAllocationDevices, _, err := c.getAllocationDevices(rc) +// handle on client node, should detach all usb for this pod +func (c *Controller) handleClientPodFinished(pod *corev1.Pod) error { + err := c.recordManager.Refresh() if err != nil { - return err - } - - myAllocationDevicesByName := make(map[string]resourcev1beta1.Device) - for _, device := range myAllocationDevices { - myAllocationDevicesByName[device.Name] = device + return fmt.Errorf("failed to Refresh record: %w", err) } - shouldUpdate := false - - for i := range rc.Status.Devices { - allocatedDeviceStatus := &rc.Status.Devices[i] + ports := make(map[int]struct{}) - usbGatewayStatus, err := vdraapi.FromData(allocatedDeviceStatus.Data) - if err != nil { - return err - } - if usbGatewayStatus == nil { - continue - } - if !usbGatewayStatus.Bound { - continue - } - - device, ok := myAllocationDevicesByName[allocatedDeviceStatus.Device] - if !ok { - continue + for _, entry := range c.recordManager.GetEntries() { + if entry.PodUID == pod.UID { + if _, ok := ports[entry.Port]; ok { + continue + } + if err = c.usbIP.Detach(entry.Port); err != nil { + return fmt.Errorf("failed to detach usb: %w", err) + } + ports[entry.Port] = struct{}{} } + } - busID := "" - if attr, ok := device.Basic.Attributes["busID"]; ok && attr.StringValue != nil { - busID = *attr.StringValue - } else { - continue - } + return c.removeFinalizerForPod(pod) +} - // TODO: device can be added to other resource claims. Not supported yet. - c.log.Info("unbinding usb") - if err = c.usbIP.Unbind(busID); err != nil { - return fmt.Errorf("failed to unbind usb: %w", err) - } +// handle on server node, should unbind usb +func (c *Controller) handleServerDeleteResourceClaim(rc *resourcev1beta1.ResourceClaim) error { + infos, err := c.usbIP.GetBindInfo() + if err != nil { + return fmt.Errorf("failed to get used info: %w", err) + } - usbGatewayStatus.Bound = false - allocatedDeviceStatus.Data, err = vdraapi.ToData(usbGatewayStatus) + for _, deviceStatus := range rc.Status.Devices { + usbGatewayStatus, err := vdraapi.FromData(deviceStatus.Data) if err != nil { return err } - shouldUpdate = true - } - if shouldUpdate { - _, err = c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).UpdateStatus(context.Background(), rc, metav1.UpdateOptions{}) - if err != nil { - return fmt.Errorf("failed to update resource claim status: %w", err) + busID := usbGatewayStatus.BusID + + for _, info := range infos { + if info.BusID == busID { + if info.Bound { + err = c.usbIP.Unbind(busID) + if err != nil { + return fmt.Errorf("failed to unbind usb: %w", err) + } + } + } } } - return nil + return c.removeFinalizerForResourceClaim(rc) } func (c *Controller) allUnBound(rc *resourcev1beta1.ResourceClaim) (bool, error) { @@ -340,35 +364,32 @@ func (c *Controller) allUnBound(rc *resourcev1beta1.ResourceClaim) (bool, error) return true, nil } -func (c *Controller) addFinalizer(rc *resourcev1beta1.ResourceClaim) error { - var newFinalizers []string - for _, fin := range rc.GetFinalizers() { - if fin == finalizer { - return nil - } - newFinalizers = append(newFinalizers, fin) +func (c *Controller) addFinalizerForResourceClaim(rc *resourcev1beta1.ResourceClaim) (err error) { + if addFinalizer(rc) { + _, err = c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).Update(context.Background(), rc, metav1.UpdateOptions{}) } - newFinalizers = append(newFinalizers, finalizer) - rc.SetFinalizers(newFinalizers) - _, err := c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).Update(context.Background(), rc, metav1.UpdateOptions{}) - return err + return } -func (c *Controller) removeFinalizer(rc *resourcev1beta1.ResourceClaim) error { - var newFinalizers []string - for _, fin := range rc.GetFinalizers() { - if fin == finalizer { - continue - } - newFinalizers = append(newFinalizers, fin) +func (c *Controller) removeFinalizerForResourceClaim(rc *resourcev1beta1.ResourceClaim) (err error) { + if removeFinalizer(rc) { + _, err = c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).Update(context.Background(), rc, metav1.UpdateOptions{}) } - if len(newFinalizers) == len(rc.GetFinalizers()) { - return nil + return +} + +func (c *Controller) addFinalizerForPod(pod *corev1.Pod) (err error) { + if addFinalizer(pod) { + _, err = c.client.CoreV1().Pods(pod.Namespace).Update(context.Background(), pod, metav1.UpdateOptions{}) } + return +} - rc.SetFinalizers(newFinalizers) - _, err := c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).Update(context.Background(), rc, metav1.UpdateOptions{}) - return err +func (c *Controller) removeFinalizerForPod(pod *corev1.Pod) (err error) { + if removeFinalizer(pod) { + _, err = c.client.CoreV1().Pods(pod.Namespace).Update(context.Background(), pod, metav1.UpdateOptions{}) + } + return } func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocationDevices []resourcev1beta1.Device) error { @@ -462,7 +483,7 @@ func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocatio } if shouldUpdate { - err := c.addFinalizer(rc) + err := c.addFinalizerForResourceClaim(rc) if err != nil { return err } @@ -475,7 +496,7 @@ func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocatio return nil } -func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAllocationDevices []resourcev1beta1.Device) error { +func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAllocationDevices []resourcev1beta1.Device, pod *corev1.Pod) error { indexAllocDevice := make(map[string]int) for i, allocDeviceStatus := range rc.Status.Devices { indexAllocDevice[allocDeviceStatus.Device] = i @@ -517,7 +538,7 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca return fmt.Errorf("failed to Refresh record: %w", err) } - if err = c.attach(busID, usbGatewayStatus); err != nil { + if err = c.attach(busID, usbGatewayStatus, rc.UID, pod.UID); err != nil { return fmt.Errorf("failed to attach usb: %w", err) } @@ -529,7 +550,12 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca } if shouldUpdate { - _, err := c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).UpdateStatus(context.Background(), rc, metav1.UpdateOptions{}) + err := c.addFinalizerForPod(pod) + if err != nil { + return fmt.Errorf("failed to add finalizer for pod: %s/%s: %w", pod.Namespace, pod.Name, err) + } + + _, err = c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).UpdateStatus(context.Background(), rc, metav1.UpdateOptions{}) if err != nil { return fmt.Errorf("failed to update resource claim status: %w", err) } @@ -538,7 +564,7 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca return nil } -func (c *Controller) attach(busID string, usbGatewayStatus *vdraapi.USBGatewayStatus) error { +func (c *Controller) attach(busID string, usbGatewayStatus *vdraapi.USBGatewayStatus, claimUID, podUID types.UID) error { entries := c.recordManager.GetEntries() for _, entry := range entries { if entry.BusID == busID { @@ -547,6 +573,7 @@ func (c *Controller) attach(busID string, usbGatewayStatus *vdraapi.USBGatewaySt return err } } + // already attached return nil } } @@ -584,11 +611,13 @@ func (c *Controller) attach(busID string, usbGatewayStatus *vdraapi.USBGatewaySt } entry := Entry{ - Port: rhport, - RemotePort: usbGatewayStatus.RemotePort, - RemoteIP: usbGatewayStatus.RemoteIP, - RemoteBusID: busID, - BusID: usedInfo.LocalBusID, + Port: rhport, + RemotePort: usbGatewayStatus.RemotePort, + RemoteIP: usbGatewayStatus.RemoteIP, + RemoteBusID: busID, + BusID: usedInfo.LocalBusID, + ResourceClaimUID: claimUID, + PodUID: podUID, } if err = c.recordManager.AddEntry(entry); err != nil { @@ -608,7 +637,7 @@ func (c *Controller) detach(port int) error { return nil } -func (c *Controller) getResourceClaim(key string) (*resourcev1beta1.ResourceClaim, error) { +func (c *Controller) getMyResourceClaim(key string) (*resourcev1beta1.ResourceClaim, error) { obj, exists, err := c.resourceClaimIndexer.GetByKey(key) if err != nil && !k8serrors.IsNotFound(err) { return nil, fmt.Errorf("failed to get resourceclaim: %w", err) @@ -622,7 +651,31 @@ func (c *Controller) getResourceClaim(key string) (*resourcev1beta1.ResourceClai return nil, fmt.Errorf("unexpected type of resourceclaim: %T", obj) } - return rc.DeepCopy(), nil + if c.isMyResourceClaim(rc) { + return rc.DeepCopy(), nil + } + + return nil, nil +} + +func (c *Controller) isMyResourceClaim(rc *resourcev1beta1.ResourceClaim) bool { + if rc == nil { + return false + } + if slices.Contains(rc.GetFinalizers(), finalizer) { + return true + } + if rc.Status.Allocation == nil { + return false + } + for _, status := range rc.Status.Allocation.Devices.Results { + // now, driver virtualization-dra supports only usb, but we can add more devices later + // so we need to check if the device is usb + if status.Driver == common.VirtualizationDraPluginName && strings.HasPrefix(status.Device, "usb") { + return true + } + } + return false } func (c *Controller) getPod(name, namespace string) (*corev1.Pod, error) { @@ -717,12 +770,12 @@ func (c *Controller) getAllocationDevices(rc *resourcev1beta1.ResourceClaim) ([] var otherDevices []resourcev1beta1.Device for pool, allocResultsByDevice := range allocResultsByPool { - slices, ok := byPoolSlices[pool] + resourceSlices, ok := byPoolSlices[pool] if !ok { return nil, nil, fmt.Errorf("no resource slices found for pool %s", pool) } - for _, slice := range slices { + for _, slice := range resourceSlices { for _, device := range slice.Spec.Devices { allocResult, ok := allocResultsByDevice[device.Name] if !ok { @@ -741,3 +794,32 @@ func (c *Controller) getAllocationDevices(rc *resourcev1beta1.ResourceClaim) ([] return myDevices, otherDevices, nil } + +func addFinalizer(obj metav1.Object) bool { + var newFinalizers []string + for _, fin := range obj.GetFinalizers() { + if fin == finalizer { + return false + } + newFinalizers = append(newFinalizers, fin) + } + newFinalizers = append(newFinalizers, finalizer) + obj.SetFinalizers(newFinalizers) + return true +} + +func removeFinalizer(obj metav1.Object) bool { + var newFinalizers []string + for _, fin := range obj.GetFinalizers() { + if fin == finalizer { + continue + } + newFinalizers = append(newFinalizers, fin) + } + if len(newFinalizers) == len(obj.GetFinalizers()) { + return false + } + + obj.SetFinalizers(newFinalizers) + return true +} diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go index e75262faa0..bf81716002 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/record.go @@ -24,6 +24,8 @@ import ( "slices" "sync" + "k8s.io/apimachinery/pkg/types" + "github.com/deckhouse/virtualization-dra/internal/usbip" ) @@ -34,11 +36,13 @@ type record struct { } type Entry struct { - Port int `json:"port"` - RemotePort int `json:"remotePort" json:"remotePort"` - RemoteIP string `json:"remoteIP" json:"remoteIP"` - RemoteBusID string `json:"remoteBusID" json:"remoteBusID"` - BusID string `json:"busID" json:"busID"` + Port int `json:"port"` + RemotePort int `json:"remotePort"` + RemoteIP string `json:"remoteIP"` + RemoteBusID string `json:"remoteBusID"` + BusID string `json:"busID"` + ResourceClaimUID types.UID `json:"resourceClaimUID"` + PodUID types.UID `json:"podUID"` } func (e Entry) Validate() error { @@ -57,6 +61,12 @@ func (e Entry) Validate() error { if e.BusID == "" { return fmt.Errorf("busID is required") } + if e.ResourceClaimUID == "" { + return fmt.Errorf("resourceClaimUID is required") + } + if e.PodUID == "" { + return fmt.Errorf("podUID is required") + } return nil } From 8cd2b0c4eecc4c1a05e18e9aca631158efca56bb Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Thu, 15 Jan 2026 22:11:29 +0300 Subject: [PATCH 29/37] add info subcommands Signed-off-by: Yaroslav Borbat --- .../cmd/go-usbip/app/app.go | 45 ++++++++++++- .../cmd/go-usbip/app/attach-info.go | 12 +--- .../cmd/go-usbip/app/bind-info.go | 12 +--- .../cmd/go-usbip/app/info.go | 65 +++++++++++++++++++ .../internal/usbip/binder.go | 44 ++++++++----- images/virtualization-dra/pkg/usb/usb.go | 6 +- 6 files changed, 142 insertions(+), 42 deletions(-) create mode 100644 images/virtualization-dra/cmd/go-usbip/app/info.go diff --git a/images/virtualization-dra/cmd/go-usbip/app/app.go b/images/virtualization-dra/cmd/go-usbip/app/app.go index 9fec58bb34..c5604c78b7 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/app.go +++ b/images/virtualization-dra/cmd/go-usbip/app/app.go @@ -16,7 +16,14 @@ limitations under the License. package app -import "github.com/spf13/cobra" +import ( + "encoding/json" + "fmt" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "sigs.k8s.io/yaml" +) const long = ` _ _ @@ -24,7 +31,7 @@ const long = ` / _' |/ _ \ _____| | | / __| '_ \| | '_ \ | (_| | (_) |_____| |_| \__ \ |_) | | |_) | \__, | \___/ \__,_|___/_.__/|_| .__/ -|___/ |_| +|___/ |_| go-usbip is a implementation of USBIP server and client. ` @@ -46,7 +53,41 @@ func NewUSBIPCommand() *cobra.Command { NewDetachCommand(), NewAttachInfoCommand(), NewBindInfoCommand(), + NewInfoCommand(), ) + printer.AddFlags(cmd.PersistentFlags()) + return cmd } + +var printer = &printOptions{} + +type printOptions struct { + output string +} + +func (o *printOptions) AddFlags(fs *pflag.FlagSet) { + fs.StringVarP(&o.output, "output", "o", "json", "Output format") +} + +func (o *printOptions) PrintObject(cmd *cobra.Command, data interface{}) error { + switch o.output { + case "json": + b, err := json.MarshalIndent(data, "", " ") + if err != nil { + return fmt.Errorf("failed to marshal json: %w", err) + } + cmd.Println(string(b)) + return nil + case "yaml": + b, err := yaml.Marshal(data) + if err != nil { + return fmt.Errorf("failed to marshal yaml: %w", err) + } + cmd.Println(string(b)) + return nil + default: + return fmt.Errorf("unsupported format %q. Supported formats: [json, yaml]", o.output) + } +} diff --git a/images/virtualization-dra/cmd/go-usbip/app/attach-info.go b/images/virtualization-dra/cmd/go-usbip/app/attach-info.go index a76bddbc93..0953f1e490 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/attach-info.go +++ b/images/virtualization-dra/cmd/go-usbip/app/attach-info.go @@ -17,9 +17,6 @@ limitations under the License. package app import ( - "encoding/json" - "fmt" - "github.com/spf13/cobra" "github.com/deckhouse/virtualization-dra/internal/usbip" @@ -51,12 +48,5 @@ func (o *attachInfoOptions) Run(cmd *cobra.Command, _ []string) error { return err } - bytes, err := json.Marshal(infos) - if err != nil { - return fmt.Errorf("failed to marshal json: %w", err) - } - - cmd.Println(string(bytes)) - - return nil + return printer.PrintObject(cmd, infos) } diff --git a/images/virtualization-dra/cmd/go-usbip/app/bind-info.go b/images/virtualization-dra/cmd/go-usbip/app/bind-info.go index 5f0814f180..b446ccf5b2 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/bind-info.go +++ b/images/virtualization-dra/cmd/go-usbip/app/bind-info.go @@ -17,9 +17,6 @@ limitations under the License. package app import ( - "encoding/json" - "fmt" - "github.com/spf13/cobra" "github.com/deckhouse/virtualization-dra/internal/usbip" @@ -51,12 +48,5 @@ func (o *bindInfoOptions) Run(cmd *cobra.Command, _ []string) error { return err } - bytes, err := json.Marshal(infos) - if err != nil { - return fmt.Errorf("failed to marshal json: %w", err) - } - - cmd.Println(string(bytes)) - - return nil + return printer.PrintObject(cmd, infos) } diff --git a/images/virtualization-dra/cmd/go-usbip/app/info.go b/images/virtualization-dra/cmd/go-usbip/app/info.go new file mode 100644 index 0000000000..434e03823e --- /dev/null +++ b/images/virtualization-dra/cmd/go-usbip/app/info.go @@ -0,0 +1,65 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package app + +import ( + "cmp" + "slices" + + "github.com/spf13/cobra" + + "github.com/deckhouse/virtualization-dra/pkg/usb" +) + +func NewInfoCommand() *cobra.Command { + o := &infoOptions{} + cmd := &cobra.Command{ + Use: "info", + Short: "Get info", + Example: o.Usage(), + RunE: o.Run, + } + + return cmd +} + +type infoOptions struct{} + +func (o *infoOptions) Usage() string { + return ` # Get info + $ go-usbip info +` +} + +func (o *infoOptions) Run(cmd *cobra.Command, _ []string) error { + discoverDevices, err := usb.DefaultDiscoverPluggedUSBDevices() + if err != nil { + return err + } + + devices := make([]*usb.Device, 0, len(discoverDevices)) + + for _, device := range discoverDevices { + devices = append(devices, device) + } + + slices.SortFunc(devices, func(a, b *usb.Device) int { + return cmp.Compare(a.Path, b.Path) + }) + + return printer.PrintObject(cmd, devices) +} diff --git a/images/virtualization-dra/internal/usbip/binder.go b/images/virtualization-dra/internal/usbip/binder.go index 647f1fc6f0..aac13bc8d2 100644 --- a/images/virtualization-dra/internal/usbip/binder.go +++ b/images/virtualization-dra/internal/usbip/binder.go @@ -17,6 +17,7 @@ limitations under the License. package usbip import ( + "bufio" "fmt" "os" "path/filepath" @@ -68,7 +69,7 @@ func (b *usbBinder) Unbind(busID string) error { return fmt.Errorf("device with bus ID %s does not exist: %w", busID, err) } - if b.isBound(devInfo) { + if !b.isBound(devInfo) { return fmt.Errorf("device %s is not bound to %s driver", devInfo.BusID, usbipHostDriverName) } @@ -87,7 +88,6 @@ func (b *usbBinder) Unbind(busID string) error { } return nil - // return b.storeBind(busID, false) } func (b *usbBinder) IsBound(busID string) (bool, error) { @@ -145,23 +145,37 @@ func (b *usbBinder) getUSBDeviceInfo(busID string) (*usbDeviceInfo, error) { } bDevClassPath := filepath.Join(path, "bDeviceClass") - if data, err := os.ReadFile(bDevClassPath); err == nil { - info.IsHub = strings.TrimSpace(string(data)) == "09" // 09 = USB Hub class + data, err := os.ReadFile(bDevClassPath) + if err != nil { + return nil, fmt.Errorf("failed to read %s: %w", bDevClassPath, err) } + info.IsHub = strings.TrimSpace(string(data)) == "09" // 09 = USB Hub class - driverLink := filepath.Join(path, "driver") - if link, err := os.Readlink(driverLink); err == nil { - info.Driver = filepath.Base(link) + ueventPath := filepath.Join(path, "uevent") + ueventFile, err := os.Open(ueventPath) + if err != nil { + return nil, fmt.Errorf("unable to open the file %s: %w", ueventPath, err) } + defer ueventFile.Close() + scanner := bufio.NewScanner(ueventFile) - ueventPath := filepath.Join(path, "uevent") - if data, err := os.ReadFile(ueventPath); err == nil { - lines := strings.Split(string(data), "\n") - for _, line := range lines { - if strings.HasPrefix(line, "DEVNAME=") { - info.DevPath = filepath.Join("/dev", strings.TrimPrefix(line, "DEVNAME=")) - break - } + count := 0 + for scanner.Scan() { + line := scanner.Text() + values := strings.Split(line, "=") + if len(values) != 2 { + continue + } + switch values[0] { + case "DEVNAME": + info.DevPath = filepath.Join("/dev", values[1]) + count++ + case "DRIVER": + info.Driver = values[1] + count++ + } + if count == 2 { + break } } diff --git a/images/virtualization-dra/pkg/usb/usb.go b/images/virtualization-dra/pkg/usb/usb.go index ae220ea329..fc6c474ab6 100644 --- a/images/virtualization-dra/pkg/usb/usb.go +++ b/images/virtualization-dra/pkg/usb/usb.go @@ -161,9 +161,10 @@ func parseSysUeventFile(path string, device *Device) error { // TYPE=0/0/0 // BUSNUM=003 // DEVNUM=002 - file, err := os.Open(filepath.Join(path, "uevent")) + ueventPath := filepath.Join(path, "uevent") + file, err := os.Open(ueventPath) if err != nil { - return fmt.Errorf("unable to open the file %s: %w", path, err) + return fmt.Errorf("unable to open the file %s: %w", ueventPath, err) } defer file.Close() @@ -252,7 +253,6 @@ func parseSysUeventFile(path string, device *Device) error { } device.DeviceNumber = uint32(val) default: - slog.Info("Skipping unhandled line", slog.String("line", line)) } } return nil From 43868dc3d852a236d4e88cc5b73cb46d43ff361e Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Thu, 15 Jan 2026 23:34:40 +0300 Subject: [PATCH 30/37] fix waiting attach Signed-off-by: Yaroslav Borbat --- .../controller/resourceclaim/controller.go | 41 ++++++++++--------- .../internal/usbip/attacher.go | 9 ++-- 2 files changed, 26 insertions(+), 24 deletions(-) diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go index 6b225b4728..cb8b662908 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -201,7 +201,7 @@ func (c *Controller) worker(ctx context.Context) { } defer c.queue.Done(key) - if err := c.sync(key); err != nil { + if err := c.sync(ctx, key); err != nil { c.log.Error("re-enqueuing", slog.String("key", key), slog.Any("err", err)) c.queue.AddRateLimited(key) } else { @@ -219,7 +219,7 @@ func (c *Controller) worker(ctx context.Context) { } } -func (c *Controller) sync(key string) error { +func (c *Controller) sync(ctx context.Context, key string) error { log := c.log.With("key", key) log.Info("syncing resource claim") @@ -280,7 +280,7 @@ func (c *Controller) sync(key string) error { } case shouldAttach: log.Info("attaching usb to my node") - if err = c.handleClient(rc, otherAllocationDevices, pod); err != nil { + if err = c.handleClient(ctx, rc, otherAllocationDevices, pod); err != nil { return fmt.Errorf("failed to handle client: %w", err) } @@ -496,7 +496,7 @@ func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocatio return nil } -func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAllocationDevices []resourcev1beta1.Device, pod *corev1.Pod) error { +func (c *Controller) handleClient(ctx context.Context, rc *resourcev1beta1.ResourceClaim, otherAllocationDevices []resourcev1beta1.Device, pod *corev1.Pod) error { indexAllocDevice := make(map[string]int) for i, allocDeviceStatus := range rc.Status.Devices { indexAllocDevice[allocDeviceStatus.Device] = i @@ -538,7 +538,7 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca return fmt.Errorf("failed to Refresh record: %w", err) } - if err = c.attach(busID, usbGatewayStatus, rc.UID, pod.UID); err != nil { + if err = c.attach(ctx, busID, usbGatewayStatus, rc.UID, pod.UID); err != nil { return fmt.Errorf("failed to attach usb: %w", err) } @@ -564,7 +564,7 @@ func (c *Controller) handleClient(rc *resourcev1beta1.ResourceClaim, otherAlloca return nil } -func (c *Controller) attach(busID string, usbGatewayStatus *vdraapi.USBGatewayStatus, claimUID, podUID types.UID) error { +func (c *Controller) attach(ctx context.Context, busID string, usbGatewayStatus *vdraapi.USBGatewayStatus, claimUID, podUID types.UID) error { entries := c.recordManager.GetEntries() for _, entry := range entries { if entry.BusID == busID { @@ -594,21 +594,24 @@ func (c *Controller) attach(busID string, usbGatewayStatus *vdraapi.USBGatewaySt return fmt.Errorf("failed to attach usb: %w", err) } - infos, err := c.usbIP.GetAttachInfo() - if err != nil { - return fmt.Errorf("failed to get used info: %w", err) - } - + // command attach was successful, but we need to wait until usb is real attached var usedInfo *usbip.AttachInfo - for _, info := range infos { - if info.Port == rhport { - usedInfo = &info - break + err = wait.PollUntilContextCancel(ctx, time.Second, true, func(ctx context.Context) (bool, error) { + c.log.Info("Get attach info for store localBusID") + infos, err := c.usbIP.GetAttachInfo() + if err != nil { + c.log.Info("Failed to get used info", slog.String("error", err.Error())) + return false, nil } - } - if usedInfo == nil { - return fmt.Errorf("failed to find used info for port %d", rhport) - } + for _, info := range infos { + if info.Port == rhport { + usedInfo = &info + return true, nil + } + } + c.log.Info("Usb are not attached yet") + return false, nil + }) entry := Entry{ Port: rhport, diff --git a/images/virtualization-dra/internal/usbip/attacher.go b/images/virtualization-dra/internal/usbip/attacher.go index 9c7df3daf6..1bc9246ba5 100644 --- a/images/virtualization-dra/internal/usbip/attacher.go +++ b/images/virtualization-dra/internal/usbip/attacher.go @@ -68,13 +68,12 @@ func (a usbAttacher) Detach(port int) error { if idev.port == port { found = true vstatus := protocol.DeviceStatus(idev.status) - if vstatus != protocol.VDeviceStatusNull { - break + if vstatus == protocol.VDeviceStatusNull { + slog.Info("Port is already detached", slog.Int("port", port)) + return fmt.Errorf("port is already detached") } - slog.Info("Port is already detached", slog.Int("port", port)) - return fmt.Errorf("port is already detached") - + break } } From e2d62f757a58cf4e55343d87e1901129702d33be Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Fri, 23 Jan 2026 13:16:57 +0300 Subject: [PATCH 31/37] add usb monitor Signed-off-by: Yaroslav Borbat --- .../cmd/usb-gateway/app/app.go | 2 +- .../cmd/virtualization-dra-plugin/app/app.go | 5 +- .../controller/resourceclaim/controller.go | 7 +- .../virtualization-dra/internal/usb/device.go | 4 +- .../internal/usb/discovery.go | 35 +- .../internal/usb/monitor.go | 169 --------- .../internal/usb/monitor_test.go | 53 --- .../virtualization-dra/internal/usb/store.go | 58 +-- .../internal/usbip/attacher.go | 29 +- .../internal/usbip/binder.go | 11 +- images/virtualization-dra/pkg/udev/conn.go | 115 ++++++ images/virtualization-dra/pkg/udev/matcher.go | 90 +++++ images/virtualization-dra/pkg/udev/monitor.go | 140 +++++++ images/virtualization-dra/pkg/udev/uevent.go | 122 ++++++ images/virtualization-dra/pkg/usb/monitor.go | 351 ++++++++++++++---- 15 files changed, 815 insertions(+), 376 deletions(-) delete mode 100644 images/virtualization-dra/internal/usb/monitor.go delete mode 100644 images/virtualization-dra/internal/usb/monitor_test.go create mode 100644 images/virtualization-dra/pkg/udev/conn.go create mode 100644 images/virtualization-dra/pkg/udev/matcher.go create mode 100644 images/virtualization-dra/pkg/udev/monitor.go create mode 100644 images/virtualization-dra/pkg/udev/uevent.go diff --git a/images/virtualization-dra/cmd/usb-gateway/app/app.go b/images/virtualization-dra/cmd/usb-gateway/app/app.go index 4f0a92dcf4..a5dcf56f7e 100644 --- a/images/virtualization-dra/cmd/usb-gateway/app/app.go +++ b/images/virtualization-dra/cmd/usb-gateway/app/app.go @@ -113,7 +113,7 @@ func (o *usbOptions) Validate() error { } func (o *usbOptions) Run(cmd *cobra.Command, _ []string) error { - monitor, err := usb.NewMonitor(cmd.Context(), o.USBResyncPeriod) + monitor, err := usb.NewMonitor(cmd.Context(), usb.WithResyncPeriod(o.USBResyncPeriod)) if err != nil { return err } diff --git a/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go b/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go index a7a69110b1..a8810f4933 100644 --- a/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go +++ b/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go @@ -76,7 +76,6 @@ func newDraOptions() *draOptions { CDIRoot: withDefault("CDI_ROOT", cdi.SpecDir), KubeletRegisterDirectoryPath: os.Getenv("KUBELET_REGISTER_DIRECTORY_PATH"), KubeletPluginsDirectoryPath: os.Getenv("KUBELET_PLUGINS_DIRECTORY_PATH"), - USBDevicesPath: withDefault("USB_DEVICES_PATH", usb.PathToUSBDevices), HealthzPort: 51515, USBResyncPeriod: usb.DefaultResyncPeriod, Logging: &logger.Options{}, @@ -99,7 +98,6 @@ type draOptions struct { CDIRoot string KubeletRegisterDirectoryPath string KubeletPluginsDirectoryPath string - USBDevicesPath string HealthzPort int USBResyncPeriod time.Duration @@ -114,7 +112,6 @@ func (o *draOptions) NamedFlags() (fs flag.NamedFlagSets) { mfs.StringVar(&o.CDIRoot, "cdi-root", o.CDIRoot, "CDI root") mfs.StringVar(&o.KubeletRegisterDirectoryPath, "kubelet-register-directory-path", o.KubeletRegisterDirectoryPath, "Kubelet register directory path") mfs.StringVar(&o.KubeletPluginsDirectoryPath, "kubelet-plugins-directory-path", o.KubeletPluginsDirectoryPath, "Kubelet plugins directory path") - mfs.StringVar(&o.USBDevicesPath, "usb-devices-path", o.USBDevicesPath, "USB Devices path") mfs.IntVar(&o.HealthzPort, "healthz-port", o.HealthzPort, "Healthz port") mfs.DurationVar(&o.USBResyncPeriod, "usb-resync-period", o.USBResyncPeriod, "USB resync period") @@ -160,7 +157,7 @@ func (o *draOptions) Run(cmd *cobra.Command, _ []string) error { return fmt.Errorf("failed to create CDI manager: %w", err) } - usbStore := usb.NewAllocationStore(o.NodeName, o.USBDevicesPath, o.USBResyncPeriod, usbCDIManager, slog.Default()) + usbStore := usb.NewAllocationStore(o.NodeName, o.USBResyncPeriod, usbCDIManager, slog.Default()) driver := plugin.NewDriver(o.NodeName, client, usbStore, slog.Default()) err = driver.Start(cmd.Context()) diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go index cb8b662908..e43f5ece78 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -247,7 +247,6 @@ func (c *Controller) sync(ctx context.Context, key string) error { } onMyNode := podExist && c.podOnMyNode(pod) - if onMyNode && c.podFinished(pod) { log.Info("Pod finished, detach all usb devices for this pod", slog.String("podName", pod.Name), @@ -257,6 +256,11 @@ func (c *Controller) sync(ctx context.Context, key string) error { } if resourceClaimDeleting { + if podExist { + log.Info("Pod exists, waiting for pod to be deleted") + c.queueAfterAdd(key, time.Second*10) + return nil + } log.Info("ResourceClaim is deleting, unbind all usb devices for this resource claim") return c.handleServerDeleteResourceClaim(rc) } @@ -283,7 +287,6 @@ func (c *Controller) sync(ctx context.Context, key string) error { if err = c.handleClient(ctx, rc, otherAllocationDevices, pod); err != nil { return fmt.Errorf("failed to handle client: %w", err) } - } return nil diff --git a/images/virtualization-dra/internal/usb/device.go b/images/virtualization-dra/internal/usb/device.go index efd8f5acb0..230ecb9773 100644 --- a/images/virtualization-dra/internal/usb/device.go +++ b/images/virtualization-dra/internal/usb/device.go @@ -28,8 +28,8 @@ import ( type DeviceSet = sets.Set[Device] -func NewDeviceSet() DeviceSet { - return sets.New[Device]() +func NewDeviceSet(devices ...Device) DeviceSet { + return sets.New[Device](devices...) } type Device struct { diff --git a/images/virtualization-dra/internal/usb/discovery.go b/images/virtualization-dra/internal/usb/discovery.go index 0eb5a59245..9fde30965f 100644 --- a/images/virtualization-dra/internal/usb/discovery.go +++ b/images/virtualization-dra/internal/usb/discovery.go @@ -18,47 +18,30 @@ package usb import ( "github.com/deckhouse/virtualization-dra/internal/featuregates" - "github.com/deckhouse/virtualization-dra/internal/usbip" - "github.com/deckhouse/virtualization-dra/pkg/usb" ) -const PathToUSBDevices = usb.PathToUSBDevices +func (s *AllocationStore) discoveryPluggedUSBDevices() (DeviceSet, DeviceSet, error) { + allUSBDevices := s.monitor.GetDevices() -func newDiscoverer() discoverer { - return discoverer{ - getter: usbip.NewUSBAttacher(), - } -} - -type discoverer struct { - getter usbip.AttachInfoGetter -} - -func (d *discoverer) DiscoveryPluggedUSBDevices(pathToUSBDevices string) (DeviceSet, DeviceSet, error) { - devices, err := usb.DiscoverPluggedUSBDevices(pathToUSBDevices) - if err != nil { - return nil, nil, err - } - - busIdMaps := make(map[string]struct{}) + busIDSet := make(map[string]struct{}) if featuregates.Default().USBGatewayEnabled() { - infos, err := d.getter.GetAttachInfo() + infos, err := s.usbipInfoGetter.GetAttachInfo() if err != nil { return nil, nil, err } for _, info := range infos { - busIdMaps[info.LocalBusID] = struct{}{} + busIDSet[info.LocalBusID] = struct{}{} } } usbDeviceSet := NewDeviceSet() usbipDeviceSet := NewDeviceSet() - for _, device := range devices { - if _, ok := busIdMaps[device.BusID]; ok { - usbipDeviceSet.Insert(toDevice(device)) + for _, device := range allUSBDevices { + if _, ok := busIDSet[device.BusID]; ok { + usbipDeviceSet.Insert(toDevice(&device)) } else { - usbDeviceSet.Insert(toDevice(device)) + usbDeviceSet.Insert(toDevice(&device)) } } diff --git a/images/virtualization-dra/internal/usb/monitor.go b/images/virtualization-dra/internal/usb/monitor.go deleted file mode 100644 index 356f7967be..0000000000 --- a/images/virtualization-dra/internal/usb/monitor.go +++ /dev/null @@ -1,169 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package usb - -import ( - "context" - "log/slog" - "time" - - "github.com/godbus/dbus/v5" -) - -type monitor struct { - callback monitorCallback - log *slog.Logger -} - -type monitorCallback struct { - Add func() - Update func() - Delete func() -} - -func newUSBMonitor(callback monitorCallback) *monitor { - return &monitor{ - callback: callback, - log: slog.With(slog.String("component", "usb-monitor")), - } -} - -func (m *monitor) Start(ctx context.Context) { - go func() { - for { - select { - case <-ctx.Done(): - return - case <-time.After(5 * time.Second): - if err := m.run(ctx); err != nil { - m.log.Error("failed to run monitor", slog.Any("err", err)) - } - } - } - }() -} - -func (m *monitor) run(ctx context.Context) error { - conn, err := dbus.ConnectSystemBus(dbus.WithContext(ctx)) - if err != nil { - return err - } - - rules := []string{ - "type='signal',interface='org.freedesktop.DBus.ObjectManager',member='InterfacesAdded'", - "type='signal',interface='org.freedesktop.DBus.ObjectManager',member='InterfacesRemoved'", - } - - for _, rule := range rules { - call := conn.BusObject().Call("org.freedesktop.DBus.AddMatch", 0, rule) - if call.Err != nil { - m.log.Error("Failed to add rule", slog.String("rule", rule), slog.Any("err", call.Err)) - } - } - - signals := make(chan *dbus.Signal, 100) - conn.Signal(signals) - - m.log.Info("Starting USB Monitor...") - defer m.log.Info("Stopping USB Monitor...") - - for { - select { - case <-ctx.Done(): - return nil - case signal := <-signals: - m.handleSignal(signal) - } - } -} - -func (m *monitor) handleSignal(signal *dbus.Signal) { - switch signal.Name { - case "org.freedesktop.DBus.ObjectManager.InterfacesAdded": - if len(signal.Body) >= 2 { - path, ok := signal.Body[0].(dbus.ObjectPath) - if !ok { - return - } - interfaces, ok := signal.Body[1].(map[string]map[string]dbus.Variant) - if !ok { - return - } - - if m.isUSBDevice(interfaces) { - slog.Info("USB Device connected", slog.String("path", string(path))) - if m.callback.Add != nil { - m.callback.Add() - } - } - } - - case "org.freedesktop.DBus.ObjectManager.InterfacesRemoved": - // Sender = {string} ":1.10" - // Path = {dbus.ObjectPath} "/org/freedesktop/UDisks2" - // Name = {string} "org.freedesktop.DBus.ObjectManager.InterfacesRemoved" - // Body = {[]interface{}} - // Body[0] = interface{} | dbus.ObjectPath "/org/freedesktop/UDisks2/block_devices/sda2" - // Body[1] = interface{} | []string{"org.freedesktop.UDisks2.Filesystem","org.freedesktop.UDisks2.Partition","org.freedesktop.UDisks2.Block","org.freedesktop.UDisks2.Drive"} - - if len(signal.Body) >= 2 { - path, ok := signal.Body[0].(dbus.ObjectPath) - if !ok { - return - } - interfaces, ok := signal.Body[1].([]string) - if !ok { - return - } - - if m.containsUSBInterface(interfaces) { - slog.Info("USB Device disconnected", slog.String("path", string(path))) - if m.callback.Delete != nil { - m.callback.Delete() - } - } - } - } -} - -func (m *monitor) isUSBDevice(interfaces map[string]map[string]dbus.Variant) bool { - if drive, ok := interfaces["org.freedesktop.UDisks2.Drive"]; ok { - if connectionBus, ok := drive["ConnectionBus"]; ok { - bus, _ := connectionBus.Value().(string) - return bus == "usb" - } - } - - if block, ok := interfaces["org.freedesktop.UDisks2.Block"]; ok { - if _, ok := block["Drive"]; ok { - // if it has a link to drive, it can be USB - return true - } - } - - return false -} - -func (m *monitor) containsUSBInterface(interfaces []string) bool { - for _, iface := range interfaces { - if iface == "org.freedesktop.UDisks2.Drive" || - iface == "org.freedesktop.UDisks2.Block" { - return true - } - } - return false -} diff --git a/images/virtualization-dra/internal/usb/monitor_test.go b/images/virtualization-dra/internal/usb/monitor_test.go deleted file mode 100644 index d01ea54377..0000000000 --- a/images/virtualization-dra/internal/usb/monitor_test.go +++ /dev/null @@ -1,53 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package usb - -import ( - "context" - "fmt" - "testing" - "time" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestMonitor(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "USB Monitor Suite") -} - -var _ = Describe("USB Monitor", func() { - - It("should add a new device", func() { - - monitor := newUSBMonitor(monitorCallback{ - Add: func() { - fmt.Println("USB Device connected") - }, - Update: func() { - fmt.Println("USB Device updated") - }, - Delete: func() { - fmt.Println("USB Device disconnected") - }, - }) - - monitor.Start(context.Background()) - time.Sleep(10 * time.Minute) - }) -}) diff --git a/images/virtualization-dra/internal/usb/store.go b/images/virtualization-dra/internal/usb/store.go index 1d3a55689c..40d7314180 100644 --- a/images/virtualization-dra/internal/usb/store.go +++ b/images/virtualization-dra/internal/usb/store.go @@ -37,20 +37,19 @@ import ( vdraapi "github.com/deckhouse/virtualization-dra/api" "github.com/deckhouse/virtualization-dra/internal/cdi" "github.com/deckhouse/virtualization-dra/internal/featuregates" + "github.com/deckhouse/virtualization-dra/internal/usbip" + "github.com/deckhouse/virtualization-dra/pkg/usb" ) const DefaultResyncPeriod = 10 * time.Minute -func NewAllocationStore(nodeName, devicesPath string, resyncPeriod time.Duration, cdiManager cdi.Manager, log *slog.Logger) *AllocationStore { +func NewAllocationStore(nodeName string, resyncPeriod time.Duration, cdiManager cdi.Manager, log *slog.Logger) *AllocationStore { if resyncPeriod == 0 { resyncPeriod = DefaultResyncPeriod } - if devicesPath == "" { - devicesPath = PathToUSBDevices - } + store := &AllocationStore{ nodeName: nodeName, - devicesPath: devicesPath, resyncPeriod: resyncPeriod, cdi: cdiManager, log: log.With(slog.String("component", "usb-allocation-store")), @@ -59,17 +58,9 @@ func NewAllocationStore(nodeName, devicesPath string, resyncPeriod time.Duration allocatableDevices: make(map[string]resourceapi.Device), allocatedDevices: sets.New[string](), resourceClaimAllocations: make(map[types.UID][]string), - discoverer: newDiscoverer(), + usbipInfoGetter: usbip.NewUSBAttacher(), } - monitor := newUSBMonitor(monitorCallback{ - Add: store.genericCallback, - Update: store.genericCallback, - Delete: store.genericCallback, - }) - - store.monitor = monitor - return store } @@ -81,12 +72,11 @@ type AllocationStore struct { cdi cdi.Manager log *slog.Logger - monitor *monitor - updateChannel chan resourceslice.DriverResources mu sync.RWMutex - discoverer discoverer + usbipInfoGetter usbip.AttachInfoGetter + monitor *usb.Monitor discoverPluggedUSBDevices DeviceSet discoverUsbIpPluggedUSBDevices DeviceSet @@ -100,15 +90,17 @@ func (s *AllocationStore) sync() error { s.mu.Lock() defer s.mu.Unlock() - discoverPluggedUSBDevices, discoverUsbIpPluggedUSBDevices, err := s.discoverer.DiscoveryPluggedUSBDevices(s.devicesPath) + discoverPluggedUSBDevices, discoverUsbIpPluggedUSBDevices, err := s.discoveryPluggedUSBDevices() if err != nil { return err } + s.discoverUsbIpPluggedUSBDevices = discoverUsbIpPluggedUSBDevices if discoverPluggedUSBDevices.Equal(s.discoverPluggedUSBDevices) { return nil } + s.discoverPluggedUSBDevices = discoverPluggedUSBDevices allocatableDevices := make([]resourceapi.Device, discoverPluggedUSBDevices.Len()) @@ -128,37 +120,23 @@ func (s *AllocationStore) sync() error { return nil } -func (s *AllocationStore) genericCallback() { +func (s *AllocationStore) callback() { if err := s.sync(); err != nil { s.log.Error("failed to sync usb state", slog.Any("err", err)) } } func (s *AllocationStore) Start(ctx context.Context) error { - if err := s.cdi.CreateCommonSpecFile(); err != nil { - return fmt.Errorf("failed to create CDI common spec file: %w", err) + monitor, err := usb.NewMonitor(ctx, usb.WithResyncPeriod(s.resyncPeriod)) + if err != nil { + return err } + s.monitor = monitor + s.monitor.AddNotifier(usb.FuncNotifier(s.callback)) - doSync := func() { - err := s.sync() - if err != nil { - s.log.Error("failed to sync usb state", slog.Any("err", err)) - } + if err := s.cdi.CreateCommonSpecFile(); err != nil { + return fmt.Errorf("failed to create CDI common spec file: %w", err) } - ticker := time.NewTicker(s.resyncPeriod) - go func() { - doSync() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - doSync() - } - } - }() - - s.monitor.Start(ctx) return nil } diff --git a/images/virtualization-dra/internal/usbip/attacher.go b/images/virtualization-dra/internal/usbip/attacher.go index 1bc9246ba5..a8eed45fa4 100644 --- a/images/virtualization-dra/internal/usbip/attacher.go +++ b/images/virtualization-dra/internal/usbip/attacher.go @@ -22,6 +22,7 @@ import ( "net" "os" "strconv" + "sync" "syscall" "github.com/deckhouse/virtualization-dra/internal/usbip/protocol" @@ -32,10 +33,15 @@ func NewUSBAttacher() USBAttacher { return &usbAttacher{} } -type usbAttacher struct{} +type usbAttacher struct { + mu sync.Mutex +} // https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_attach.c#L174 -func (a usbAttacher) Attach(host, busID string, port int) (int, error) { +func (a *usbAttacher) Attach(host, busID string, port int) (int, error) { + a.mu.Lock() + defer a.mu.Unlock() + conn, err := a.usbipNetTCPConnect(host, fmt.Sprintf("%d", port)) if err != nil { return -1, fmt.Errorf("failed to connect to usbipd: %w", err) @@ -55,7 +61,10 @@ func (a usbAttacher) Attach(host, busID string, port int) (int, error) { } // https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_detach.c#L32 -func (a usbAttacher) Detach(port int) error { +func (a *usbAttacher) Detach(port int) error { + a.mu.Lock() + defer a.mu.Unlock() + driver, err := newVhciDriver() if err != nil { return fmt.Errorf("failed to get vhci driver: %w", err) @@ -100,7 +109,7 @@ func (a usbAttacher) Detach(port int) error { return nil } -func (a usbAttacher) GetAttachInfo() ([]AttachInfo, error) { +func (a *usbAttacher) GetAttachInfo() ([]AttachInfo, error) { driver, err := newVhciDriver() if err != nil { return nil, fmt.Errorf("failed to get vhci driver: %w", err) @@ -126,7 +135,7 @@ func (a usbAttacher) GetAttachInfo() ([]AttachInfo, error) { } // https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_network.c#L261 -func (a usbAttacher) usbipNetTCPConnect(host, port string) (*net.TCPConn, error) { +func (a *usbAttacher) usbipNetTCPConnect(host, port string) (*net.TCPConn, error) { tcpAddr, err := net.ResolveTCPAddr("tcp", net.JoinHostPort(host, port)) if err != nil { return nil, fmt.Errorf("resolve TCP address: %w", err) @@ -151,7 +160,7 @@ func (a usbAttacher) usbipNetTCPConnect(host, port string) (*net.TCPConn, error) } // https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_attach.c#L120 -func (a usbAttacher) queryImportDevice(conn *net.TCPConn, busID string) (int, error) { +func (a *usbAttacher) queryImportDevice(conn *net.TCPConn, busID string) (int, error) { opCommon := protocol.NewOpCommon(protocol.OpReqImport, protocol.OpStatusOk) importReq := protocol.NewImportRequest(busID) @@ -184,7 +193,7 @@ func (a usbAttacher) queryImportDevice(conn *net.TCPConn, busID string) (int, er } // https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_attach.c#L81 -func (a usbAttacher) importDevice(conn *net.TCPConn, usbDevice protocol.USBDevice) (int, error) { +func (a *usbAttacher) importDevice(conn *net.TCPConn, usbDevice protocol.USBDevice) (int, error) { port, err := a.getFreePort(usbDevice.Speed) if err != nil { return -1, fmt.Errorf("failed to get free port: %w", err) @@ -213,7 +222,7 @@ func (a usbAttacher) importDevice(conn *net.TCPConn, usbDevice protocol.USBDevic } // https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/libsrc/vhci_driver.c#L334 -func (a usbAttacher) getFreePort(speed uint32) (int, error) { +func (a *usbAttacher) getFreePort(speed uint32) (int, error) { driver, err := newVhciDriver() if err != nil { return -1, err @@ -244,7 +253,7 @@ func (a usbAttacher) getFreePort(speed uint32) (int, error) { return -1, nil } -func (a usbAttacher) getSockFd(conn *net.TCPConn) (int, error) { +func (a *usbAttacher) getSockFd(conn *net.TCPConn) (int, error) { file, err := conn.File() if err != nil { return -1, err @@ -262,7 +271,7 @@ func (a usbAttacher) getSockFd(conn *net.TCPConn) (int, error) { } // https://github.com/torvalds/linux/blob/b927546677c876e26eba308550207c2ddf812a43/tools/usb/usbip/src/usbip_attach.c#L39 -func (a usbAttacher) recordConnection(host, port, busID string, rhport int) error { +func (a *usbAttacher) recordConnection(host, port, busID string, rhport int) error { err := os.MkdirAll(vhciStatePath, 0700) if err != nil { return fmt.Errorf("failed to create vhci state path: %w", err) diff --git a/images/virtualization-dra/internal/usbip/binder.go b/images/virtualization-dra/internal/usbip/binder.go index aac13bc8d2..fd87533569 100644 --- a/images/virtualization-dra/internal/usbip/binder.go +++ b/images/virtualization-dra/internal/usbip/binder.go @@ -22,6 +22,7 @@ import ( "os" "path/filepath" "strings" + "sync" "github.com/deckhouse/virtualization-dra/pkg/usb" ) @@ -30,11 +31,16 @@ func NewUSBBinder() USBBinder { return &usbBinder{} } -type usbBinder struct{} +type usbBinder struct { + mu sync.Mutex +} // Bind binds the USB device to the USBIP server. // https://github.com/torvalds/linux/blob/40fbbd64bba6c6e7a72885d2f59b6a3be9991eeb/tools/usb/usbip/src/usbip_bind.c#L130 func (b *usbBinder) Bind(busID string) error { + b.mu.Lock() + defer b.mu.Unlock() + devInfo, err := b.getUSBDeviceInfo(busID) if err != nil { return fmt.Errorf("device with bus ID %s does not exist: %w", busID, err) @@ -64,6 +70,9 @@ func (b *usbBinder) Bind(busID string) error { // Unbind unbinds the USB device from the USBIP server. // https://github.com/torvalds/linux/blob/40fbbd64bba6c6e7a72885d2f59b6a3be9991eeb/tools/usb/usbip/src/usbip_unbind.c#L30 func (b *usbBinder) Unbind(busID string) error { + b.mu.Lock() + defer b.mu.Unlock() + devInfo, err := b.getUSBDeviceInfo(busID) if err != nil { return fmt.Errorf("device with bus ID %s does not exist: %w", busID, err) diff --git a/images/virtualization-dra/pkg/udev/conn.go b/images/virtualization-dra/pkg/udev/conn.go new file mode 100644 index 0000000000..320a8f125e --- /dev/null +++ b/images/virtualization-dra/pkg/udev/conn.go @@ -0,0 +1,115 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package udev + +import ( + "fmt" + "os" + "syscall" +) + +// Mode determines the event source +type Mode int + +const ( + // KernelEvent - raw kernel events (faster, less info) + KernelEvent Mode = 1 + // UdevEvent - events processed by udevd (richer, with more attributes like vendor info, serial numbers) + UdevEvent Mode = 2 +) + +// Conn represents a netlink connection for uevents +type Conn struct { + fd int + addr syscall.SockaddrNetlink +} + +// NewConn creates a new udev connection +func NewConn() *Conn { + return &Conn{} +} + +// Connect establishes a connection to the netlink socket +func (c *Conn) Connect(mode Mode) error { + var err error + c.fd, err = syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_KOBJECT_UEVENT) + if err != nil { + return fmt.Errorf("failed to create netlink socket: %w", err) + } + + c.addr = syscall.SockaddrNetlink{ + Family: syscall.AF_NETLINK, + Groups: uint32(mode), + } + + if err := syscall.Bind(c.fd, &c.addr); err != nil { + _ = syscall.Close(c.fd) + return fmt.Errorf("failed to bind netlink socket: %w", err) + } + + return nil +} + +// Close closes the netlink connection +func (c *Conn) Close() error { + if c.fd != 0 { + return syscall.Close(c.fd) + } + return nil +} + +// ReadMsg reads a raw message from the netlink socket (blocking) +func (c *Conn) ReadMsg() ([]byte, error) { + buf := make([]byte, os.Getpagesize()) + + // Peek to get actual message size + n, _, err := syscall.Recvfrom(c.fd, buf, syscall.MSG_PEEK) + if err != nil { + return nil, err + } + + // If message is larger than buffer, resize + for n >= len(buf) { + buf = make([]byte, len(buf)+os.Getpagesize()) + n, _, err = syscall.Recvfrom(c.fd, buf, syscall.MSG_PEEK) + if err != nil { + return nil, err + } + } + + // Now read the actual message + n, _, err = syscall.Recvfrom(c.fd, buf, 0) + if err != nil { + return nil, err + } + + return buf[:n], nil +} + +// ReadUEvent reads and parses a uevent from the socket (blocking) +func (c *Conn) ReadUEvent() (*UEvent, error) { + msg, err := c.ReadMsg() + if err != nil { + return nil, err + } + return ParseUEvent(msg) +} + +// Fd returns the file descriptor of the connection +func (c *Conn) Fd() int { + return c.fd +} diff --git a/images/virtualization-dra/pkg/udev/matcher.go b/images/virtualization-dra/pkg/udev/matcher.go new file mode 100644 index 0000000000..19aa98b94c --- /dev/null +++ b/images/virtualization-dra/pkg/udev/matcher.go @@ -0,0 +1,90 @@ +/* +Copyright 2025 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package udev + +// Matcher is an interface for filtering uevents +type Matcher interface { + // Match returns true if the event should be processed + Match(event *UEvent) bool +} + +// MatcherFunc is a function adapter for Matcher +type MatcherFunc func(event *UEvent) bool + +// Match implements Matcher interface +func (f MatcherFunc) Match(event *UEvent) bool { + return f(event) +} + +// SubsystemMatcher matches events by subsystem +type SubsystemMatcher struct { + Subsystem string +} + +// Match implements Matcher interface +func (m *SubsystemMatcher) Match(event *UEvent) bool { + return event.Subsystem() == m.Subsystem +} + +// SubsystemDevTypeMatcher matches events by subsystem and device type +type SubsystemDevTypeMatcher struct { + Subsystem string + DevType string +} + +// Match implements Matcher interface +func (m *SubsystemDevTypeMatcher) Match(event *UEvent) bool { + return event.Subsystem() == m.Subsystem && event.DevType() == m.DevType +} + +// AndMatcher combines multiple matchers with AND logic +type AndMatcher struct { + Matchers []Matcher +} + +// Match implements Matcher interface +func (m *AndMatcher) Match(event *UEvent) bool { + for _, matcher := range m.Matchers { + if !matcher.Match(event) { + return false + } + } + return true +} + +// OrMatcher combines multiple matchers with OR logic +type OrMatcher struct { + Matchers []Matcher +} + +// Match implements Matcher interface +func (m *OrMatcher) Match(event *UEvent) bool { + for _, matcher := range m.Matchers { + if matcher.Match(event) { + return true + } + } + return false +} + +// AllMatcher matches all events +type AllMatcher struct{} + +// Match implements Matcher interface +func (m *AllMatcher) Match(_ *UEvent) bool { + return true +} diff --git a/images/virtualization-dra/pkg/udev/monitor.go b/images/virtualization-dra/pkg/udev/monitor.go new file mode 100644 index 0000000000..1242742e13 --- /dev/null +++ b/images/virtualization-dra/pkg/udev/monitor.go @@ -0,0 +1,140 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package udev + +import ( + "context" + "errors" + "log/slog" + "syscall" +) + +// Monitor listens for uevents from netlink and sends matching events to a channel. +// It runs until the context is canceled or an unrecoverable error occurs. +type Monitor struct { + conn *Conn + mode Mode + matcher Matcher + log *slog.Logger +} + +// MonitorOption is a functional option for Monitor +type MonitorOption func(*Monitor) + +// WithMode sets the netlink mode +func WithMode(mode Mode) MonitorOption { + return func(m *Monitor) { + m.mode = mode + } +} + +// WithLogger sets the logger +func WithLogger(log *slog.Logger) MonitorOption { + return func(m *Monitor) { + m.log = log + } +} + +// NewMonitor creates a new udev monitor +func NewMonitor(matcher Matcher, opts ...MonitorOption) *Monitor { + m := &Monitor{ + mode: KernelEvent, + matcher: matcher, + log: slog.Default(), + } + + for _, opt := range opts { + opt(m) + } + + return m +} + +// Run starts the monitor and sends matching events to the provided channel. +// It blocks until the context is canceled or an error occurs. +// The channel is NOT closed when the monitor stops - the caller is responsible for that. +func (m *Monitor) Run(ctx context.Context, eventCh chan<- *UEvent) error { + m.conn = NewConn() + if err := m.conn.Connect(m.mode); err != nil { + return err + } + defer func() { + _ = m.conn.Close() + }() + + m.log.Info("udev monitor started", slog.Int("mode", int(m.mode))) + + errCh := make(chan error, 1) + + go func() { + for { + select { + case <-ctx.Done(): + return + default: + } + + uevent, err := m.conn.ReadUEvent() + if err != nil { + if errors.Is(err, syscall.EINTR) { + continue + } + // Check if context was canceled + select { + case <-ctx.Done(): + return + default: + } + errCh <- err + return + } + + if m.matcher == nil || m.matcher.Match(uevent) { + select { + case eventCh <- uevent: + case <-ctx.Done(): + return + } + } + } + }() + + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-errCh: + return err + } +} + +// Start starts the monitor in a goroutine and returns channels for events and errors. +// The caller must consume from both channels to prevent blocking. +func (m *Monitor) Start(ctx context.Context) (<-chan *UEvent, <-chan error) { + eventCh := make(chan *UEvent, 100) + errCh := make(chan error, 1) + + go func() { + defer close(eventCh) + defer close(errCh) + + if err := m.Run(ctx, eventCh); err != nil && !errors.Is(err, context.Canceled) { + errCh <- err + } + }() + + return eventCh, errCh +} diff --git a/images/virtualization-dra/pkg/udev/uevent.go b/images/virtualization-dra/pkg/udev/uevent.go new file mode 100644 index 0000000000..9882de3849 --- /dev/null +++ b/images/virtualization-dra/pkg/udev/uevent.go @@ -0,0 +1,122 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package udev + +import ( + "bytes" + "errors" + "fmt" +) + +// Action represents the action type from kobject events +type Action string + +const ( + ActionAdd Action = "add" + ActionRemove Action = "remove" + ActionChange Action = "change" + ActionMove Action = "move" + ActionOnline Action = "online" + ActionOffline Action = "offline" + ActionBind Action = "bind" + ActionUnbind Action = "unbind" +) + +// String returns string representation of Action +func (a Action) String() string { + return string(a) +} + +// ParseAction parses a string into Action +func ParseAction(s string) (Action, error) { + a := Action(s) + switch a { + case ActionAdd, ActionRemove, ActionChange, ActionMove, ActionOnline, ActionOffline, ActionBind, ActionUnbind: + return a, nil + default: + return "", fmt.Errorf("unknown action: %s", s) + } +} + +// UEvent represents a parsed uevent from netlink +type UEvent struct { + // Action is the event action (add, remove, change, etc.) + Action Action + // KObj is the kernel object path (e.g., /devices/pci0000:00/.../3-2) + KObj string + // Env contains the environment variables from the event + Env map[string]string +} + +// Subsystem returns the subsystem from the event environment +func (e *UEvent) Subsystem() string { + return e.Env["SUBSYSTEM"] +} + +// DevType returns the device type from the event environment +func (e *UEvent) DevType() string { + return e.Env["DEVTYPE"] +} + +// DevPath returns the device path from the event environment +func (e *UEvent) DevPath() string { + return e.Env["DEVPATH"] +} + +// ErrInvalidUEvent is returned when an uevent cannot be parsed +var ErrInvalidUEvent = errors.New("invalid uevent format") + +// ParseUEvent parses a raw uevent message from netlink +func ParseUEvent(raw []byte) (*UEvent, error) { + // Split by null bytes + fields := bytes.Split(raw, []byte{0x00}) + if len(fields) == 0 { + return nil, ErrInvalidUEvent + } + + // First field is like "add@/devices/pci0000:00/..." + headers := bytes.Split(fields[0], []byte("@")) + if len(headers) != 2 { + return nil, ErrInvalidUEvent + } + + action, err := ParseAction(string(headers[0])) + if err != nil { + return nil, err + } + + e := &UEvent{ + Action: action, + KObj: string(headers[1]), + Env: make(map[string]string), + } + + // Parse environment variables + for _, envs := range fields[1 : len(fields)-1] { + if len(envs) == 0 { + continue + } + idx := bytes.IndexByte(envs, '=') + if idx > 0 { + key := string(envs[:idx]) + value := string(envs[idx+1:]) + e.Env[key] = value + } + } + + return e, nil +} diff --git a/images/virtualization-dra/pkg/usb/monitor.go b/images/virtualization-dra/pkg/usb/monitor.go index d5b12204e5..8b0508c029 100644 --- a/images/virtualization-dra/pkg/usb/monitor.go +++ b/images/virtualization-dra/pkg/usb/monitor.go @@ -1,11 +1,11 @@ /* -Copyright 2025 Flant JSC +Copyright 2026 Flant JSC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 + http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, @@ -18,25 +18,66 @@ package usb import ( "context" - "fmt" "log/slog" - "maps" + "os" + "path/filepath" "slices" "strings" "sync" "time" - "github.com/fsnotify/fsnotify" + "github.com/deckhouse/virtualization-dra/pkg/udev" ) +// Monitor is a USB device monitor that uses the udev package for netlink events. +// It provides a clean separation between the generic udev event handling and +// USB-specific device management. type Monitor struct { mu sync.RWMutex devices map[string]*Device - watcher *fsnotify.Watcher notifiers []Notifier + log *slog.Logger + + // Configuration + resyncPeriod time.Duration + debounceDuration time.Duration + + // Debouncing + pendingEvents map[string]*debounceEntry + debounceMu sync.Mutex +} + +type debounceEntry struct { + action udev.Action + timer *time.Timer +} + +// MonitorOption is a functional option for MonitorV5 +type MonitorOption func(*Monitor) + +// WithResyncPeriod sets the resync period +func WithResyncPeriod(d time.Duration) MonitorOption { + return func(m *Monitor) { + m.resyncPeriod = d + } +} + +// WithDebounceDuration sets the debounce duration +func WithDebounceDuration(d time.Duration) MonitorOption { + return func(m *Monitor) { + m.debounceDuration = d + } +} + +// WithLogger sets the logger +func WithLogger(log *slog.Logger) MonitorOption { + return func(m *Monitor) { + m.log = log + } } -func NewMonitor(ctx context.Context, resyncPeriod time.Duration) (*Monitor, error) { +// NewMonitor creates a new USB monitor that uses the udev package +func NewMonitor(ctx context.Context, opts ...MonitorOption) (*Monitor, error) { devices, err := DiscoverPluggedUSBDevices(PathToUSBDevices) if err != nil { return nil, err @@ -45,108 +86,227 @@ func NewMonitor(ctx context.Context, resyncPeriod time.Duration) (*Monitor, erro devices = make(map[string]*Device) } - // TODO: recursive watcher - watcher, err := fsnotify.NewWatcher() - if err != nil { - return nil, err + m := &Monitor{ + devices: devices, + log: slog.With(slog.String("component", "usb-monitor")), + resyncPeriod: 5 * time.Minute, + debounceDuration: 100 * time.Millisecond, + pendingEvents: make(map[string]*debounceEntry), } - if err = watcher.Add(PathToUSBDevices); err != nil { - _ = watcher.Close() - return nil, fmt.Errorf("failed to add USB devices path to fsnotify watcher: %w", err) + for _, opt := range opts { + opt(m) } - monitor := &Monitor{ - devices: devices, - watcher: watcher, - } + go m.run(ctx) - go func() { - monitor.run(ctx, resyncPeriod) - }() - - return monitor, nil + return m, nil } -func (m *Monitor) run(ctx context.Context, resyncPeriod time.Duration) { +func (m *Monitor) run(ctx context.Context) { + // Create udev monitor + udevMonitor := udev.NewMonitor( + newUSBDeviceMatcher(), + udev.WithMode(udev.KernelEvent), + udev.WithLogger(m.log), + ) + + // Start udev monitor and get event channel + eventCh, errCh := udevMonitor.Start(ctx) + + // Create resync ticker + resyncTicker := time.NewTicker(m.resyncPeriod) + defer resyncTicker.Stop() + + m.log.Info("USB monitor started", + slog.Duration("resync_period", m.resyncPeriod), + slog.Duration("debounce_duration", m.debounceDuration), + ) + for { select { case <-ctx.Done(): - _ = m.watcher.Close() + m.log.Info("USB monitor stopped") return - case event := <-m.watcher.Events: - switch event.Op { - case fsnotify.Create, fsnotify.Write: - if err := m.handleUpdate(event); err != nil { - slog.Error("failed to handle update", slog.String("error", err.Error())) - } - case fsnotify.Remove: - m.handleRemove(event) - default: - continue + + case event, ok := <-eventCh: + if !ok { + m.log.Debug("event channel closed") + return } - case err := <-m.watcher.Errors: - slog.Error("error watching USB devices", slog.String("error", err.Error())) + m.handleEvent(event) - case <-time.After(resyncPeriod): - devices, err := DiscoverPluggedUSBDevices(PathToUSBDevices) - if err != nil { - slog.Error("failed to discover USB devices", slog.String("error", err.Error())) + case err, ok := <-errCh: + if !ok { continue } - if devices == nil { - devices = make(map[string]*Device) - } - m.mu.Lock() - if !maps.Equal(m.devices, devices) { - m.devices = devices - m.notify() - } - m.mu.Unlock() + m.log.Error("udev monitor error", slog.String("error", err.Error())) + return + + case <-resyncTicker.C: + m.resync() } } } -func (m *Monitor) handleUpdate(event fsnotify.Event) error { - path := event.Name +func (m *Monitor) handleEvent(event *udev.UEvent) { + m.log.Debug("received uevent", + slog.String("action", event.Action.String()), + slog.String("kobj", event.KObj), + ) + + // Convert KObj to sysfs path + // KObj is like /devices/pci0000:00/.../3-2 + // We need /sys/bus/usb/devices/3-2 + busID := filepath.Base(event.KObj) + sysfsPath := filepath.Join(PathToUSBDevices, busID) + + m.scheduleEvent(sysfsPath, event.Action) +} + +func (m *Monitor) scheduleEvent(path string, action udev.Action) { + m.debounceMu.Lock() + defer m.debounceMu.Unlock() + + // Cancel existing timer if any + if entry, ok := m.pendingEvents[path]; ok { + entry.timer.Stop() + // Prioritize remove action + if action == udev.ActionRemove { + entry.action = udev.ActionRemove + } + delete(m.pendingEvents, path) + } + + finalAction := action + timer := time.AfterFunc(m.debounceDuration, func() { + m.debounceMu.Lock() + delete(m.pendingEvents, path) + m.debounceMu.Unlock() + + m.processEvent(path, finalAction) + }) + + m.pendingEvents[path] = &debounceEntry{ + action: action, + timer: timer, + } +} + +func (m *Monitor) processEvent(path string, action udev.Action) { + m.mu.Lock() + defer m.mu.Unlock() + + switch action { + case udev.ActionAdd, udev.ActionChange, udev.ActionBind, udev.ActionOnline: + m.handleDeviceUpdate(path) + + case udev.ActionRemove, udev.ActionUnbind, udev.ActionOffline: + m.handleDeviceRemove(path) + } +} + +func (m *Monitor) handleDeviceUpdate(path string) { + // Small delay for sysfs to be fully populated + time.Sleep(50 * time.Millisecond) + + // Check if path exists + if _, err := os.Stat(path); os.IsNotExist(err) { + return + } + if !isUsbPath(path) { - return nil + return } - // Get device information device, err := LoadDevice(path) if err != nil { - return err - } - if err = device.Validate(); err != nil { - slog.Error("failed to validate device, skip...", slog.Any("device", device), slog.String("error", err.Error())) - return nil + m.log.Debug("failed to load device", slog.String("path", path), slog.String("error", err.Error())) + return } - m.mu.Lock() - defer m.mu.Unlock() + if err := device.Validate(); err != nil { + m.log.Debug("device validation failed", slog.String("path", path), slog.String("error", err.Error())) + return + } - oldDevice, ok := m.devices[path] - if !ok || !device.Equal(oldDevice) { + oldDevice, exists := m.devices[path] + if !exists { m.devices[path] = &device + m.log.Info("device added", + slog.String("path", path), + slog.String("busid", device.BusID), + slog.String("product", device.Product), + slog.String("manufacturer", device.Manufacturer), + slog.String("serial", device.Serial), + ) + m.notify() + } else if !device.Equal(oldDevice) { + m.devices[path] = &device + m.log.Info("device updated", + slog.String("path", path), + slog.String("busid", device.BusID), + slog.String("product", device.Product), + ) m.notify() } +} - return nil +func (m *Monitor) handleDeviceRemove(path string) { + if device, exists := m.devices[path]; exists { + m.log.Info("device removed", + slog.String("path", path), + slog.String("busid", device.BusID), + ) + delete(m.devices, path) + m.notify() + } } -func (m *Monitor) handleRemove(event fsnotify.Event) { - path := event.Name +func (m *Monitor) resync() { + devices, err := DiscoverPluggedUSBDevices(PathToUSBDevices) + if err != nil { + m.log.Error("failed to discover USB devices during resync", slog.String("error", err.Error())) + return + } + if devices == nil { + devices = make(map[string]*Device) + } m.mu.Lock() defer m.mu.Unlock() - if _, ok := m.devices[path]; ok { - delete(m.devices, path) + changed := false + + // Check for removed devices + for path := range m.devices { + if _, exists := devices[path]; !exists { + m.log.Info("device removed (resync)", slog.String("path", path)) + delete(m.devices, path) + changed = true + } + } + + // Check for added or changed devices + for path, device := range devices { + oldDevice, exists := m.devices[path] + if !exists { + m.log.Info("device added (resync)", slog.String("path", path)) + m.devices[path] = device + changed = true + } else if !device.Equal(oldDevice) { + m.log.Info("device changed (resync)", slog.String("path", path)) + m.devices[path] = device + changed = true + } + } + + if changed { m.notify() } } +// GetDevices returns a copy of all discovered USB devices func (m *Monitor) GetDevices() []Device { m.mu.RLock() devices := make([]Device, 0, len(m.devices)) @@ -162,6 +322,35 @@ func (m *Monitor) GetDevices() []Device { return devices } +// GetDevice returns a device by path +func (m *Monitor) GetDevice(path string) (*Device, bool) { + m.mu.RLock() + defer m.mu.RUnlock() + + device, ok := m.devices[path] + if !ok { + return nil, false + } + + deviceCopy := *device + return &deviceCopy, true +} + +// GetDeviceByBusID returns a device by BusID +func (m *Monitor) GetDeviceByBusID(busID string) (*Device, bool) { + m.mu.RLock() + defer m.mu.RUnlock() + + for _, device := range m.devices { + if device.BusID == busID { + deviceCopy := *device + return &deviceCopy, true + } + } + return nil, false +} + +// AddNotifier adds a notifier to be called on device changes func (m *Monitor) AddNotifier(notifier Notifier) { m.mu.Lock() defer m.mu.Unlock() @@ -169,6 +358,19 @@ func (m *Monitor) AddNotifier(notifier Notifier) { m.notifiers = append(m.notifiers, notifier) } +// RemoveNotifier removes a notifier +func (m *Monitor) RemoveNotifier(notifier Notifier) { + m.mu.Lock() + defer m.mu.Unlock() + + for i, n := range m.notifiers { + if n == notifier { + m.notifiers = slices.Delete(m.notifiers, i, i+1) + return + } + } +} + func (m *Monitor) notify() { for _, notifier := range m.notifiers { notifier.Notify() @@ -178,3 +380,16 @@ func (m *Monitor) notify() { type Notifier interface { Notify() } + +type FuncNotifier func() + +func (f FuncNotifier) Notify() { + f() +} + +func newUSBDeviceMatcher() udev.Matcher { + return &udev.SubsystemDevTypeMatcher{ + Subsystem: "usb", + DevType: "usb_device", + } +} From e2a88d110f691b847066b9c4bda89fbb78a626f2 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Fri, 23 Jan 2026 13:23:37 +0300 Subject: [PATCH 32/37] add inital sync Signed-off-by: Yaroslav Borbat --- images/virtualization-dra/internal/usb/store.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/images/virtualization-dra/internal/usb/store.go b/images/virtualization-dra/internal/usb/store.go index 40d7314180..5d0f184e35 100644 --- a/images/virtualization-dra/internal/usb/store.go +++ b/images/virtualization-dra/internal/usb/store.go @@ -134,6 +134,9 @@ func (s *AllocationStore) Start(ctx context.Context) error { s.monitor = monitor s.monitor.AddNotifier(usb.FuncNotifier(s.callback)) + // Initial sync to publish already discovered devices + s.callback() + if err := s.cdi.CreateCommonSpecFile(); err != nil { return fmt.Errorf("failed to create CDI common spec file: %w", err) } From 25aa169f29bd248f5ee562f4b1caf5adb46244d5 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Fri, 23 Jan 2026 14:30:14 +0300 Subject: [PATCH 33/37] add set host namespace for monitor Signed-off-by: Yaroslav Borbat --- .../cmd/go-usbip/app/info.go | 2 +- .../cmd/go-usbip/app/run.go | 2 +- .../cmd/usb-gateway/app/app.go | 10 ++- .../cmd/virtualization-dra-plugin/app/app.go | 24 ++++-- .../virtualization-dra/internal/usb/store.go | 41 +++------ .../internal/usbip/binder.go | 2 +- images/virtualization-dra/pkg/udev/conn.go | 83 ++++++++++++++++++- images/virtualization-dra/pkg/udev/monitor.go | 19 +++-- .../virtualization-dra/pkg/usb/discovery.go | 8 +- images/virtualization-dra/pkg/usb/monitor.go | 25 ++++-- 10 files changed, 158 insertions(+), 58 deletions(-) diff --git a/images/virtualization-dra/cmd/go-usbip/app/info.go b/images/virtualization-dra/cmd/go-usbip/app/info.go index 434e03823e..7a7d362200 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/info.go +++ b/images/virtualization-dra/cmd/go-usbip/app/info.go @@ -46,7 +46,7 @@ func (o *infoOptions) Usage() string { } func (o *infoOptions) Run(cmd *cobra.Command, _ []string) error { - discoverDevices, err := usb.DefaultDiscoverPluggedUSBDevices() + discoverDevices, err := usb.DiscoverPluggedUSBDevices() if err != nil { return err } diff --git a/images/virtualization-dra/cmd/go-usbip/app/run.go b/images/virtualization-dra/cmd/go-usbip/app/run.go index 0962f174b2..1e5fb77817 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/run.go +++ b/images/virtualization-dra/cmd/go-usbip/app/run.go @@ -59,7 +59,7 @@ func (o *runOptions) AddFlags(fs *pflag.FlagSet) { } func (o *runOptions) Run(cmd *cobra.Command, _ []string) error { - monitor, err := usb.NewMonitor(context.Background(), o.resyncPeriod) + monitor, err := usb.NewMonitor(context.Background(), usb.WithResyncPeriod(o.resyncPeriod)) if err != nil { return err } diff --git a/images/virtualization-dra/cmd/usb-gateway/app/app.go b/images/virtualization-dra/cmd/usb-gateway/app/app.go index a5dcf56f7e..3672d756a8 100644 --- a/images/virtualization-dra/cmd/usb-gateway/app/app.go +++ b/images/virtualization-dra/cmd/usb-gateway/app/app.go @@ -77,6 +77,7 @@ type usbOptions struct { Kubeconfig string NodeName string PodIP string + Isolated bool USBIPPort int USBResyncPeriod time.Duration Logging *logger.Options @@ -87,6 +88,7 @@ func (o *usbOptions) NamedFlags() (fs flag.NamedFlagSets) { mfs.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to kubeconfig file") mfs.StringVar(&o.NodeName, "node-name", os.Getenv("NODE_NAME"), "Node name") mfs.StringVar(&o.PodIP, "pod-ip", os.Getenv("POD_IP"), "Pod IP") + mfs.BoolVar(&o.Isolated, "isolated", true, "Application running in isolated environment and should run USBMonitor in host network namespace") mfs.IntVar(&o.USBIPPort, "usbip-port", 3240, "USBIP port") mfs.DurationVar(&o.USBResyncPeriod, "usb-resync-period", 5*time.Minute, "USB resync period") @@ -113,7 +115,13 @@ func (o *usbOptions) Validate() error { } func (o *usbOptions) Run(cmd *cobra.Command, _ []string) error { - monitor, err := usb.NewMonitor(cmd.Context(), usb.WithResyncPeriod(o.USBResyncPeriod)) + monitorOpts := []usb.MonitorOption{ + usb.WithResyncPeriod(o.USBResyncPeriod), + } + if o.Isolated { + monitorOpts = append(monitorOpts, usb.WithHostNetNS()) + } + monitor, err := usb.NewMonitor(cmd.Context(), monitorOpts...) if err != nil { return err } diff --git a/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go b/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go index a8810f4933..4235eac714 100644 --- a/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go +++ b/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go @@ -33,6 +33,7 @@ import ( "github.com/deckhouse/virtualization-dra/internal/plugin" "github.com/deckhouse/virtualization-dra/internal/usb" "github.com/deckhouse/virtualization-dra/pkg/logger" + libusb "github.com/deckhouse/virtualization-dra/pkg/usb" ) func NewVirtualizationDraPluginCommand() *cobra.Command { @@ -99,6 +100,7 @@ type draOptions struct { KubeletRegisterDirectoryPath string KubeletPluginsDirectoryPath string HealthzPort int + Isolated bool USBResyncPeriod time.Duration Logging *logger.Options @@ -113,6 +115,7 @@ func (o *draOptions) NamedFlags() (fs flag.NamedFlagSets) { mfs.StringVar(&o.KubeletRegisterDirectoryPath, "kubelet-register-directory-path", o.KubeletRegisterDirectoryPath, "Kubelet register directory path") mfs.StringVar(&o.KubeletPluginsDirectoryPath, "kubelet-plugins-directory-path", o.KubeletPluginsDirectoryPath, "Kubelet plugins directory path") mfs.IntVar(&o.HealthzPort, "healthz-port", o.HealthzPort, "Healthz port") + mfs.BoolVar(&o.Isolated, "isolated", true, "Application running in isolated environment and should run USBMonitor in host network namespace") mfs.DurationVar(&o.USBResyncPeriod, "usb-resync-period", o.USBResyncPeriod, "USB resync period") o.Logging.AddFlags(fs.FlagSet("logging")) @@ -157,7 +160,21 @@ func (o *draOptions) Run(cmd *cobra.Command, _ []string) error { return fmt.Errorf("failed to create CDI manager: %w", err) } - usbStore := usb.NewAllocationStore(o.NodeName, o.USBResyncPeriod, usbCDIManager, slog.Default()) + monitorOpts := []libusb.MonitorOption{ + libusb.WithResyncPeriod(o.USBResyncPeriod), + } + if o.Isolated { + monitorOpts = append(monitorOpts, libusb.WithHostNetNS()) + } + monitor, err := libusb.NewMonitor(cmd.Context(), monitorOpts...) + if err != nil { + return fmt.Errorf("failed to create USB monitor: %w", err) + } + + usbStore, err := usb.NewAllocationStore(o.NodeName, usbCDIManager, monitor, slog.Default()) + if err != nil { + return fmt.Errorf("failed to create USB store: %w", err) + } driver := plugin.NewDriver(o.NodeName, client, usbStore, slog.Default()) err = driver.Start(cmd.Context()) @@ -171,11 +188,6 @@ func (o *draOptions) Run(cmd *cobra.Command, _ []string) error { return fmt.Errorf("failed to start health check: %w", err) } - err = usbStore.Start(cmd.Context()) - if err != nil { - return fmt.Errorf("failed to start usb store: %w", err) - } - driver.Wait() driver.Shutdown() healthCheck.Stop() diff --git a/images/virtualization-dra/internal/usb/store.go b/images/virtualization-dra/internal/usb/store.go index 5d0f184e35..a1e8b18b69 100644 --- a/images/virtualization-dra/internal/usb/store.go +++ b/images/virtualization-dra/internal/usb/store.go @@ -43,14 +43,10 @@ import ( const DefaultResyncPeriod = 10 * time.Minute -func NewAllocationStore(nodeName string, resyncPeriod time.Duration, cdiManager cdi.Manager, log *slog.Logger) *AllocationStore { - if resyncPeriod == 0 { - resyncPeriod = DefaultResyncPeriod - } - +func NewAllocationStore(nodeName string, cdiManager cdi.Manager, monitor *usb.Monitor, log *slog.Logger) (*AllocationStore, error) { store := &AllocationStore{ nodeName: nodeName, - resyncPeriod: resyncPeriod, + monitor: monitor, cdi: cdiManager, log: log.With(slog.String("component", "usb-allocation-store")), updateChannel: make(chan resourceslice.DriverResources, 2), @@ -61,13 +57,20 @@ func NewAllocationStore(nodeName string, resyncPeriod time.Duration, cdiManager usbipInfoGetter: usbip.NewUSBAttacher(), } - return store + monitor.AddNotifier(usb.FuncNotifier(store.callback)) + + store.callback() + + if err := store.cdi.CreateCommonSpecFile(); err != nil { + return nil, fmt.Errorf("failed to create CDI common spec file: %w", err) + } + + return store, nil } type AllocationStore struct { - nodeName string - devicesPath string - resyncPeriod time.Duration + nodeName string + devicesPath string cdi cdi.Manager log *slog.Logger @@ -126,24 +129,6 @@ func (s *AllocationStore) callback() { } } -func (s *AllocationStore) Start(ctx context.Context) error { - monitor, err := usb.NewMonitor(ctx, usb.WithResyncPeriod(s.resyncPeriod)) - if err != nil { - return err - } - s.monitor = monitor - s.monitor.AddNotifier(usb.FuncNotifier(s.callback)) - - // Initial sync to publish already discovered devices - s.callback() - - if err := s.cdi.CreateCommonSpecFile(); err != nil { - return fmt.Errorf("failed to create CDI common spec file: %w", err) - } - - return nil -} - func (s *AllocationStore) UpdateChannel() chan resourceslice.DriverResources { return s.updateChannel } diff --git a/images/virtualization-dra/internal/usbip/binder.go b/images/virtualization-dra/internal/usbip/binder.go index fd87533569..cebb77402d 100644 --- a/images/virtualization-dra/internal/usbip/binder.go +++ b/images/virtualization-dra/internal/usbip/binder.go @@ -108,7 +108,7 @@ func (b *usbBinder) IsBound(busID string) (bool, error) { } func (b *usbBinder) GetBindInfo() ([]BindInfo, error) { - usbDevices, err := usb.DefaultDiscoverPluggedUSBDevices() + usbDevices, err := usb.DiscoverPluggedUSBDevices() if err != nil { return nil, fmt.Errorf("failed to discover USB devices: %w", err) } diff --git a/images/virtualization-dra/pkg/udev/conn.go b/images/virtualization-dra/pkg/udev/conn.go index 320a8f125e..76c289de1c 100644 --- a/images/virtualization-dra/pkg/udev/conn.go +++ b/images/virtualization-dra/pkg/udev/conn.go @@ -19,7 +19,10 @@ package udev import ( "fmt" "os" + "runtime" "syscall" + + "golang.org/x/sys/unix" ) // Mode determines the event source @@ -32,19 +35,48 @@ const ( UdevEvent Mode = 2 ) +// HostNetNS is a path to the host network namespace. +// Use this to receive udev events when running in a container without host networking. +const HostNetNS = "/proc/1/ns/net" + // Conn represents a netlink connection for uevents type Conn struct { - fd int - addr syscall.SockaddrNetlink + fd int + addr syscall.SockaddrNetlink + netNS string // optional: path to network namespace (e.g., /proc/1/ns/net) +} + +// ConnOption is a functional option for Conn +type ConnOption func(*Conn) + +// WithNetNS sets the network namespace path for the connection. +// The socket will be created in the specified network namespace. +// This is useful when running in a container without host networking - +// use HostNetNS ("/proc/1/ns/net") to receive udev events from the host. +func WithNetNS(path string) ConnOption { + return func(c *Conn) { + c.netNS = path + } } // NewConn creates a new udev connection -func NewConn() *Conn { - return &Conn{} +func NewConn(opts ...ConnOption) *Conn { + c := &Conn{} + for _, opt := range opts { + opt(c) + } + return c } // Connect establishes a connection to the netlink socket func (c *Conn) Connect(mode Mode) error { + if c.netNS != "" { + return c.connectInNetNS(mode) + } + return c.connect(mode) +} + +func (c *Conn) connect(mode Mode) error { var err error c.fd, err = syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_KOBJECT_UEVENT) if err != nil { @@ -64,6 +96,49 @@ func (c *Conn) Connect(mode Mode) error { return nil } +// connectInNetNS creates the netlink socket in the specified network namespace. +// This allows receiving udev events from the host when running in a container. +// +// IMPORTANT: If the function cannot restore the original network namespace, +// it will panic because the OS thread would be left in an undefined state. +func (c *Conn) connectInNetNS(mode Mode) error { + // Lock the OS thread to ensure namespace operations affect only this goroutine. + // This is critical because Go scheduler can migrate goroutines between OS threads, + // and namespace is a per-thread property. + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + // Save current network namespace + currentNS, err := unix.Open("/proc/self/ns/net", unix.O_RDONLY|unix.O_CLOEXEC, 0) + if err != nil { + return fmt.Errorf("failed to open current netns: %w", err) + } + defer unix.Close(currentNS) + + // Open target network namespace + targetNS, err := unix.Open(c.netNS, unix.O_RDONLY|unix.O_CLOEXEC, 0) + if err != nil { + return fmt.Errorf("failed to open target netns %s: %w", c.netNS, err) + } + defer unix.Close(targetNS) + + // Switch to target network namespace + if err := unix.Setns(targetNS, unix.CLONE_NEWNET); err != nil { + return fmt.Errorf("failed to switch to target netns: %w", err) + } + + defer func() { + if err := unix.Setns(currentNS, unix.CLONE_NEWNET); err != nil { + panic(fmt.Sprintf("FATAL: failed to restore original netns: %v", err)) + } + }() + + // Create socket in target namespace. + // The socket will remain bound to the target namespace even after + // we switch back to the original namespace. + return c.connect(mode) +} + // Close closes the netlink connection func (c *Conn) Close() error { if c.fd != 0 { diff --git a/images/virtualization-dra/pkg/udev/monitor.go b/images/virtualization-dra/pkg/udev/monitor.go index 1242742e13..f8cc9b9e3d 100644 --- a/images/virtualization-dra/pkg/udev/monitor.go +++ b/images/virtualization-dra/pkg/udev/monitor.go @@ -26,10 +26,11 @@ import ( // Monitor listens for uevents from netlink and sends matching events to a channel. // It runs until the context is canceled or an unrecoverable error occurs. type Monitor struct { - conn *Conn - mode Mode - matcher Matcher - log *slog.Logger + conn *Conn + mode Mode + matcher Matcher + log *slog.Logger + connOpts []ConnOption } // MonitorOption is a functional option for Monitor @@ -49,6 +50,14 @@ func WithLogger(log *slog.Logger) MonitorOption { } } +// WithConnOptions sets the connection options for the underlying Conn. +// Use this to configure network namespace for the netlink socket. +func WithConnOptions(opts ...ConnOption) MonitorOption { + return func(m *Monitor) { + m.connOpts = append(m.connOpts, opts...) + } +} + // NewMonitor creates a new udev monitor func NewMonitor(matcher Matcher, opts ...MonitorOption) *Monitor { m := &Monitor{ @@ -68,7 +77,7 @@ func NewMonitor(matcher Matcher, opts ...MonitorOption) *Monitor { // It blocks until the context is canceled or an error occurs. // The channel is NOT closed when the monitor stops - the caller is responsible for that. func (m *Monitor) Run(ctx context.Context, eventCh chan<- *UEvent) error { - m.conn = NewConn() + m.conn = NewConn(m.connOpts...) if err := m.conn.Connect(m.mode); err != nil { return err } diff --git a/images/virtualization-dra/pkg/usb/discovery.go b/images/virtualization-dra/pkg/usb/discovery.go index ebe1e357fd..22c9a3433f 100644 --- a/images/virtualization-dra/pkg/usb/discovery.go +++ b/images/virtualization-dra/pkg/usb/discovery.go @@ -24,14 +24,10 @@ import ( "strings" ) -func DefaultDiscoverPluggedUSBDevices() (map[string]*Device, error) { - return DiscoverPluggedUSBDevices(PathToUSBDevices) -} - -func DiscoverPluggedUSBDevices(pathToUSBDevices string) (map[string]*Device, error) { +func DiscoverPluggedUSBDevices() (map[string]*Device, error) { devices := make(map[string]*Device) - err := filepath.Walk(pathToUSBDevices, func(path string, info os.FileInfo, err error) error { + err := filepath.Walk(PathToUSBDevices, func(path string, info os.FileInfo, err error) error { if err != nil { return err } diff --git a/images/virtualization-dra/pkg/usb/monitor.go b/images/virtualization-dra/pkg/usb/monitor.go index 8b0508c029..f6f0b2f572 100644 --- a/images/virtualization-dra/pkg/usb/monitor.go +++ b/images/virtualization-dra/pkg/usb/monitor.go @@ -41,6 +41,7 @@ type Monitor struct { // Configuration resyncPeriod time.Duration debounceDuration time.Duration + useHostNetNS bool // Debouncing pendingEvents map[string]*debounceEntry @@ -76,9 +77,18 @@ func WithLogger(log *slog.Logger) MonitorOption { } } +// WithHostNetNS configures the monitor to create the netlink socket in the host +// network namespace. This is required when running in a container without host +// networking to receive udev events. +func WithHostNetNS() MonitorOption { + return func(m *Monitor) { + m.useHostNetNS = true + } +} + // NewMonitor creates a new USB monitor that uses the udev package func NewMonitor(ctx context.Context, opts ...MonitorOption) (*Monitor, error) { - devices, err := DiscoverPluggedUSBDevices(PathToUSBDevices) + devices, err := DiscoverPluggedUSBDevices() if err != nil { return nil, err } @@ -105,11 +115,16 @@ func NewMonitor(ctx context.Context, opts ...MonitorOption) (*Monitor, error) { func (m *Monitor) run(ctx context.Context) { // Create udev monitor - udevMonitor := udev.NewMonitor( - newUSBDeviceMatcher(), + udevOpts := []udev.MonitorOption{ udev.WithMode(udev.KernelEvent), udev.WithLogger(m.log), - ) + } + + if m.useHostNetNS { + udevOpts = append(udevOpts, udev.WithConnOptions(udev.WithNetNS(udev.HostNetNS))) + } + + udevMonitor := udev.NewMonitor(newUSBDeviceMatcher(), udevOpts...) // Start udev monitor and get event channel eventCh, errCh := udevMonitor.Start(ctx) @@ -264,7 +279,7 @@ func (m *Monitor) handleDeviceRemove(path string) { } func (m *Monitor) resync() { - devices, err := DiscoverPluggedUSBDevices(PathToUSBDevices) + devices, err := DiscoverPluggedUSBDevices() if err != nil { m.log.Error("failed to discover USB devices during resync", slog.String("error", err.Error())) return From 29b3c918b4860b9e5ca18672aece6267949dbe52 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Fri, 23 Jan 2026 15:52:39 +0300 Subject: [PATCH 34/37] refactor monitor usb Signed-off-by: Yaroslav Borbat --- .../cmd/go-usbip/app/info.go | 6 +- .../cmd/go-usbip/app/run.go | 15 +-- .../cmd/usb-gateway/app/app.go | 30 +++--- .../cmd/virtualization-dra-plugin/app/app.go | 20 ++-- .../virtualization-dra/internal/usb/device.go | 4 +- .../virtualization-dra/internal/usb/store.go | 8 +- .../internal/usbip/attacher.go | 6 +- .../internal/usbip/binder.go | 4 +- .../internal/usbip/usbip.go | 17 --- .../internal/usbip/usbipd.go | 18 ++-- .../internal/usbip/usbipd_config.go | 4 +- .../pkg/{usb => libusb}/discovery.go | 8 +- .../virtualization-dra/pkg/libusb/monitor.go | 100 ++++++++++++++++++ .../pkg/{usb => libusb}/speed.go | 30 +++--- .../monitor.go => libusb/udev-monitor.go} | 88 +++++++-------- .../pkg/{usb => libusb}/usb.go | 36 +++---- 16 files changed, 227 insertions(+), 167 deletions(-) delete mode 100644 images/virtualization-dra/internal/usbip/usbip.go rename images/virtualization-dra/pkg/{usb => libusb}/discovery.go (91%) create mode 100644 images/virtualization-dra/pkg/libusb/monitor.go rename images/virtualization-dra/pkg/{usb => libusb}/speed.go (54%) rename images/virtualization-dra/pkg/{usb/monitor.go => libusb/udev-monitor.go} (79%) rename images/virtualization-dra/pkg/{usb => libusb}/usb.go (92%) diff --git a/images/virtualization-dra/cmd/go-usbip/app/info.go b/images/virtualization-dra/cmd/go-usbip/app/info.go index 7a7d362200..3f336d42cd 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/info.go +++ b/images/virtualization-dra/cmd/go-usbip/app/info.go @@ -46,18 +46,18 @@ func (o *infoOptions) Usage() string { } func (o *infoOptions) Run(cmd *cobra.Command, _ []string) error { - discoverDevices, err := usb.DiscoverPluggedUSBDevices() + discoverDevices, err := libusb.DiscoverPluggedUSBDevices() if err != nil { return err } - devices := make([]*usb.Device, 0, len(discoverDevices)) + devices := make([]*libusb.Device, 0, len(discoverDevices)) for _, device := range discoverDevices { devices = append(devices, device) } - slices.SortFunc(devices, func(a, b *usb.Device) int { + slices.SortFunc(devices, func(a, b *libusb.Device) int { return cmp.Compare(a.Path, b.Path) }) diff --git a/images/virtualization-dra/cmd/go-usbip/app/run.go b/images/virtualization-dra/cmd/go-usbip/app/run.go index 1e5fb77817..b6da54eede 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/run.go +++ b/images/virtualization-dra/cmd/go-usbip/app/run.go @@ -18,17 +18,18 @@ package app import ( "context" - "time" "github.com/spf13/cobra" "github.com/spf13/pflag" "github.com/deckhouse/virtualization-dra/internal/usbip" - "github.com/deckhouse/virtualization-dra/pkg/usb" + "github.com/deckhouse/virtualization-dra/pkg/libusb" ) func NewRunCommand() *cobra.Command { - o := &runOptions{} + o := &runOptions{ + monitor: &libusb.MonitorConfig{}, + } cmd := &cobra.Command{ Use: "run", Short: "Run USBIP server", @@ -43,8 +44,8 @@ func NewRunCommand() *cobra.Command { } type runOptions struct { - port int - resyncPeriod time.Duration + port int + monitor *libusb.MonitorConfig } func (o *runOptions) Usage() string { @@ -55,11 +56,11 @@ func (o *runOptions) Usage() string { func (o *runOptions) AddFlags(fs *pflag.FlagSet) { fs.IntVar(&o.port, "port", 3240, "Port to listen on") - fs.DurationVar(&o.resyncPeriod, "resync-period", time.Second*300, "Resync period") + o.monitor.AddFlags(fs) } func (o *runOptions) Run(cmd *cobra.Command, _ []string) error { - monitor, err := usb.NewMonitor(context.Background(), usb.WithResyncPeriod(o.resyncPeriod)) + monitor, err := o.monitor.Complete(context.Background(), nil) if err != nil { return err } diff --git a/images/virtualization-dra/cmd/usb-gateway/app/app.go b/images/virtualization-dra/cmd/usb-gateway/app/app.go index 3672d756a8..28e947ae25 100644 --- a/images/virtualization-dra/cmd/usb-gateway/app/app.go +++ b/images/virtualization-dra/cmd/usb-gateway/app/app.go @@ -20,7 +20,6 @@ import ( "fmt" "net" "os" - "time" "github.com/spf13/cobra" "golang.org/x/sync/errgroup" @@ -33,8 +32,8 @@ import ( "github.com/deckhouse/virtualization-dra/internal/usb-gateway/informer" "github.com/deckhouse/virtualization-dra/internal/usb-gateway/prepare" "github.com/deckhouse/virtualization-dra/internal/usbip" + "github.com/deckhouse/virtualization-dra/pkg/libusb" "github.com/deckhouse/virtualization-dra/pkg/logger" - "github.com/deckhouse/virtualization-dra/pkg/usb" ) func NewUSBGatewayCommand() *cobra.Command { @@ -70,17 +69,17 @@ func NewUSBGatewayCommand() *cobra.Command { func newUsbOptions() *usbOptions { return &usbOptions{ Logging: &logger.Options{}, + Monitor: &libusb.MonitorConfig{}, } } type usbOptions struct { - Kubeconfig string - NodeName string - PodIP string - Isolated bool - USBIPPort int - USBResyncPeriod time.Duration - Logging *logger.Options + Kubeconfig string + NodeName string + PodIP string + USBIPPort int + Logging *logger.Options + Monitor *libusb.MonitorConfig } func (o *usbOptions) NamedFlags() (fs flag.NamedFlagSets) { @@ -88,12 +87,12 @@ func (o *usbOptions) NamedFlags() (fs flag.NamedFlagSets) { mfs.StringVar(&o.Kubeconfig, "kubeconfig", o.Kubeconfig, "Path to kubeconfig file") mfs.StringVar(&o.NodeName, "node-name", os.Getenv("NODE_NAME"), "Node name") mfs.StringVar(&o.PodIP, "pod-ip", os.Getenv("POD_IP"), "Pod IP") - mfs.BoolVar(&o.Isolated, "isolated", true, "Application running in isolated environment and should run USBMonitor in host network namespace") mfs.IntVar(&o.USBIPPort, "usbip-port", 3240, "USBIP port") - mfs.DurationVar(&o.USBResyncPeriod, "usb-resync-period", 5*time.Minute, "USB resync period") o.Logging.AddFlags(fs.FlagSet("logging")) + o.Monitor.AddFlags(fs.FlagSet("usb-monitor")) + return fs } @@ -115,16 +114,11 @@ func (o *usbOptions) Validate() error { } func (o *usbOptions) Run(cmd *cobra.Command, _ []string) error { - monitorOpts := []usb.MonitorOption{ - usb.WithResyncPeriod(o.USBResyncPeriod), - } - if o.Isolated { - monitorOpts = append(monitorOpts, usb.WithHostNetNS()) - } - monitor, err := usb.NewMonitor(cmd.Context(), monitorOpts...) + monitor, err := o.Monitor.Complete(cmd.Context(), nil) if err != nil { return err } + config := usbip.USBIPDConfig{ Port: o.USBIPPort, Monitor: monitor, diff --git a/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go b/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go index 4235eac714..a706d56b37 100644 --- a/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go +++ b/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go @@ -21,7 +21,6 @@ import ( "log/slog" "os" "strconv" - "time" "github.com/spf13/cobra" "k8s.io/client-go/kubernetes" @@ -32,8 +31,8 @@ import ( "github.com/deckhouse/virtualization-dra/internal/featuregates" "github.com/deckhouse/virtualization-dra/internal/plugin" "github.com/deckhouse/virtualization-dra/internal/usb" + "github.com/deckhouse/virtualization-dra/pkg/libusb" "github.com/deckhouse/virtualization-dra/pkg/logger" - libusb "github.com/deckhouse/virtualization-dra/pkg/usb" ) func NewVirtualizationDraPluginCommand() *cobra.Command { @@ -78,8 +77,8 @@ func newDraOptions() *draOptions { KubeletRegisterDirectoryPath: os.Getenv("KUBELET_REGISTER_DIRECTORY_PATH"), KubeletPluginsDirectoryPath: os.Getenv("KUBELET_PLUGINS_DIRECTORY_PATH"), HealthzPort: 51515, - USBResyncPeriod: usb.DefaultResyncPeriod, Logging: &logger.Options{}, + Monitor: &libusb.MonitorConfig{}, featureGates: featuregates.AddFlags, } @@ -100,10 +99,9 @@ type draOptions struct { KubeletRegisterDirectoryPath string KubeletPluginsDirectoryPath string HealthzPort int - Isolated bool - USBResyncPeriod time.Duration Logging *logger.Options + Monitor *libusb.MonitorConfig featureGates featuregates.AddFlagsFunc } @@ -115,11 +113,11 @@ func (o *draOptions) NamedFlags() (fs flag.NamedFlagSets) { mfs.StringVar(&o.KubeletRegisterDirectoryPath, "kubelet-register-directory-path", o.KubeletRegisterDirectoryPath, "Kubelet register directory path") mfs.StringVar(&o.KubeletPluginsDirectoryPath, "kubelet-plugins-directory-path", o.KubeletPluginsDirectoryPath, "Kubelet plugins directory path") mfs.IntVar(&o.HealthzPort, "healthz-port", o.HealthzPort, "Healthz port") - mfs.BoolVar(&o.Isolated, "isolated", true, "Application running in isolated environment and should run USBMonitor in host network namespace") - mfs.DurationVar(&o.USBResyncPeriod, "usb-resync-period", o.USBResyncPeriod, "USB resync period") o.Logging.AddFlags(fs.FlagSet("logging")) + o.Monitor.AddFlags(fs.FlagSet("usb-monitor")) + o.featureGates(fs.FlagSet("feature-gates")) return fs @@ -160,13 +158,7 @@ func (o *draOptions) Run(cmd *cobra.Command, _ []string) error { return fmt.Errorf("failed to create CDI manager: %w", err) } - monitorOpts := []libusb.MonitorOption{ - libusb.WithResyncPeriod(o.USBResyncPeriod), - } - if o.Isolated { - monitorOpts = append(monitorOpts, libusb.WithHostNetNS()) - } - monitor, err := libusb.NewMonitor(cmd.Context(), monitorOpts...) + monitor, err := o.Monitor.Complete(cmd.Context(), nil) if err != nil { return fmt.Errorf("failed to create USB monitor: %w", err) } diff --git a/images/virtualization-dra/internal/usb/device.go b/images/virtualization-dra/internal/usb/device.go index 230ecb9773..290e6511a8 100644 --- a/images/virtualization-dra/internal/usb/device.go +++ b/images/virtualization-dra/internal/usb/device.go @@ -23,7 +23,7 @@ import ( "k8s.io/apimachinery/pkg/util/sets" - "github.com/deckhouse/virtualization-dra/pkg/usb" + "github.com/deckhouse/virtualization-dra/pkg/libusb" ) type DeviceSet = sets.Set[Device] @@ -82,7 +82,7 @@ func (d *Device) Validate() error { return nil } -func toDevice(device *usb.Device) Device { +func toDevice(device *libusb.USBDevice) Device { return Device{ Path: device.Path, BusID: device.BusID, diff --git a/images/virtualization-dra/internal/usb/store.go b/images/virtualization-dra/internal/usb/store.go index a1e8b18b69..d806b828de 100644 --- a/images/virtualization-dra/internal/usb/store.go +++ b/images/virtualization-dra/internal/usb/store.go @@ -38,12 +38,12 @@ import ( "github.com/deckhouse/virtualization-dra/internal/cdi" "github.com/deckhouse/virtualization-dra/internal/featuregates" "github.com/deckhouse/virtualization-dra/internal/usbip" - "github.com/deckhouse/virtualization-dra/pkg/usb" + "github.com/deckhouse/virtualization-dra/pkg/libusb" ) const DefaultResyncPeriod = 10 * time.Minute -func NewAllocationStore(nodeName string, cdiManager cdi.Manager, monitor *usb.Monitor, log *slog.Logger) (*AllocationStore, error) { +func NewAllocationStore(nodeName string, cdiManager cdi.Manager, monitor libusb.Monitor, log *slog.Logger) (*AllocationStore, error) { store := &AllocationStore{ nodeName: nodeName, monitor: monitor, @@ -57,7 +57,7 @@ func NewAllocationStore(nodeName string, cdiManager cdi.Manager, monitor *usb.Mo usbipInfoGetter: usbip.NewUSBAttacher(), } - monitor.AddNotifier(usb.FuncNotifier(store.callback)) + monitor.AddNotifier(libusb.FuncNotifier(store.callback)) store.callback() @@ -79,7 +79,7 @@ type AllocationStore struct { mu sync.RWMutex usbipInfoGetter usbip.AttachInfoGetter - monitor *usb.Monitor + monitor libusb.Monitor discoverPluggedUSBDevices DeviceSet discoverUsbIpPluggedUSBDevices DeviceSet diff --git a/images/virtualization-dra/internal/usbip/attacher.go b/images/virtualization-dra/internal/usbip/attacher.go index a8eed45fa4..ce4ab73a6c 100644 --- a/images/virtualization-dra/internal/usbip/attacher.go +++ b/images/virtualization-dra/internal/usbip/attacher.go @@ -26,7 +26,7 @@ import ( "syscall" "github.com/deckhouse/virtualization-dra/internal/usbip/protocol" - "github.com/deckhouse/virtualization-dra/pkg/usb" + "github.com/deckhouse/virtualization-dra/pkg/libusb" ) func NewUSBAttacher() USBAttacher { @@ -228,11 +228,11 @@ func (a *usbAttacher) getFreePort(speed uint32) (int, error) { return -1, err } - deviceSpeed := usb.DeviceSpeed(speed) + deviceSpeed := libusb.USBDeviceSpeed(speed) for i := 0; i < driver.nports; i++ { switch deviceSpeed { - case usb.DeviceSpeedSuper: + case libusb.USBDeviceSpeedSuper: if driver.idevs[i].hub != hubSpeedSuper { continue } diff --git a/images/virtualization-dra/internal/usbip/binder.go b/images/virtualization-dra/internal/usbip/binder.go index cebb77402d..c91e5efd66 100644 --- a/images/virtualization-dra/internal/usbip/binder.go +++ b/images/virtualization-dra/internal/usbip/binder.go @@ -24,7 +24,7 @@ import ( "strings" "sync" - "github.com/deckhouse/virtualization-dra/pkg/usb" + "github.com/deckhouse/virtualization-dra/pkg/libusb" ) func NewUSBBinder() USBBinder { @@ -108,7 +108,7 @@ func (b *usbBinder) IsBound(busID string) (bool, error) { } func (b *usbBinder) GetBindInfo() ([]BindInfo, error) { - usbDevices, err := usb.DiscoverPluggedUSBDevices() + usbDevices, err := libusb.DiscoverPluggedUSBDevices() if err != nil { return nil, fmt.Errorf("failed to discover USB devices: %w", err) } diff --git a/images/virtualization-dra/internal/usbip/usbip.go b/images/virtualization-dra/internal/usbip/usbip.go deleted file mode 100644 index 959d25ae5f..0000000000 --- a/images/virtualization-dra/internal/usbip/usbip.go +++ /dev/null @@ -1,17 +0,0 @@ -/* -Copyright 2025 Flant JSC - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package usbip diff --git a/images/virtualization-dra/internal/usbip/usbipd.go b/images/virtualization-dra/internal/usbip/usbipd.go index d538232d1d..863dcdc83f 100644 --- a/images/virtualization-dra/internal/usbip/usbipd.go +++ b/images/virtualization-dra/internal/usbip/usbipd.go @@ -33,7 +33,7 @@ import ( "time" "github.com/deckhouse/virtualization-dra/internal/usbip/protocol" - "github.com/deckhouse/virtualization-dra/pkg/usb" + "github.com/deckhouse/virtualization-dra/pkg/libusb" ) const ( @@ -79,7 +79,7 @@ func WithMaxTCPConnection(maxTCPConnection int) Option { } } -func NewUSBIPD(port int, monitor *usb.Monitor, opts ...Option) *USBIPD { +func NewUSBIPD(port int, monitor libusb.Monitor, opts ...Option) *USBIPD { options := makeOptions(opts...) return &USBIPD{ addr: ":" + strconv.Itoa(port), @@ -104,7 +104,7 @@ type USBIPD struct { connCount atomic.Int64 quit chan struct{} - monitor *usb.Monitor + monitor libusb.Monitor } func (u *USBIPD) Start(ctx context.Context) error { @@ -253,7 +253,7 @@ func (u *USBIPD) handleImportRequest(conn net.Conn) error { devices := u.monitor.GetDevices() - var bindDevice *usb.Device + var bindDevice *libusb.USBDevice for _, device := range devices { if device.BusID == busID { log.Info("found device for export") @@ -287,7 +287,7 @@ func (u *USBIPD) handleImportRequest(conn net.Conn) error { } // https://github.com/torvalds/linux/blob/9448598b22c50c8a5bb77a9103e2d49f134c9578/tools/usb/usbip/libsrc/usbip_host_common.c#L212 -func (u *USBIPD) exportDevice(conn net.Conn, device *usb.Device) protocol.OpStatus { +func (u *USBIPD) exportDevice(conn net.Conn, device *libusb.USBDevice) protocol.OpStatus { log := u.logger.With(slog.String("busID", device.BusID)) log.Info("export request") @@ -352,7 +352,7 @@ func (a sockFdAttr) Complete() string { return fmt.Sprintf("%d\n", a.sockFd) } -func (u *USBIPD) getUSBIPStatus(device *usb.Device) (protocol.DeviceStatus, error) { +func (u *USBIPD) getUSBIPStatus(device *libusb.USBDevice) (protocol.DeviceStatus, error) { statusPath := usbipStatusPath(device.BusID) data, err := os.ReadFile(statusPath) @@ -400,7 +400,7 @@ func (u *USBIPD) getUSBDeviceInfo() []protocol.USBDeviceInfo { return bindDevices } -func toUSBDeviceInfo(device *usb.Device) protocol.USBDeviceInfo { +func toUSBDeviceInfo(device *libusb.USBDevice) protocol.USBDeviceInfo { if device == nil { return protocol.USBDeviceInfo{} } @@ -425,7 +425,7 @@ func toUSBDeviceInfo(device *usb.Device) protocol.USBDeviceInfo { } } -func toInterfaces(interfaces []usb.DeviceInterface) []protocol.USBDeviceInterface { +func toInterfaces(interfaces []libusb.USBDeviceInterface) []protocol.USBDeviceInterface { result := make([]protocol.USBDeviceInterface, len(interfaces)) for i, iface := range interfaces { result[i] = protocol.USBDeviceInterface{ @@ -438,5 +438,5 @@ func toInterfaces(interfaces []usb.DeviceInterface) []protocol.USBDeviceInterfac } func toSpeed(speed uint32) uint32 { - return uint32(usb.ResolveDeviceSpeed(speed)) + return uint32(libusb.ResolveDeviceSpeed(speed)) } diff --git a/images/virtualization-dra/internal/usbip/usbipd_config.go b/images/virtualization-dra/internal/usbip/usbipd_config.go index 49ddadcea6..7008e12044 100644 --- a/images/virtualization-dra/internal/usbip/usbipd_config.go +++ b/images/virtualization-dra/internal/usbip/usbipd_config.go @@ -24,7 +24,7 @@ import ( "os" "time" - "github.com/deckhouse/virtualization-dra/pkg/usb" + "github.com/deckhouse/virtualization-dra/pkg/libusb" ) type ClientAuthType tls.ClientAuthType @@ -66,7 +66,7 @@ type USBIPDConfig struct { Port int GracefulShutdownTimeout time.Duration - Monitor *usb.Monitor + Monitor libusb.Monitor } func (c *USBIPDConfig) AddFlags(fs *flag.FlagSet) { diff --git a/images/virtualization-dra/pkg/usb/discovery.go b/images/virtualization-dra/pkg/libusb/discovery.go similarity index 91% rename from images/virtualization-dra/pkg/usb/discovery.go rename to images/virtualization-dra/pkg/libusb/discovery.go index 22c9a3433f..176b7dca59 100644 --- a/images/virtualization-dra/pkg/usb/discovery.go +++ b/images/virtualization-dra/pkg/libusb/discovery.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package usb +package libusb import ( "fmt" @@ -24,8 +24,8 @@ import ( "strings" ) -func DiscoverPluggedUSBDevices() (map[string]*Device, error) { - devices := make(map[string]*Device) +func DiscoverPluggedUSBDevices() (map[string]*USBDevice, error) { + devices := make(map[string]*USBDevice) err := filepath.Walk(PathToUSBDevices, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -37,7 +37,7 @@ func DiscoverPluggedUSBDevices() (map[string]*Device, error) { } // Get device information - device, err := LoadDevice(path) + device, err := LoadUSBDevice(path) if err != nil { return err } diff --git a/images/virtualization-dra/pkg/libusb/monitor.go b/images/virtualization-dra/pkg/libusb/monitor.go new file mode 100644 index 0000000000..a168f90973 --- /dev/null +++ b/images/virtualization-dra/pkg/libusb/monitor.go @@ -0,0 +1,100 @@ +package libusb + +import ( + "context" + "fmt" + "log/slog" + "time" + + "github.com/spf13/pflag" + "k8s.io/utils/ptr" +) + +type Monitor interface { + GetDevices() []USBDevice + GetDevice(path string) (*USBDevice, bool) + GetDeviceByBusID(busID string) (*USBDevice, bool) + AddNotifier(notifier Notifier) + RemoveNotifier(notifier Notifier) +} + +type Notifier interface { + Notify() +} +type FuncNotifier func() + +func (f FuncNotifier) Notify() { + f() +} + +type MonitorType string + +const ( + UdevMonitorType MonitorType = "udev" +) + +func (m *MonitorType) String() string { + return string(*m) +} + +func (m *MonitorType) Set(s string) error { + switch s { + case ptr.To(UdevMonitorType).String(): + *m = UdevMonitorType + default: + return fmt.Errorf("invalid monitor type: %s", s) + } + return nil +} + +func (m *MonitorType) Type() string { + return "monitor-type" +} + +type MonitorConfig struct { + MonitorType MonitorType + + // ALL + ResyncPeriod time.Duration + Logger *slog.Logger + + // UDEV + DebounceDuration time.Duration + HostNetNs bool +} + +func (c *MonitorConfig) AddFlags(fs *pflag.FlagSet) { + fs.Var(&c.MonitorType, "usb-monitor-type", "USB monitor type") + fs.DurationVar(&c.ResyncPeriod, "usb-monitor-resync-period", c.ResyncPeriod, "USB monitor resync period") + fs.DurationVar(&c.DebounceDuration, "udev-usb-monitor-debounce-duration", c.DebounceDuration, "UDEV USB monitor debounce duration") + fs.BoolVar(&c.HostNetNs, "udev-usb-monitor-host-netns", c.HostNetNs, "UDEV USB monitor host netns") +} + +func (c *MonitorConfig) Complete(ctx context.Context, logger *slog.Logger) (Monitor, error) { + c.Logger = logger + + switch c.MonitorType { + case UdevMonitorType: + return NewUdevMonitor(ctx, c.makeUdevOpts()...) + default: + return nil, fmt.Errorf("unsupported monitor type: %s", c.MonitorType) + } +} + +func (c *MonitorConfig) makeUdevOpts() []UdevMonitorOption { + var opts []UdevMonitorOption + if c.ResyncPeriod > 0 { + opts = append(opts, UdevWithResyncPeriod(c.ResyncPeriod)) + } + if c.Logger != nil { + opts = append(opts, UdevWithLogger(c.Logger)) + } + if c.DebounceDuration > 0 { + opts = append(opts, UdevWithDebounceDuration(c.DebounceDuration)) + } + if c.HostNetNs { + opts = append(opts, UdevWithHostNetNS()) + } + + return opts +} diff --git a/images/virtualization-dra/pkg/usb/speed.go b/images/virtualization-dra/pkg/libusb/speed.go similarity index 54% rename from images/virtualization-dra/pkg/usb/speed.go rename to images/virtualization-dra/pkg/libusb/speed.go index 874d45b72e..4bad121ce2 100644 --- a/images/virtualization-dra/pkg/usb/speed.go +++ b/images/virtualization-dra/pkg/libusb/speed.go @@ -14,33 +14,33 @@ See the License for the specific language governing permissions and limitations under the License. */ -package usb +package libusb -type DeviceSpeed uint32 +type USBDeviceSpeed uint32 const ( - DeviceSpeedUnknown DeviceSpeed = iota // enumerating - DeviceSpeedLow // usb 1.1 - DeviceSpeedFull // usb 1.1 - DeviceSpeedHigh // usb 2.0 - DeviceSpeedSuper // usb 3.0 - DeviceSpeedSuperPlus // usb 3.1 + USBDeviceSpeedUnknown USBDeviceSpeed = iota // enumerating + USBDeviceSpeedLow // usb 1.1 + USBDeviceSpeedFull // usb 1.1 + USBDeviceSpeedHigh // usb 2.0 + USBDeviceSpeedSuper // usb 3.0 + USBDeviceSpeedSuperPlus // usb 3.1 ) // https://mjmwired.net/kernel/Documentation/ABI/testing/sysfs-bus-usb#502 -func ResolveDeviceSpeed(speed uint32) DeviceSpeed { +func ResolveDeviceSpeed(speed uint32) USBDeviceSpeed { switch speed { case 1: - return DeviceSpeedLow + return USBDeviceSpeedLow case 12, 15: - return DeviceSpeedFull + return USBDeviceSpeedFull case 480: - return DeviceSpeedHigh + return USBDeviceSpeedHigh case 5000: - return DeviceSpeedSuper + return USBDeviceSpeedSuper case 10000, 20000: - return DeviceSpeedSuperPlus + return USBDeviceSpeedSuperPlus default: - return DeviceSpeedUnknown + return USBDeviceSpeedUnknown } } diff --git a/images/virtualization-dra/pkg/usb/monitor.go b/images/virtualization-dra/pkg/libusb/udev-monitor.go similarity index 79% rename from images/virtualization-dra/pkg/usb/monitor.go rename to images/virtualization-dra/pkg/libusb/udev-monitor.go index f6f0b2f572..a589d4826e 100644 --- a/images/virtualization-dra/pkg/usb/monitor.go +++ b/images/virtualization-dra/pkg/libusb/udev-monitor.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package usb +package libusb import ( "context" @@ -29,12 +29,12 @@ import ( "github.com/deckhouse/virtualization-dra/pkg/udev" ) -// Monitor is a USB device monitor that uses the udev package for netlink events. +// UdevMonitor is a USB device monitor that uses the udev package for netlink events. // It provides a clean separation between the generic udev event handling and // USB-specific device management. -type Monitor struct { +type UdevMonitor struct { mu sync.RWMutex - devices map[string]*Device + devices map[string]*USBDevice notifiers []Notifier log *slog.Logger @@ -53,50 +53,50 @@ type debounceEntry struct { timer *time.Timer } -// MonitorOption is a functional option for MonitorV5 -type MonitorOption func(*Monitor) +// UdevMonitorOption is a functional option for MonitorV5 +type UdevMonitorOption func(*UdevMonitor) -// WithResyncPeriod sets the resync period -func WithResyncPeriod(d time.Duration) MonitorOption { - return func(m *Monitor) { +// UdevWithResyncPeriod sets the resync period +func UdevWithResyncPeriod(d time.Duration) UdevMonitorOption { + return func(m *UdevMonitor) { m.resyncPeriod = d } } -// WithDebounceDuration sets the debounce duration -func WithDebounceDuration(d time.Duration) MonitorOption { - return func(m *Monitor) { +// UdevWithDebounceDuration sets the debounce duration +func UdevWithDebounceDuration(d time.Duration) UdevMonitorOption { + return func(m *UdevMonitor) { m.debounceDuration = d } } -// WithLogger sets the logger -func WithLogger(log *slog.Logger) MonitorOption { - return func(m *Monitor) { +// UdevWithLogger sets the logger +func UdevWithLogger(log *slog.Logger) UdevMonitorOption { + return func(m *UdevMonitor) { m.log = log } } -// WithHostNetNS configures the monitor to create the netlink socket in the host +// UdevWithHostNetNS configures the monitor to create the netlink socket in the host // network namespace. This is required when running in a container without host // networking to receive udev events. -func WithHostNetNS() MonitorOption { - return func(m *Monitor) { +func UdevWithHostNetNS() UdevMonitorOption { + return func(m *UdevMonitor) { m.useHostNetNS = true } } -// NewMonitor creates a new USB monitor that uses the udev package -func NewMonitor(ctx context.Context, opts ...MonitorOption) (*Monitor, error) { +// NewUdevMonitor creates a new USB monitor that uses the udev package +func NewUdevMonitor(ctx context.Context, opts ...UdevMonitorOption) (Monitor, error) { devices, err := DiscoverPluggedUSBDevices() if err != nil { return nil, err } if devices == nil { - devices = make(map[string]*Device) + devices = make(map[string]*USBDevice) } - m := &Monitor{ + m := &UdevMonitor{ devices: devices, log: slog.With(slog.String("component", "usb-monitor")), resyncPeriod: 5 * time.Minute, @@ -113,7 +113,7 @@ func NewMonitor(ctx context.Context, opts ...MonitorOption) (*Monitor, error) { return m, nil } -func (m *Monitor) run(ctx context.Context) { +func (m *UdevMonitor) run(ctx context.Context) { // Create udev monitor udevOpts := []udev.MonitorOption{ udev.WithMode(udev.KernelEvent), @@ -164,7 +164,7 @@ func (m *Monitor) run(ctx context.Context) { } } -func (m *Monitor) handleEvent(event *udev.UEvent) { +func (m *UdevMonitor) handleEvent(event *udev.UEvent) { m.log.Debug("received uevent", slog.String("action", event.Action.String()), slog.String("kobj", event.KObj), @@ -179,7 +179,7 @@ func (m *Monitor) handleEvent(event *udev.UEvent) { m.scheduleEvent(sysfsPath, event.Action) } -func (m *Monitor) scheduleEvent(path string, action udev.Action) { +func (m *UdevMonitor) scheduleEvent(path string, action udev.Action) { m.debounceMu.Lock() defer m.debounceMu.Unlock() @@ -208,7 +208,7 @@ func (m *Monitor) scheduleEvent(path string, action udev.Action) { } } -func (m *Monitor) processEvent(path string, action udev.Action) { +func (m *UdevMonitor) processEvent(path string, action udev.Action) { m.mu.Lock() defer m.mu.Unlock() @@ -221,7 +221,7 @@ func (m *Monitor) processEvent(path string, action udev.Action) { } } -func (m *Monitor) handleDeviceUpdate(path string) { +func (m *UdevMonitor) handleDeviceUpdate(path string) { // Small delay for sysfs to be fully populated time.Sleep(50 * time.Millisecond) @@ -234,7 +234,7 @@ func (m *Monitor) handleDeviceUpdate(path string) { return } - device, err := LoadDevice(path) + device, err := LoadUSBDevice(path) if err != nil { m.log.Debug("failed to load device", slog.String("path", path), slog.String("error", err.Error())) return @@ -267,7 +267,7 @@ func (m *Monitor) handleDeviceUpdate(path string) { } } -func (m *Monitor) handleDeviceRemove(path string) { +func (m *UdevMonitor) handleDeviceRemove(path string) { if device, exists := m.devices[path]; exists { m.log.Info("device removed", slog.String("path", path), @@ -278,14 +278,14 @@ func (m *Monitor) handleDeviceRemove(path string) { } } -func (m *Monitor) resync() { +func (m *UdevMonitor) resync() { devices, err := DiscoverPluggedUSBDevices() if err != nil { m.log.Error("failed to discover USB devices during resync", slog.String("error", err.Error())) return } if devices == nil { - devices = make(map[string]*Device) + devices = make(map[string]*USBDevice) } m.mu.Lock() @@ -322,15 +322,15 @@ func (m *Monitor) resync() { } // GetDevices returns a copy of all discovered USB devices -func (m *Monitor) GetDevices() []Device { +func (m *UdevMonitor) GetDevices() []USBDevice { m.mu.RLock() - devices := make([]Device, 0, len(m.devices)) + devices := make([]USBDevice, 0, len(m.devices)) for _, device := range m.devices { devices = append(devices, *device) } m.mu.RUnlock() - slices.SortFunc(devices, func(a, b Device) int { + slices.SortFunc(devices, func(a, b USBDevice) int { return strings.Compare(a.DevicePath, b.DevicePath) }) @@ -338,7 +338,7 @@ func (m *Monitor) GetDevices() []Device { } // GetDevice returns a device by path -func (m *Monitor) GetDevice(path string) (*Device, bool) { +func (m *UdevMonitor) GetDevice(path string) (*USBDevice, bool) { m.mu.RLock() defer m.mu.RUnlock() @@ -352,7 +352,7 @@ func (m *Monitor) GetDevice(path string) (*Device, bool) { } // GetDeviceByBusID returns a device by BusID -func (m *Monitor) GetDeviceByBusID(busID string) (*Device, bool) { +func (m *UdevMonitor) GetDeviceByBusID(busID string) (*USBDevice, bool) { m.mu.RLock() defer m.mu.RUnlock() @@ -366,7 +366,7 @@ func (m *Monitor) GetDeviceByBusID(busID string) (*Device, bool) { } // AddNotifier adds a notifier to be called on device changes -func (m *Monitor) AddNotifier(notifier Notifier) { +func (m *UdevMonitor) AddNotifier(notifier Notifier) { m.mu.Lock() defer m.mu.Unlock() @@ -374,7 +374,7 @@ func (m *Monitor) AddNotifier(notifier Notifier) { } // RemoveNotifier removes a notifier -func (m *Monitor) RemoveNotifier(notifier Notifier) { +func (m *UdevMonitor) RemoveNotifier(notifier Notifier) { m.mu.Lock() defer m.mu.Unlock() @@ -386,22 +386,12 @@ func (m *Monitor) RemoveNotifier(notifier Notifier) { } } -func (m *Monitor) notify() { +func (m *UdevMonitor) notify() { for _, notifier := range m.notifiers { notifier.Notify() } } -type Notifier interface { - Notify() -} - -type FuncNotifier func() - -func (f FuncNotifier) Notify() { - f() -} - func newUSBDeviceMatcher() udev.Matcher { return &udev.SubsystemDevTypeMatcher{ Subsystem: "usb", diff --git a/images/virtualization-dra/pkg/usb/usb.go b/images/virtualization-dra/pkg/libusb/usb.go similarity index 92% rename from images/virtualization-dra/pkg/usb/usb.go rename to images/virtualization-dra/pkg/libusb/usb.go index fc6c474ab6..706fa9fb84 100644 --- a/images/virtualization-dra/pkg/usb/usb.go +++ b/images/virtualization-dra/pkg/libusb/usb.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package usb +package libusb import ( "bufio" @@ -29,7 +29,7 @@ import ( const PathToUSBDevices = "/sys/bus/usb/devices" -type Device struct { +type USBDevice struct { Path string BusID string Manufacturer string @@ -52,16 +52,16 @@ type Device struct { BConfigurationValue uint8 BNumConfigurations uint8 BNumInterfaces uint8 - Interfaces []DeviceInterface + Interfaces []USBDeviceInterface } -type DeviceInterface struct { +type USBDeviceInterface struct { BInterfaceClass uint8 BInterfaceSubClass uint8 BInterfaceProtocol uint8 } -func (d *Device) Equal(other *Device) bool { +func (d *USBDevice) Equal(other *USBDevice) bool { return d.Path == other.Path && d.BusID == other.BusID && d.Manufacturer == other.Manufacturer && @@ -86,7 +86,7 @@ func (d *Device) Equal(other *Device) bool { slices.Equal(d.Interfaces, other.Interfaces) } -func (d *Device) Validate() error { +func (d *USBDevice) Validate() error { if d.VendorID == 0 { return fmt.Errorf("VendorID is required") } @@ -111,7 +111,7 @@ func (d *Device) Validate() error { return nil } -func LoadDevice(path string) (device Device, err error) { +func LoadUSBDevice(path string) (device USBDevice, err error) { if !strings.HasPrefix(path, PathToUSBDevices) { return device, fmt.Errorf("path %s is not a usb device", path) } @@ -150,7 +150,7 @@ func LoadDevice(path string) (device Device, err error) { return } -func parseSysUeventFile(path string, device *Device) error { +func parseSysUeventFile(path string, device *USBDevice) error { // Example uevent file: // MAJOR=189 // MINOR=257 @@ -258,7 +258,7 @@ func parseSysUeventFile(path string, device *Device) error { return nil } -func parseSerial(path string, device *Device) error { +func parseSerial(path string, device *USBDevice) error { serial, err := parseStringValue(path, "serial") if err != nil { return err @@ -267,7 +267,7 @@ func parseSerial(path string, device *Device) error { return nil } -func parseManufacturer(path string, device *Device) error { +func parseManufacturer(path string, device *USBDevice) error { manufacturer, err := parseStringValue(path, "manufacturer") if err != nil { return err @@ -276,7 +276,7 @@ func parseManufacturer(path string, device *Device) error { return nil } -func parseProduct(path string, device *Device) error { +func parseProduct(path string, device *USBDevice) error { product, err := parseStringValue(path, "product") if err != nil { return err @@ -285,7 +285,7 @@ func parseProduct(path string, device *Device) error { return nil } -func parseBConfigurationValue(path string, device *Device) error { +func parseBConfigurationValue(path string, device *USBDevice) error { val, err := parseUintValue(path, "bConfigurationValue", 8, true) if err != nil { return err @@ -294,7 +294,7 @@ func parseBConfigurationValue(path string, device *Device) error { return nil } -func parseBNumConfigurations(path string, device *Device) error { +func parseBNumConfigurations(path string, device *USBDevice) error { val, err := parseUintValue(path, "bNumConfigurations", 8, false) if err != nil { return err @@ -303,7 +303,7 @@ func parseBNumConfigurations(path string, device *Device) error { return nil } -func parseBNumInterfaces(path string, device *Device) error { +func parseBNumInterfaces(path string, device *USBDevice) error { val, err := parseUintValue(path, "bNumInterfaces", 8, true) if err != nil { return err @@ -312,7 +312,7 @@ func parseBNumInterfaces(path string, device *Device) error { return nil } -func parseSpeed(path string, device *Device) error { +func parseSpeed(path string, device *USBDevice) error { val, err := parseUintValue(path, "speed", 32, false) if err != nil { return err @@ -321,7 +321,7 @@ func parseSpeed(path string, device *Device) error { return nil } -func parseSysUeventInterfaces(path string, device *Device) error { +func parseSysUeventInterfaces(path string, device *USBDevice) error { // 3-2.1.1:1.0 // | | | // │ | |- bInterfaceNumber @@ -336,7 +336,7 @@ func parseSysUeventInterfaces(path string, device *Device) error { return nil } - var deviceInterfaces []DeviceInterface + var deviceInterfaces []USBDeviceInterface parent := filepath.Dir(path) entries, err := os.ReadDir(parent) @@ -376,7 +376,7 @@ func parseSysUeventInterfaces(path string, device *Device) error { } switch values[0] { case "INTERFACE": - deviceInterface := DeviceInterface{} + deviceInterface := USBDeviceInterface{} interfaces := strings.Split(values[1], "/") if len(interfaces) != 3 { From c5005b72f2ac31e62d06580663e362e9ca005529 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Fri, 23 Jan 2026 16:24:07 +0300 Subject: [PATCH 35/37] add patch package Signed-off-by: Yaroslav Borbat --- .../pkg/common/patch/patch.go | 16 +-- .../controller/resourceclaim/controller.go | 85 +++++++++++--- .../internal/usb-gateway/labeler/labeler.go | 28 +++-- images/virtualization-dra/pkg/patch/patch.go | 110 ++++++++++++++++++ 4 files changed, 199 insertions(+), 40 deletions(-) create mode 100644 images/virtualization-dra/pkg/patch/patch.go diff --git a/images/virtualization-artifact/pkg/common/patch/patch.go b/images/virtualization-artifact/pkg/common/patch/patch.go index b6e4640755..573cfcfc2d 100644 --- a/images/virtualization-artifact/pkg/common/patch/patch.go +++ b/images/virtualization-artifact/pkg/common/patch/patch.go @@ -19,6 +19,7 @@ package patch import ( "encoding/json" "fmt" + "slices" "strings" ) @@ -74,18 +75,9 @@ func (jp *JSONPatch) Append(patches ...JSONPatchOperation) { } func (jp *JSONPatch) Delete(op, path string) { - var idx int - var found bool - for i, o := range jp.operations { - if o.Op == op && o.Path == path { - idx = i - found = true - break - } - } - if found { - jp.operations = append(jp.operations[:idx], jp.operations[idx+1:]...) - } + slices.DeleteFunc(jp.operations, func(o JSONPatchOperation) bool { + return o.Op == op && o.Path == path + }) } func (jp *JSONPatch) Len() int { diff --git a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go index e43f5ece78..8d60cfdfa8 100644 --- a/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go +++ b/images/virtualization-dra/internal/usb-gateway/controller/resourceclaim/controller.go @@ -40,6 +40,7 @@ import ( "github.com/deckhouse/virtualization-dra/internal/common" "github.com/deckhouse/virtualization-dra/internal/usb-gateway/informer" "github.com/deckhouse/virtualization-dra/internal/usbip" + "github.com/deckhouse/virtualization-dra/pkg/patch" ) const controllerName = "resourceclaim-controller" @@ -367,32 +368,52 @@ func (c *Controller) allUnBound(rc *resourcev1beta1.ResourceClaim) (bool, error) return true, nil } -func (c *Controller) addFinalizerForResourceClaim(rc *resourcev1beta1.ResourceClaim) (err error) { - if addFinalizer(rc) { - _, err = c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).Update(context.Background(), rc, metav1.UpdateOptions{}) +func (c *Controller) addFinalizerForResourceClaim(rc *resourcev1beta1.ResourceClaim) error { + patchBytes, err := makeAddFinalizerPatch(rc) + if err != nil { + return err } - return + if patchBytes != nil { + _, err = c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).Patch(context.Background(), rc.Name, types.JSONPatchType, patchBytes, metav1.PatchOptions{}) + return err + } + return nil } -func (c *Controller) removeFinalizerForResourceClaim(rc *resourcev1beta1.ResourceClaim) (err error) { - if removeFinalizer(rc) { - _, err = c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).Update(context.Background(), rc, metav1.UpdateOptions{}) +func (c *Controller) removeFinalizerForResourceClaim(rc *resourcev1beta1.ResourceClaim) error { + patchBytes, err := makeRemoveFinalizerPatch(rc) + if err != nil { + return err } - return + if patchBytes != nil { + _, err = c.client.ResourceV1beta1().ResourceClaims(rc.Namespace).Patch(context.Background(), rc.Name, types.JSONPatchType, patchBytes, metav1.PatchOptions{}) + return err + } + return nil } -func (c *Controller) addFinalizerForPod(pod *corev1.Pod) (err error) { - if addFinalizer(pod) { - _, err = c.client.CoreV1().Pods(pod.Namespace).Update(context.Background(), pod, metav1.UpdateOptions{}) +func (c *Controller) addFinalizerForPod(pod *corev1.Pod) error { + patchBytes, err := makeAddFinalizerPatch(pod) + if err != nil { + return err } - return + if patchBytes != nil { + _, err = c.client.CoreV1().Pods(pod.Namespace).Patch(context.Background(), pod.Name, types.JSONPatchType, patchBytes, metav1.PatchOptions{}) + return err + } + return nil } -func (c *Controller) removeFinalizerForPod(pod *corev1.Pod) (err error) { - if removeFinalizer(pod) { - _, err = c.client.CoreV1().Pods(pod.Namespace).Update(context.Background(), pod, metav1.UpdateOptions{}) +func (c *Controller) removeFinalizerForPod(pod *corev1.Pod) error { + patchBytes, err := makeRemoveFinalizerPatch(pod) + if err != nil { + return err } - return + if patchBytes != nil { + _, err = c.client.CoreV1().Pods(pod.Namespace).Patch(context.Background(), pod.Name, types.JSONPatchType, patchBytes, metav1.PatchOptions{}) + return err + } + return nil } func (c *Controller) handleServer(rc *resourcev1beta1.ResourceClaim, myAllocationDevices []resourcev1beta1.Device) error { @@ -829,3 +850,35 @@ func removeFinalizer(obj metav1.Object) bool { obj.SetFinalizers(newFinalizers) return true } + +func makeAddFinalizerPatch(obj metav1.Object) ([]byte, error) { + var newFinalizers []string + for _, fin := range obj.GetFinalizers() { + if fin == finalizer { + return nil, nil + } + newFinalizers = append(newFinalizers, fin) + } + newFinalizers = append(newFinalizers, finalizer) + + return patch.NewJSONPatch( + patch.WithTest("/metadata/finalizers", obj.GetFinalizers()), + patch.WithReplace("/metadata/finalizers", newFinalizers), + ).Bytes() +} + +func makeRemoveFinalizerPatch(obj metav1.Object) ([]byte, error) { + var newFinalizers []string + for _, fin := range obj.GetFinalizers() { + if fin == finalizer { + return nil, nil + } + newFinalizers = append(newFinalizers, fin) + } + newFinalizers = append(newFinalizers, finalizer) + + return patch.NewJSONPatch( + patch.WithTest("/metadata/finalizers", obj.GetFinalizers()), + patch.WithReplace("/metadata/finalizers", newFinalizers), + ).Bytes() +} diff --git a/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go b/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go index 6f8af80b21..f5050963ba 100644 --- a/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go +++ b/images/virtualization-dra/internal/usb-gateway/labeler/labeler.go @@ -18,7 +18,6 @@ package labeler import ( "context" - "encoding/json" "fmt" "maps" @@ -26,10 +25,12 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/dynamic" + + "github.com/deckhouse/virtualization-dra/pkg/patch" ) type Labeler interface { - Label(ctx context.Context, name, namespace string, labels map[string]string) error + Label(ctx context.Context, name, namespace string, addLabels map[string]string) error } type genericLabeler struct { @@ -44,23 +45,26 @@ func NewGenericLabeler(client dynamic.Interface, gvr schema.GroupVersionResource } } -func (l *genericLabeler) Label(ctx context.Context, name, namespace string, newLabels map[string]string) error { +func (l *genericLabeler) Label(ctx context.Context, name, namespace string, addLabels map[string]string) error { obj, err := l.client.Resource(l.gvr).Namespace(namespace).Get(ctx, name, metav1.GetOptions{}) if err != nil { return err } - labels := obj.GetLabels() - maps.Copy(labels, newLabels) + oldLabels := obj.GetLabels() + newLabels := make(map[string]string) + maps.Copy(newLabels, oldLabels) + maps.Copy(newLabels, addLabels) - value, err := json.Marshal(labels) + patchBytes, err := patch.NewJSONPatch( + patch.WithTest("/metadata/labels", oldLabels), + patch.WithReplace("/metadata/labels", newLabels), + ).Bytes() if err != nil { - return err + return fmt.Errorf("failed to create patch: %w", err) } - patch := []byte(fmt.Sprintf(`[{"op": "replace", "path": "/metadata/labels", "value": %s}]`, string(value))) - - _, err = l.client.Resource(l.gvr).Namespace(namespace).Patch(ctx, name, types.JSONPatchType, patch, metav1.PatchOptions{}) + _, err = l.client.Resource(l.gvr).Namespace(namespace).Patch(ctx, name, types.JSONPatchType, patchBytes, metav1.PatchOptions{}) return err } @@ -78,6 +82,6 @@ func NewNodeLabeler(client dynamic.Interface) NodeLabeler { } } -func (l NodeLabeler) Label(ctx context.Context, name, namespace string, newLabels map[string]string) error { - return l.generic.Label(ctx, name, namespace, newLabels) +func (l NodeLabeler) Label(ctx context.Context, name, namespace string, addLabels map[string]string) error { + return l.generic.Label(ctx, name, namespace, addLabels) } diff --git a/images/virtualization-dra/pkg/patch/patch.go b/images/virtualization-dra/pkg/patch/patch.go new file mode 100644 index 0000000000..eabe069522 --- /dev/null +++ b/images/virtualization-dra/pkg/patch/patch.go @@ -0,0 +1,110 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package patch + +import ( + "encoding/json" + "fmt" + "slices" + "strings" +) + +const ( + PatchReplaceOp = "replace" + PatchAddOp = "add" + PatchRemoveOp = "remove" + PatchTestOp = "test" +) + +type JSONPatch struct { + operations []JSONPatchOperation +} + +type JSONPatchOperation struct { + Op string `json:"op"` + Path string `json:"path"` + Value interface{} `json:"value,omitempty"` +} + +func NewJSONPatch(patches ...JSONPatchOperation) *JSONPatch { + return &JSONPatch{ + operations: patches, + } +} + +func NewJSONPatchOperation(op, path string, value interface{}) JSONPatchOperation { + return JSONPatchOperation{ + Op: op, + Path: path, + Value: value, + } +} + +func WithAdd(path string, value interface{}) JSONPatchOperation { + return NewJSONPatchOperation(PatchAddOp, path, value) +} + +func WithRemove(path string) JSONPatchOperation { + return NewJSONPatchOperation(PatchRemoveOp, path, nil) +} + +func WithReplace(path string, value interface{}) JSONPatchOperation { + return NewJSONPatchOperation(PatchReplaceOp, path, value) +} + +func WithTest(path string, value interface{}) JSONPatchOperation { + return NewJSONPatchOperation(PatchTestOp, path, value) +} + +func (jp *JSONPatch) Operations() []JSONPatchOperation { + return jp.operations +} + +func (jp *JSONPatch) Append(patches ...JSONPatchOperation) { + jp.operations = append(jp.operations, patches...) +} + +func (jp *JSONPatch) Delete(op, path string) { + slices.DeleteFunc(jp.operations, func(o JSONPatchOperation) bool { + return o.Op == op && o.Path == path + }) +} + +func (jp *JSONPatch) Len() int { + return len(jp.operations) +} + +func (jp *JSONPatch) String() (string, error) { + bytes, err := jp.Bytes() + if err != nil { + return "", err + } + return string(bytes), nil +} + +func (jp *JSONPatch) Bytes() ([]byte, error) { + if jp.Len() == 0 { + return nil, fmt.Errorf("list of patches is empty") + } + return json.Marshal(jp.operations) +} + +func EscapeJSONPointer(path string) string { + path = strings.ReplaceAll(path, "~", "~0") + path = strings.ReplaceAll(path, "/", "~1") + return path +} From 5881eb938f2aa47373cd183a68e526eb4c093a4d Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Fri, 23 Jan 2026 18:02:52 +0300 Subject: [PATCH 36/37] add inotify monitor Signed-off-by: Yaroslav Borbat --- .../cmd/go-usbip/app/run.go | 2 +- .../cmd/usb-gateway/app/app.go | 2 +- .../cmd/virtualization-dra-plugin/app/app.go | 2 +- .../pkg/libusb/dbus-monitor.go | 307 ++++++++++++++++ .../pkg/libusb/device-store.go | 218 +++++++++++ .../pkg/libusb/discovery.go | 2 +- .../pkg/libusb/inotify-monitor.go | 342 ++++++++++++++++++ .../virtualization-dra/pkg/libusb/monitor.go | 88 ++++- .../pkg/libusb/udev-monitor.go | 141 +------- 9 files changed, 966 insertions(+), 138 deletions(-) create mode 100644 images/virtualization-dra/pkg/libusb/dbus-monitor.go create mode 100644 images/virtualization-dra/pkg/libusb/device-store.go create mode 100644 images/virtualization-dra/pkg/libusb/inotify-monitor.go diff --git a/images/virtualization-dra/cmd/go-usbip/app/run.go b/images/virtualization-dra/cmd/go-usbip/app/run.go index b6da54eede..7b501e82f1 100644 --- a/images/virtualization-dra/cmd/go-usbip/app/run.go +++ b/images/virtualization-dra/cmd/go-usbip/app/run.go @@ -28,7 +28,7 @@ import ( func NewRunCommand() *cobra.Command { o := &runOptions{ - monitor: &libusb.MonitorConfig{}, + monitor: libusb.NewDefaultMonitorConfig(), } cmd := &cobra.Command{ Use: "run", diff --git a/images/virtualization-dra/cmd/usb-gateway/app/app.go b/images/virtualization-dra/cmd/usb-gateway/app/app.go index 28e947ae25..ba4b684569 100644 --- a/images/virtualization-dra/cmd/usb-gateway/app/app.go +++ b/images/virtualization-dra/cmd/usb-gateway/app/app.go @@ -69,7 +69,7 @@ func NewUSBGatewayCommand() *cobra.Command { func newUsbOptions() *usbOptions { return &usbOptions{ Logging: &logger.Options{}, - Monitor: &libusb.MonitorConfig{}, + Monitor: libusb.NewDefaultMonitorConfig(), } } diff --git a/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go b/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go index a706d56b37..94874c997c 100644 --- a/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go +++ b/images/virtualization-dra/cmd/virtualization-dra-plugin/app/app.go @@ -78,7 +78,7 @@ func newDraOptions() *draOptions { KubeletPluginsDirectoryPath: os.Getenv("KUBELET_PLUGINS_DIRECTORY_PATH"), HealthzPort: 51515, Logging: &logger.Options{}, - Monitor: &libusb.MonitorConfig{}, + Monitor: libusb.NewDefaultMonitorConfig(), featureGates: featuregates.AddFlags, } diff --git a/images/virtualization-dra/pkg/libusb/dbus-monitor.go b/images/virtualization-dra/pkg/libusb/dbus-monitor.go new file mode 100644 index 0000000000..fd6e03f9af --- /dev/null +++ b/images/virtualization-dra/pkg/libusb/dbus-monitor.go @@ -0,0 +1,307 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package libusb + +import ( + "context" + "log/slog" + "sync" + "time" + + "github.com/godbus/dbus/v5" +) + +// DBusMonitor is a USB device monitor that uses D-Bus to listen for USB device events +// via UDisks2 interface. It provides an alternative to udev-based monitoring. +type DBusMonitor struct { + store *USBDeviceStore + log *slog.Logger + + // Configuration + resyncPeriod time.Duration + reconnectDelay time.Duration + debounceDuration time.Duration + + // Debouncing + debounceTimer *time.Timer + debounceMu sync.Mutex +} + +// DBusMonitorOption is a functional option for DBusMonitor +type DBusMonitorOption func(*DBusMonitor) + +// DBusWithResyncPeriod sets the resync period +func DBusWithResyncPeriod(d time.Duration) DBusMonitorOption { + return func(m *DBusMonitor) { + m.resyncPeriod = d + } +} + +// DBusWithReconnectDelay sets the delay before reconnecting after an error +func DBusWithReconnectDelay(d time.Duration) DBusMonitorOption { + return func(m *DBusMonitor) { + m.reconnectDelay = d + } +} + +// DBusWithLogger sets the logger +func DBusWithLogger(log *slog.Logger) DBusMonitorOption { + return func(m *DBusMonitor) { + m.log = log + } +} + +// DBusWithDebounceDuration sets the debounce duration for events +func DBusWithDebounceDuration(d time.Duration) DBusMonitorOption { + return func(m *DBusMonitor) { + m.debounceDuration = d + } +} + +// NewDBusMonitor creates a new USB monitor that uses D-Bus +func NewDBusMonitor(ctx context.Context, opts ...DBusMonitorOption) (Monitor, error) { + devices, err := DiscoverPluggedUSBDevices() + if err != nil { + return nil, err + } + + log := slog.With(slog.String("component", "dbus-usb-monitor")) + + m := &DBusMonitor{ + store: NewUSBDeviceStore(devices, log), + log: log, + resyncPeriod: 5 * time.Minute, + reconnectDelay: 5 * time.Second, + debounceDuration: 200 * time.Millisecond, + } + + for _, opt := range opts { + opt(m) + } + + go m.run(ctx) + + return m, nil +} + +func (m *DBusMonitor) run(ctx context.Context) { + m.log.Info("D-Bus USB monitor started", + slog.Duration("resync_period", m.resyncPeriod), + slog.Duration("reconnect_delay", m.reconnectDelay), + ) + + for { + select { + case <-ctx.Done(): + m.log.Info("D-Bus USB monitor stopped") + return + default: + if err := m.runMonitor(ctx); err != nil { + m.log.Error("D-Bus monitor error, will reconnect", + slog.String("error", err.Error()), + slog.Duration("delay", m.reconnectDelay), + ) + } + + select { + case <-ctx.Done(): + return + case <-time.After(m.reconnectDelay): + // Retry connection + } + } + } +} + +func (m *DBusMonitor) runMonitor(ctx context.Context) error { + conn, err := dbus.ConnectSystemBus(dbus.WithContext(ctx)) + if err != nil { + return err + } + defer conn.Close() + + // Subscribe to UDisks2 ObjectManager signals + rules := []string{ + "type='signal',interface='org.freedesktop.DBus.ObjectManager',member='InterfacesAdded'", + "type='signal',interface='org.freedesktop.DBus.ObjectManager',member='InterfacesRemoved'", + } + + for _, rule := range rules { + call := conn.BusObject().Call("org.freedesktop.DBus.AddMatch", 0, rule) + if call.Err != nil { + m.log.Error("failed to add D-Bus match rule", + slog.String("rule", rule), + slog.Any("error", call.Err), + ) + return call.Err + } + } + + signals := make(chan *dbus.Signal, 100) + conn.Signal(signals) + + resyncTicker := time.NewTicker(m.resyncPeriod) + defer resyncTicker.Stop() + + m.log.Debug("D-Bus signal listener started") + + for { + select { + case <-ctx.Done(): + return nil + case signal, ok := <-signals: + if !ok { + return nil + } + m.handleSignal(signal) + case <-resyncTicker.C: + m.resync() + } + } +} + +func (m *DBusMonitor) handleSignal(signal *dbus.Signal) { + switch signal.Name { + case "org.freedesktop.DBus.ObjectManager.InterfacesAdded": + m.handleInterfacesAdded(signal) + case "org.freedesktop.DBus.ObjectManager.InterfacesRemoved": + m.handleInterfacesRemoved(signal) + } +} + +func (m *DBusMonitor) handleInterfacesAdded(signal *dbus.Signal) { + if len(signal.Body) < 2 { + return + } + + path, ok := signal.Body[0].(dbus.ObjectPath) + if !ok { + return + } + + interfaces, ok := signal.Body[1].(map[string]map[string]dbus.Variant) + if !ok { + return + } + + if m.isUSBDevice(interfaces) { + m.log.Debug("USB device connected via D-Bus", slog.String("path", string(path))) + m.scheduleResync() + } +} + +func (m *DBusMonitor) handleInterfacesRemoved(signal *dbus.Signal) { + if len(signal.Body) < 2 { + return + } + + path, ok := signal.Body[0].(dbus.ObjectPath) + if !ok { + return + } + + interfaces, ok := signal.Body[1].([]string) + if !ok { + return + } + + if m.containsUSBInterface(interfaces) { + m.log.Debug("USB device disconnected via D-Bus", slog.String("path", string(path))) + m.scheduleResync() + } +} + +// scheduleResync schedules a resync with debouncing to avoid multiple +// rapid resyncs when multiple events arrive in quick succession. +func (m *DBusMonitor) scheduleResync() { + m.debounceMu.Lock() + defer m.debounceMu.Unlock() + + // Cancel existing timer if any + if m.debounceTimer != nil { + m.debounceTimer.Stop() + } + + m.debounceTimer = time.AfterFunc(m.debounceDuration, func() { + m.resync() + }) +} + +func (m *DBusMonitor) isUSBDevice(interfaces map[string]map[string]dbus.Variant) bool { + if drive, ok := interfaces["org.freedesktop.UDisks2.Drive"]; ok { + if connectionBus, ok := drive["ConnectionBus"]; ok { + bus, _ := connectionBus.Value().(string) + return bus == "usb" + } + } + + if block, ok := interfaces["org.freedesktop.UDisks2.Block"]; ok { + if _, ok := block["Drive"]; ok { + // If it has a link to drive, it can be USB + return true + } + } + + return false +} + +func (m *DBusMonitor) containsUSBInterface(interfaces []string) bool { + for _, iface := range interfaces { + if iface == "org.freedesktop.UDisks2.Drive" || + iface == "org.freedesktop.UDisks2.Block" { + return true + } + } + return false +} + +func (m *DBusMonitor) resync() { + devices, err := DiscoverPluggedUSBDevices() + if err != nil { + m.log.Error("failed to discover USB devices during resync", + slog.String("error", err.Error()), + ) + return + } + + m.store.Resync(devices) +} + +// GetDevices returns a copy of all discovered USB devices +func (m *DBusMonitor) GetDevices() []USBDevice { + return m.store.GetDevices() +} + +// GetDevice returns a device by path +func (m *DBusMonitor) GetDevice(path string) (*USBDevice, bool) { + return m.store.GetDevice(path) +} + +// GetDeviceByBusID returns a device by BusID +func (m *DBusMonitor) GetDeviceByBusID(busID string) (*USBDevice, bool) { + return m.store.GetDeviceByBusID(busID) +} + +// AddNotifier adds a notifier to be called on device changes +func (m *DBusMonitor) AddNotifier(notifier Notifier) { + m.store.AddNotifier(notifier) +} + +// RemoveNotifier removes a notifier +func (m *DBusMonitor) RemoveNotifier(notifier Notifier) { + m.store.RemoveNotifier(notifier) +} diff --git a/images/virtualization-dra/pkg/libusb/device-store.go b/images/virtualization-dra/pkg/libusb/device-store.go new file mode 100644 index 0000000000..f7e3a7794a --- /dev/null +++ b/images/virtualization-dra/pkg/libusb/device-store.go @@ -0,0 +1,218 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package libusb + +import ( + "log/slog" + "slices" + "strings" + "sync" +) + +// USBDeviceStore provides thread-safe storage for USB devices with notification support. +type USBDeviceStore struct { + mu sync.RWMutex + devices map[string]*USBDevice + notifiers []Notifier + log *slog.Logger +} + +// NewUSBDeviceStore creates a new device store with initial devices. +func NewUSBDeviceStore(devices map[string]*USBDevice, log *slog.Logger) *USBDeviceStore { + if devices == nil { + devices = make(map[string]*USBDevice) + } + return &USBDeviceStore{ + devices: devices, + log: log, + } +} + +// GetDevices returns a sorted copy of all discovered USB devices. +func (s *USBDeviceStore) GetDevices() []USBDevice { + s.mu.RLock() + devices := make([]USBDevice, 0, len(s.devices)) + for _, device := range s.devices { + devices = append(devices, *device) + } + s.mu.RUnlock() + + slices.SortFunc(devices, func(a, b USBDevice) int { + return strings.Compare(a.DevicePath, b.DevicePath) + }) + + return devices +} + +// GetDevice returns a device by path. +func (s *USBDeviceStore) GetDevice(path string) (*USBDevice, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + + device, ok := s.devices[path] + if !ok { + return nil, false + } + + deviceCopy := *device + return &deviceCopy, true +} + +// GetDeviceByBusID returns a device by BusID. +func (s *USBDeviceStore) GetDeviceByBusID(busID string) (*USBDevice, bool) { + s.mu.RLock() + defer s.mu.RUnlock() + + for _, device := range s.devices { + if device.BusID == busID { + deviceCopy := *device + return &deviceCopy, true + } + } + return nil, false +} + +// AddNotifier adds a notifier to be called on device changes. +func (s *USBDeviceStore) AddNotifier(notifier Notifier) { + s.mu.Lock() + defer s.mu.Unlock() + + s.notifiers = append(s.notifiers, notifier) +} + +// RemoveNotifier removes a notifier. +func (s *USBDeviceStore) RemoveNotifier(notifier Notifier) { + s.mu.Lock() + defer s.mu.Unlock() + + for i, n := range s.notifiers { + if n == notifier { + s.notifiers = slices.Delete(s.notifiers, i, i+1) + return + } + } +} + +// AddDevice adds or updates a device and notifies if changed. +func (s *USBDeviceStore) AddDevice(path string, device *USBDevice) bool { + s.mu.Lock() + defer s.mu.Unlock() + + oldDevice, exists := s.devices[path] + if !exists { + s.devices[path] = device + s.log.Info("device added", + slog.String("path", path), + slog.String("busid", device.BusID), + slog.String("product", device.Product), + slog.String("manufacturer", device.Manufacturer), + slog.String("serial", device.Serial), + ) + s.notify() + return true + } + + if !device.Equal(oldDevice) { + s.devices[path] = device + s.log.Info("device updated", + slog.String("path", path), + slog.String("busid", device.BusID), + slog.String("product", device.Product), + ) + s.notify() + return true + } + + return false +} + +// RemoveDevice removes a device and notifies if it existed. +func (s *USBDeviceStore) RemoveDevice(path string) bool { + s.mu.Lock() + defer s.mu.Unlock() + + device, exists := s.devices[path] + if exists { + s.log.Info("device removed", + slog.String("path", path), + slog.String("busid", device.BusID), + ) + delete(s.devices, path) + s.notify() + return true + } + return false +} + +// Resync synchronizes the store with discovered devices and notifies if changed. +func (s *USBDeviceStore) Resync(devices map[string]*USBDevice) bool { + if devices == nil { + devices = make(map[string]*USBDevice) + } + + s.mu.Lock() + defer s.mu.Unlock() + + changed := false + + // Check for removed devices + for path := range s.devices { + if _, exists := devices[path]; !exists { + s.log.Info("device removed (resync)", slog.String("path", path)) + delete(s.devices, path) + changed = true + } + } + + // Check for added or changed devices + for path, device := range devices { + oldDevice, exists := s.devices[path] + if !exists { + s.log.Info("device added (resync)", + slog.String("path", path), + slog.String("busid", device.BusID), + slog.String("product", device.Product), + ) + s.devices[path] = device + changed = true + } else if !device.Equal(oldDevice) { + s.log.Info("device changed (resync)", slog.String("path", path)) + s.devices[path] = device + changed = true + } + } + + if changed { + s.notify() + } + + return changed +} + +// Exists checks if a device exists at the given path. +func (s *USBDeviceStore) Exists(path string) bool { + s.mu.RLock() + defer s.mu.RUnlock() + _, exists := s.devices[path] + return exists +} + +func (s *USBDeviceStore) notify() { + for _, notifier := range s.notifiers { + notifier.Notify() + } +} diff --git a/images/virtualization-dra/pkg/libusb/discovery.go b/images/virtualization-dra/pkg/libusb/discovery.go index 176b7dca59..85aa3fff25 100644 --- a/images/virtualization-dra/pkg/libusb/discovery.go +++ b/images/virtualization-dra/pkg/libusb/discovery.go @@ -1,5 +1,5 @@ /* -Copyright 2025 Flant JSC +Copyright 2026 Flant JSC Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/images/virtualization-dra/pkg/libusb/inotify-monitor.go b/images/virtualization-dra/pkg/libusb/inotify-monitor.go new file mode 100644 index 0000000000..644fd55c92 --- /dev/null +++ b/images/virtualization-dra/pkg/libusb/inotify-monitor.go @@ -0,0 +1,342 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package libusb + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "sync" + "time" + + "github.com/fsnotify/fsnotify" +) + +// InotifyMonitor is a USB device monitor that uses inotify/fsnotify to watch +// the sysfs USB devices directory and uevent files for changes. +type InotifyMonitor struct { + store *USBDeviceStore + watcher *fsnotify.Watcher + log *slog.Logger + + // Configuration + resyncPeriod time.Duration + debounceDuration time.Duration + + // Debouncing + debounceTimer *time.Timer + debounceMu sync.Mutex + + // Track watched paths + watchedPaths map[string]struct{} + watchMu sync.Mutex +} + +// InotifyMonitorOption is a functional option for InotifyMonitor +type InotifyMonitorOption func(*InotifyMonitor) + +// InotifyWithResyncPeriod sets the resync period +func InotifyWithResyncPeriod(d time.Duration) InotifyMonitorOption { + return func(m *InotifyMonitor) { + m.resyncPeriod = d + } +} + +// InotifyWithDebounceDuration sets the debounce duration for events +func InotifyWithDebounceDuration(d time.Duration) InotifyMonitorOption { + return func(m *InotifyMonitor) { + m.debounceDuration = d + } +} + +// InotifyWithLogger sets the logger +func InotifyWithLogger(log *slog.Logger) InotifyMonitorOption { + return func(m *InotifyMonitor) { + m.log = log + } +} + +// NewInotifyMonitor creates a new USB monitor that uses inotify/fsnotify +func NewInotifyMonitor(ctx context.Context, opts ...InotifyMonitorOption) (Monitor, error) { + devices, err := DiscoverPluggedUSBDevices() + if err != nil { + return nil, err + } + + watcher, err := fsnotify.NewWatcher() + if err != nil { + return nil, err + } + + log := slog.With(slog.String("component", "inotify-usb-monitor")) + + m := &InotifyMonitor{ + store: NewUSBDeviceStore(devices, log), + watcher: watcher, + log: log, + resyncPeriod: 5 * time.Minute, + debounceDuration: 200 * time.Millisecond, + watchedPaths: make(map[string]struct{}), + } + + for _, opt := range opts { + opt(m) + } + + // Watch the main USB devices directory + if err := m.watchDirectory(PathToUSBDevices); err != nil { + watcher.Close() + return nil, err + } + + // Watch existing device directories and their uevent files + m.watchExistingDevices() + + go m.run(ctx) + + return m, nil +} + +func (m *InotifyMonitor) run(ctx context.Context) { + resyncTicker := time.NewTicker(m.resyncPeriod) + defer resyncTicker.Stop() + defer m.watcher.Close() + + m.log.Info("Inotify USB monitor started", + slog.Duration("resync_period", m.resyncPeriod), + slog.Duration("debounce_duration", m.debounceDuration), + ) + + for { + select { + case <-ctx.Done(): + m.log.Info("Inotify USB monitor stopped") + return + + case event, ok := <-m.watcher.Events: + if !ok { + m.log.Debug("watcher events channel closed") + return + } + m.handleEvent(event) + + case err, ok := <-m.watcher.Errors: + if !ok { + m.log.Debug("watcher errors channel closed") + return + } + m.log.Error("inotify watcher error", slog.String("error", err.Error())) + + case <-resyncTicker.C: + m.resync() + } + } +} + +func (m *InotifyMonitor) handleEvent(event fsnotify.Event) { + m.log.Debug("received fsnotify event", + slog.String("name", event.Name), + slog.String("op", event.Op.String()), + ) + + // Handle directory creation (new USB device) + if event.Op&fsnotify.Create != 0 { + m.handleCreate(event.Name) + } + + // Handle directory removal (USB device disconnected) + if event.Op&fsnotify.Remove != 0 { + m.handleRemove(event.Name) + } + + // Handle uevent file changes + if event.Op&fsnotify.Write != 0 { + m.handleWrite(event.Name) + } +} + +func (m *InotifyMonitor) handleCreate(path string) { + // Only handle new USB device directories in /sys/bus/usb/devices/ + if filepath.Dir(path) != PathToUSBDevices { + return + } + + if !isUsbPath(path) { + return + } + + m.log.Debug("new USB device directory", slog.String("path", path)) + m.watchDeviceDirectory(path) + m.scheduleResync() +} + +func (m *InotifyMonitor) handleRemove(path string) { + // Check if this was a watched USB device directory + if filepath.Dir(path) == PathToUSBDevices { + m.log.Debug("USB device directory removed", slog.String("path", path)) + m.unwatchPath(path) + m.unwatchPath(filepath.Join(path, "uevent")) + m.scheduleResync() + return + } + + // For any other watched path that was removed + m.unwatchPath(path) +} + +func (m *InotifyMonitor) handleWrite(path string) { + // Write events are only for files, not directories. + // We only care about uevent file changes. + if filepath.Base(path) != "uevent" { + return + } + + devicePath := filepath.Dir(path) + m.log.Debug("uevent file changed", slog.String("path", devicePath)) + m.scheduleResync() +} + +func (m *InotifyMonitor) watchDirectory(path string) error { + m.watchMu.Lock() + defer m.watchMu.Unlock() + + if _, exists := m.watchedPaths[path]; exists { + return nil + } + + if err := m.watcher.Add(path); err != nil { + return err + } + + m.watchedPaths[path] = struct{}{} + m.log.Debug("watching directory", slog.String("path", path)) + return nil +} + +func (m *InotifyMonitor) unwatchPath(path string) { + m.watchMu.Lock() + defer m.watchMu.Unlock() + + if _, exists := m.watchedPaths[path]; !exists { + return + } + + _ = m.watcher.Remove(path) + delete(m.watchedPaths, path) + m.log.Debug("unwatched path", slog.String("path", path)) +} + +func (m *InotifyMonitor) watchExistingDevices() { + entries, err := os.ReadDir(PathToUSBDevices) + if err != nil { + m.log.Error("failed to read USB devices directory", + slog.String("path", PathToUSBDevices), + slog.String("error", err.Error()), + ) + return + } + + for _, entry := range entries { + if !entry.IsDir() { + continue + } + + devicePath := filepath.Join(PathToUSBDevices, entry.Name()) + if !isUsbPath(devicePath) { + continue + } + + m.watchDeviceDirectory(devicePath) + } +} + +func (m *InotifyMonitor) watchDeviceDirectory(devicePath string) { + // Watch the device directory itself + if err := m.watchDirectory(devicePath); err != nil { + m.log.Debug("failed to watch device directory", + slog.String("path", devicePath), + slog.String("error", err.Error()), + ) + return + } + + // Watch the uevent file for changes + ueventPath := filepath.Join(devicePath, "uevent") + if _, err := os.Stat(ueventPath); err == nil { + if err := m.watchDirectory(ueventPath); err != nil { + m.log.Debug("failed to watch uevent file", + slog.String("path", ueventPath), + slog.String("error", err.Error()), + ) + } + } +} + +// scheduleResync schedules a resync with debouncing +func (m *InotifyMonitor) scheduleResync() { + m.debounceMu.Lock() + defer m.debounceMu.Unlock() + + if m.debounceTimer != nil { + m.debounceTimer.Stop() + } + + m.debounceTimer = time.AfterFunc(m.debounceDuration, func() { + m.resync() + }) +} + +func (m *InotifyMonitor) resync() { + devices, err := DiscoverPluggedUSBDevices() + if err != nil { + m.log.Error("failed to discover USB devices during resync", + slog.String("error", err.Error()), + ) + return + } + + // Update watches for any new devices + m.watchExistingDevices() + + m.store.Resync(devices) +} + +// GetDevices returns a copy of all discovered USB devices +func (m *InotifyMonitor) GetDevices() []USBDevice { + return m.store.GetDevices() +} + +// GetDevice returns a device by path +func (m *InotifyMonitor) GetDevice(path string) (*USBDevice, bool) { + return m.store.GetDevice(path) +} + +// GetDeviceByBusID returns a device by BusID +func (m *InotifyMonitor) GetDeviceByBusID(busID string) (*USBDevice, bool) { + return m.store.GetDeviceByBusID(busID) +} + +// AddNotifier adds a notifier to be called on device changes +func (m *InotifyMonitor) AddNotifier(notifier Notifier) { + m.store.AddNotifier(notifier) +} + +// RemoveNotifier removes a notifier +func (m *InotifyMonitor) RemoveNotifier(notifier Notifier) { + m.store.RemoveNotifier(notifier) +} diff --git a/images/virtualization-dra/pkg/libusb/monitor.go b/images/virtualization-dra/pkg/libusb/monitor.go index a168f90973..6be94fa98e 100644 --- a/images/virtualization-dra/pkg/libusb/monitor.go +++ b/images/virtualization-dra/pkg/libusb/monitor.go @@ -1,3 +1,19 @@ +/* +Copyright 2026 Flant JSC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package libusb import ( @@ -30,7 +46,11 @@ func (f FuncNotifier) Notify() { type MonitorType string const ( - UdevMonitorType MonitorType = "udev" + UdevMonitorType MonitorType = "udev" + DBusMonitorType MonitorType = "dbus" + InotifyMonitorType MonitorType = "inotify" + + DefaultMonitorType = InotifyMonitorType ) func (m *MonitorType) String() string { @@ -41,6 +61,10 @@ func (m *MonitorType) Set(s string) error { switch s { case ptr.To(UdevMonitorType).String(): *m = UdevMonitorType + case ptr.To(DBusMonitorType).String(): + *m = DBusMonitorType + case ptr.To(InotifyMonitorType).String(): + *m = InotifyMonitorType default: return fmt.Errorf("invalid monitor type: %s", s) } @@ -55,19 +79,32 @@ type MonitorConfig struct { MonitorType MonitorType // ALL - ResyncPeriod time.Duration - Logger *slog.Logger + ResyncPeriod time.Duration + DebounceDuration time.Duration + Logger *slog.Logger // UDEV - DebounceDuration time.Duration - HostNetNs bool + HostNetNs bool + + // DBUS + ReconnectDelay time.Duration +} + +// NewDefaultMonitorConfig creates a MonitorConfig with default values. +func NewDefaultMonitorConfig() *MonitorConfig { + return &MonitorConfig{ + MonitorType: DefaultMonitorType, + ResyncPeriod: 5 * time.Minute, + DebounceDuration: 200 * time.Millisecond, + } } func (c *MonitorConfig) AddFlags(fs *pflag.FlagSet) { - fs.Var(&c.MonitorType, "usb-monitor-type", "USB monitor type") + fs.Var(&c.MonitorType, "usb-monitor-type", fmt.Sprintf("USB monitor type: %s, %s, %s (default %q)", UdevMonitorType, DBusMonitorType, InotifyMonitorType, DefaultMonitorType)) fs.DurationVar(&c.ResyncPeriod, "usb-monitor-resync-period", c.ResyncPeriod, "USB monitor resync period") - fs.DurationVar(&c.DebounceDuration, "udev-usb-monitor-debounce-duration", c.DebounceDuration, "UDEV USB monitor debounce duration") + fs.DurationVar(&c.DebounceDuration, "usb-monitor-debounce-duration", c.DebounceDuration, "USB monitor debounce duration") fs.BoolVar(&c.HostNetNs, "udev-usb-monitor-host-netns", c.HostNetNs, "UDEV USB monitor host netns") + fs.DurationVar(&c.ReconnectDelay, "dbus-usb-monitor-reconnect-delay", c.ReconnectDelay, "D-Bus USB monitor reconnect delay") } func (c *MonitorConfig) Complete(ctx context.Context, logger *slog.Logger) (Monitor, error) { @@ -76,6 +113,10 @@ func (c *MonitorConfig) Complete(ctx context.Context, logger *slog.Logger) (Moni switch c.MonitorType { case UdevMonitorType: return NewUdevMonitor(ctx, c.makeUdevOpts()...) + case DBusMonitorType: + return NewDBusMonitor(ctx, c.makeDBusOpts()...) + case InotifyMonitorType: + return NewInotifyMonitor(ctx, c.makeInotifyOpts()...) default: return nil, fmt.Errorf("unsupported monitor type: %s", c.MonitorType) } @@ -98,3 +139,36 @@ func (c *MonitorConfig) makeUdevOpts() []UdevMonitorOption { return opts } + +func (c *MonitorConfig) makeDBusOpts() []DBusMonitorOption { + var opts []DBusMonitorOption + if c.ResyncPeriod > 0 { + opts = append(opts, DBusWithResyncPeriod(c.ResyncPeriod)) + } + if c.Logger != nil { + opts = append(opts, DBusWithLogger(c.Logger)) + } + if c.ReconnectDelay > 0 { + opts = append(opts, DBusWithReconnectDelay(c.ReconnectDelay)) + } + if c.DebounceDuration > 0 { + opts = append(opts, DBusWithDebounceDuration(c.DebounceDuration)) + } + + return opts +} + +func (c *MonitorConfig) makeInotifyOpts() []InotifyMonitorOption { + var opts []InotifyMonitorOption + if c.ResyncPeriod > 0 { + opts = append(opts, InotifyWithResyncPeriod(c.ResyncPeriod)) + } + if c.Logger != nil { + opts = append(opts, InotifyWithLogger(c.Logger)) + } + if c.DebounceDuration > 0 { + opts = append(opts, InotifyWithDebounceDuration(c.DebounceDuration)) + } + + return opts +} diff --git a/images/virtualization-dra/pkg/libusb/udev-monitor.go b/images/virtualization-dra/pkg/libusb/udev-monitor.go index a589d4826e..c35f22f652 100644 --- a/images/virtualization-dra/pkg/libusb/udev-monitor.go +++ b/images/virtualization-dra/pkg/libusb/udev-monitor.go @@ -21,8 +21,6 @@ import ( "log/slog" "os" "path/filepath" - "slices" - "strings" "sync" "time" @@ -33,10 +31,8 @@ import ( // It provides a clean separation between the generic udev event handling and // USB-specific device management. type UdevMonitor struct { - mu sync.RWMutex - devices map[string]*USBDevice - notifiers []Notifier - log *slog.Logger + store *USBDeviceStore + log *slog.Logger // Configuration resyncPeriod time.Duration @@ -92,13 +88,12 @@ func NewUdevMonitor(ctx context.Context, opts ...UdevMonitorOption) (Monitor, er if err != nil { return nil, err } - if devices == nil { - devices = make(map[string]*USBDevice) - } + + log := slog.With(slog.String("component", "udev-usb-monitor")) m := &UdevMonitor{ - devices: devices, - log: slog.With(slog.String("component", "usb-monitor")), + store: NewUSBDeviceStore(devices, log), + log: log, resyncPeriod: 5 * time.Minute, debounceDuration: 100 * time.Millisecond, pendingEvents: make(map[string]*debounceEntry), @@ -209,9 +204,6 @@ func (m *UdevMonitor) scheduleEvent(path string, action udev.Action) { } func (m *UdevMonitor) processEvent(path string, action udev.Action) { - m.mu.Lock() - defer m.mu.Unlock() - switch action { case udev.ActionAdd, udev.ActionChange, udev.ActionBind, udev.ActionOnline: m.handleDeviceUpdate(path) @@ -245,37 +237,11 @@ func (m *UdevMonitor) handleDeviceUpdate(path string) { return } - oldDevice, exists := m.devices[path] - if !exists { - m.devices[path] = &device - m.log.Info("device added", - slog.String("path", path), - slog.String("busid", device.BusID), - slog.String("product", device.Product), - slog.String("manufacturer", device.Manufacturer), - slog.String("serial", device.Serial), - ) - m.notify() - } else if !device.Equal(oldDevice) { - m.devices[path] = &device - m.log.Info("device updated", - slog.String("path", path), - slog.String("busid", device.BusID), - slog.String("product", device.Product), - ) - m.notify() - } + m.store.AddDevice(path, &device) } func (m *UdevMonitor) handleDeviceRemove(path string) { - if device, exists := m.devices[path]; exists { - m.log.Info("device removed", - slog.String("path", path), - slog.String("busid", device.BusID), - ) - delete(m.devices, path) - m.notify() - } + m.store.RemoveDevice(path) } func (m *UdevMonitor) resync() { @@ -284,112 +250,33 @@ func (m *UdevMonitor) resync() { m.log.Error("failed to discover USB devices during resync", slog.String("error", err.Error())) return } - if devices == nil { - devices = make(map[string]*USBDevice) - } - - m.mu.Lock() - defer m.mu.Unlock() - changed := false - - // Check for removed devices - for path := range m.devices { - if _, exists := devices[path]; !exists { - m.log.Info("device removed (resync)", slog.String("path", path)) - delete(m.devices, path) - changed = true - } - } - - // Check for added or changed devices - for path, device := range devices { - oldDevice, exists := m.devices[path] - if !exists { - m.log.Info("device added (resync)", slog.String("path", path)) - m.devices[path] = device - changed = true - } else if !device.Equal(oldDevice) { - m.log.Info("device changed (resync)", slog.String("path", path)) - m.devices[path] = device - changed = true - } - } - - if changed { - m.notify() - } + m.store.Resync(devices) } // GetDevices returns a copy of all discovered USB devices func (m *UdevMonitor) GetDevices() []USBDevice { - m.mu.RLock() - devices := make([]USBDevice, 0, len(m.devices)) - for _, device := range m.devices { - devices = append(devices, *device) - } - m.mu.RUnlock() - - slices.SortFunc(devices, func(a, b USBDevice) int { - return strings.Compare(a.DevicePath, b.DevicePath) - }) - - return devices + return m.store.GetDevices() } // GetDevice returns a device by path func (m *UdevMonitor) GetDevice(path string) (*USBDevice, bool) { - m.mu.RLock() - defer m.mu.RUnlock() - - device, ok := m.devices[path] - if !ok { - return nil, false - } - - deviceCopy := *device - return &deviceCopy, true + return m.store.GetDevice(path) } // GetDeviceByBusID returns a device by BusID func (m *UdevMonitor) GetDeviceByBusID(busID string) (*USBDevice, bool) { - m.mu.RLock() - defer m.mu.RUnlock() - - for _, device := range m.devices { - if device.BusID == busID { - deviceCopy := *device - return &deviceCopy, true - } - } - return nil, false + return m.store.GetDeviceByBusID(busID) } // AddNotifier adds a notifier to be called on device changes func (m *UdevMonitor) AddNotifier(notifier Notifier) { - m.mu.Lock() - defer m.mu.Unlock() - - m.notifiers = append(m.notifiers, notifier) + m.store.AddNotifier(notifier) } // RemoveNotifier removes a notifier func (m *UdevMonitor) RemoveNotifier(notifier Notifier) { - m.mu.Lock() - defer m.mu.Unlock() - - for i, n := range m.notifiers { - if n == notifier { - m.notifiers = slices.Delete(m.notifiers, i, i+1) - return - } - } -} - -func (m *UdevMonitor) notify() { - for _, notifier := range m.notifiers { - notifier.Notify() - } + m.store.RemoveNotifier(notifier) } func newUSBDeviceMatcher() udev.Matcher { From 8e6edd9cedf608d52c6d20e10ce5e364e68ae362 Mon Sep 17 00:00:00 2001 From: Yaroslav Borbat Date: Mon, 26 Jan 2026 13:38:27 +0300 Subject: [PATCH 37/37] reduce deckhouse version requirement (test: delete me) Signed-off-by: Yaroslav Borbat --- module.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/module.yaml b/module.yaml index 75b996428a..7ad413c6e1 100644 --- a/module.yaml +++ b/module.yaml @@ -8,7 +8,7 @@ descriptions: ru: Модуль виртуализации позволяет запускать и управлять виртуальными машинами в рамках платформы Deckhouse. tags: ["virtualization"] requirements: - deckhouse: ">= 1.74.2" + deckhouse: ">= 1.74" # TODO: FOR TEST. REVERT ME ">= 1.74.2" modules: cni-cilium: ">= 0.0.0" disable: