Add kernel args for rdma mode to complement the modprobe file

Signed-off-by: Sebastian Sch <[email protected]>

Author: SchSeba

Makefile

@@ -226,7 +226,7 @@ test-e2e-k8s: export NAMESPACE=sriov-network-operator
 test-e2e-k8s: test-e2e
 
 test-bindata-scripts: fakechroot
-	fakechroot ./test/scripts/enable-kargs_test.sh
+	fakechroot ./test/scripts/kargs_test.sh
 
 test-%: generate manifests envtest
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir=/tmp -p path)" HOME="$(shell pwd)" go test ./$*/... -coverprofile cover-$*.out -coverpkg ./... -v

bindata/scripts/enable-kargs.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+set -x
+
+command=$1
+shift
+declare -a kargs=( "$@" )
+ret=0
+args=$(chroot /host/ cat /proc/cmdline)
+
+if chroot /host/ test -f /run/ostree-booted ; then
+    for t in "${kargs[@]}";do
+        if [[ $command == "add" ]];then
+          if [[ $args != *${t}* ]];then
+              if chroot /host/ rpm-ostree kargs | grep -vq ${t}; then
+                  chroot /host/ rpm-ostree kargs --append ${t} > /dev/null 2>&1
+              fi
+              let ret++
+          fi
+        fi
+        if [[ $command == "remove" ]];then
+          if [[ $args == *${t}* ]];then
+                if chroot /host/ rpm-ostree kargs | grep -q ${t}; then
+                    chroot /host/ rpm-ostree kargs --delete ${t} > /dev/null 2>&1
+                fi
+                let ret++
+            fi
+        fi
+    done
+else
+    chroot /host/ which grubby > /dev/null 2>&1
+    # if grubby is not there, let's tell it
+    if [ $? -ne 0 ]; then
+        exit 127
+    fi
+    for t in "${kargs[@]}";do
+      if [[ $command == "add" ]];then
+        if [[ $args != *${t}* ]];then
+            if chroot /host/ grubby --info=DEFAULT | grep args | grep -vq ${t}; then
+                chroot /host/ grubby --update-kernel=DEFAULT --args=${t} > /dev/null 2>&1
+            fi
+            let ret++
+        fi
+      fi
+      if [[ $command == "remove" ]];then
+          if [[ $args == *${t}* ]];then
+            if chroot /host/ grubby --info=DEFAULT | grep args | grep -q ${t}; then
+                chroot /host/ grubby --update-kernel=DEFAULT --remove-args=${t} > /dev/null 2>&1
+            fi
+            let ret++
+          fi
+      fi
+    done
+fi
+
+echo $ret

bindata/scripts/kargs.sh

@@ -155,22 +155,22 @@ func (r *SriovNetworkNodePolicyReconciler) SetupWithManager(mgr ctrl.Manager) er
 	delayedEventHandler := handler.Funcs{
 		CreateFunc: func(ctx context.Context, e event.CreateEvent, q workqueue.RateLimitingInterface) {
 			log.Log.WithName("SriovNetworkNodePolicy").
-				Info("Enqueuing sync for create event", "resource", e.Object.GetName())
+				Info("Enqueuing sync for create event", "resource", e.Object.GetName(), "type", e.Object.GetObjectKind().GroupVersionKind().String())
 			qHandler(q)
 		},
 		UpdateFunc: func(ctx context.Context, e event.UpdateEvent, q workqueue.RateLimitingInterface) {
 			log.Log.WithName("SriovNetworkNodePolicy").
-				Info("Enqueuing sync for update event", "resource", e.ObjectNew.GetName())
+				Info("Enqueuing sync for update event", "resource", e.ObjectNew.GetName(), "type", e.ObjectNew.GetObjectKind().GroupVersionKind().String())
 			qHandler(q)
 		},
 		DeleteFunc: func(ctx context.Context, e event.DeleteEvent, q workqueue.RateLimitingInterface) {
 			log.Log.WithName("SriovNetworkNodePolicy").
-				Info("Enqueuing sync for delete event", "resource", e.Object.GetName())
+				Info("Enqueuing sync for delete event", "resource", e.Object.GetName(), "type", e.Object.GetObjectKind().GroupVersionKind().String())
 			qHandler(q)
 		},
 		GenericFunc: func(ctx context.Context, e event.GenericEvent, q workqueue.RateLimitingInterface) {
 			log.Log.WithName("SriovNetworkNodePolicy").
-				Info("Enqueuing sync for generic event", "resource", e.Object.GetName())
+				Info("Enqueuing sync for generic event", "resource", e.Object.GetName(), "type", e.Object.GetObjectKind().GroupVersionKind().String())
 			qHandler(q)
 		},
 	}
@@ -199,6 +199,7 @@ func (r *SriovNetworkNodePolicyReconciler) SetupWithManager(mgr ctrl.Manager) er
 		For(&sriovnetworkv1.SriovNetworkNodePolicy{}).
 		Watches(&corev1.Node{}, nodeEvenHandler).
 		Watches(&sriovnetworkv1.SriovNetworkNodePolicy{}, delayedEventHandler).
+		Watches(&sriovnetworkv1.SriovNetworkPoolConfig{}, delayedEventHandler).
 		WatchesRawSource(&source.Channel{Source: eventChan}, delayedEventHandler).
 		Complete(r)
 }
@@ -271,14 +272,14 @@ func (r *SriovNetworkNodePolicyReconciler) syncAllSriovNetworkNodeStates(ctx con
 		ns := &sriovnetworkv1.SriovNetworkNodeState{}
 		ns.Name = node.Name
 		ns.Namespace = vars.Namespace
-		j, _ := json.Marshal(ns)
 		netPoolConfig, _, err := findNodePoolConfig(ctx, &node, r.Client)
 		if err != nil {
-			log.Log.Error(err, "nodeStateSyncHandler(): failed to get SriovNetworkPoolConfig for the current node")
+			logger.Error(err, "failed to get SriovNetworkPoolConfig for the current node")
 		}
 		if netPoolConfig != nil {
 			ns.Spec.System.RdmaMode = netPoolConfig.Spec.RdmaMode
 		}
+		j, _ := json.Marshal(ns)
 		logger.V(2).Info("SriovNetworkNodeState CR", "content", j)
 		if err := r.syncSriovNetworkNodeState(ctx, dc, npl, ns, &node); err != nil {
 			logger.Error(err, "Fail to sync", "SriovNetworkNodeState", ns.Name)

controllers/sriovnetworknodepolicy_controller.go

@@ -38,6 +38,7 @@ require (
 	github.com/vishvananda/netlink v1.2.1-beta.2.0.20240221172127-ec7bcb248e94
 	github.com/vishvananda/netns v0.0.4
 	go.uber.org/zap v1.25.0
+	golang.org/x/net v0.23.0
 	golang.org/x/time v0.3.0
 	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c
 	gopkg.in/yaml.v3 v3.0.1
@@ -144,7 +145,6 @@ require (
 	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
 	golang.org/x/mod v0.13.0 // indirect
-	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/oauth2 v0.13.0 // indirect
 	golang.org/x/sync v0.4.0 // indirect
 	golang.org/x/sys v0.20.0 // indirect

go.mod

@@ -124,9 +124,11 @@ const (
 		`IMPORT{program}="/etc/udev/switchdev-vf-link-name.sh $attr{phys_port_name}", ` +
 		`NAME="%s_$env{NUMBER}"`
 
-	KernelArgPciRealloc = "pci=realloc"
-	KernelArgIntelIommu = "intel_iommu=on"
-	KernelArgIommuPt    = "iommu=pt"
+	KernelArgPciRealloc    = "pci=realloc"
+	KernelArgIntelIommu    = "intel_iommu=on"
+	KernelArgIommuPt       = "iommu=pt"
+	KernelArgRdmaShared    = "ib_core.netns_mode=1"
+	KernelArgRdmaExclusive = "ib_core.netns_mode=0"
 
 	// Feature gates
 	// ParallelNicConfigFeatureGate: allow to configure nics in parallel

pkg/consts/constants.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"math/rand"
-	"os/exec"
 	"reflect"
 	"sync"
 	"time"
@@ -429,16 +428,6 @@ func (dn *Daemon) nodeStateSyncHandler() error {
 		reqReboot = reqReboot || r
 	}
 
-	if dn.currentNodeState.Status.System.RdmaMode != dn.desiredNodeState.Spec.System.RdmaMode {
-		err = dn.HostHelpers.SetRDMASubsystem(dn.desiredNodeState.Spec.System.RdmaMode)
-		if err != nil {
-			log.Log.Error(err, "nodeStateSyncHandler(): failed to set RDMA subsystem")
-			return err
-		}
-		reqReboot = true
-		reqDrain = true
-	}
-
 	// When running using systemd check if the applied configuration is the latest one
 	// or there is a new config we need to apply
 	// When using systemd configuration we write the file
@@ -761,11 +750,11 @@ func (dn *Daemon) rebootNode() {
 	// However note we use `;` instead of `&&` so we keep rebooting even
 	// if kubelet failed to shutdown - that way the machine will still eventually reboot
 	// as systemd will time out the stop invocation.
-	cmd := exec.Command("systemd-run", "--unit", "sriov-network-config-daemon-reboot",
+	stdOut, StdErr, err := dn.HostHelpers.RunCommand("systemd-run", "--unit", "sriov-network-config-daemon-reboot",
 		"--description", "sriov-network-config-daemon reboot node", "/bin/sh", "-c", "systemctl stop kubelet.service; reboot")
 
-	if err := cmd.Run(); err != nil {
-		log.Log.Error(err, "failed to reboot node")
+	if err != nil {
+		log.Log.Error(err, "failed to reboot node", "stdOut", stdOut, "StdErr", StdErr)
 	}
 }
 

pkg/daemon/daemon.go

@@ -41,6 +41,13 @@ var _ = Describe("config daemon plugin loading tests", func() {
 			vars.ClusterType = consts.ClusterTypeKubernetes
 			gmockController = gomock.NewController(GinkgoT())
 			helperMock = helperMocks.NewMockHostHelpersInterface(gmockController)
+			helperMock.EXPECT().GetCurrentKernelArgs().Return("", nil).AnyTimes()
+			helperMock.EXPECT().IsKernelArgsSet("", consts.KernelArgIntelIommu).Return(false)
+			helperMock.EXPECT().IsKernelArgsSet("", consts.KernelArgIommuPt).Return(false)
+			helperMock.EXPECT().IsKernelArgsSet("", consts.KernelArgPciRealloc).Return(false)
+			helperMock.EXPECT().IsKernelArgsSet("", consts.KernelArgRdmaExclusive).Return(false)
+			helperMock.EXPECT().IsKernelArgsSet("", consts.KernelArgRdmaShared).Return(false)
+
 			// k8s plugin is ATM the only plugin which require mocking/faking, as its New method performs additional logic
 			// other than simple plugin struct initialization
 			K8sPlugin = func(_ helper.HostHelpersInterface) (plugin.VendorPlugin, error) {

pkg/daemon/plugin_test.go

@@ -189,6 +189,7 @@ func (w *NodeStateStatusWriter) setNodeStateStatus(msg Message) (*sriovnetworkv1
 	nodeState, err := w.updateNodeStateStatusRetry(func(nodeState *sriovnetworkv1.SriovNetworkNodeState) {
 		nodeState.Status.Interfaces = w.status.Interfaces
 		nodeState.Status.Bridges = w.status.Bridges
+		nodeState.Status.System = w.status.System
 		if msg.lastSyncError != "" || msg.syncStatus == consts.SyncStatusSucceeded {
 			// clear lastSyncError when sync Succeeded
 			nodeState.Status.LastSyncError = msg.lastSyncError