[AKS] az aks create, aks nodepool add: Add --kubelet-config and --linux-os-config to support custom node configuration

src/azure-cli/azure/cli/command_modules/acs/_help.py

@@ -503,6 +503,12 @@
   - name: --snapshot-id
     type: string
     short-summary: The source snapshot id used to create this cluster.
+  - name: --kubelet-config
+    type: string
+    short-summary: Path to JSON file containing Kubelet configurations for agent nodes. https://aka.ms/aks/custom-node-config
+  - name: --linux-os-config
+    type: string
+    short-summary: Path to JSON file containing OS configurations for Linux agent nodes. https://aka.ms/aks/custom-node-config
 examples:
   - name: Create a Kubernetes cluster with an existing SSH public key.
     text: az aks create -g MyResourceGroup -n MyManagedCluster --ssh-key-value /path/to/publickey
@@ -994,6 +1000,12 @@
   - name: --aks-custom-headers
     type: string
     short-summary: Comma-separated key-value pairs to specify custom headers.
+  - name: --kubelet-config
+    type: string
+    short-summary: Path to JSON file containing Kubelet configurations for agent nodes. https://aka.ms/aks/custom-node-config
+  - name: --linux-os-config
+    type: string
+    short-summary: Path to JSON file containing OS configurations for Linux agent nodes. https://aka.ms/aks/custom-node-config
 examples:
   - name: Create a nodepool in an existing AKS cluster with ephemeral os enabled.
     text: az aks nodepool add -g MyResourceGroup -n nodepool1 --cluster-name MyManagedCluster --node-osdisk-type Ephemeral --node-osdisk-size 48

src/azure-cli/azure/cli/command_modules/acs/_params.py

@@ -314,6 +314,8 @@ def load_arguments(self, _):
         c.argument('enable_sgxquotehelper', action='store_true')
         c.argument('enable_fips_image', action='store_true')
         c.argument('snapshot_id', validator=validate_snapshot_id)
+        c.argument('kubelet_config')
+        c.argument('linux_os_config')
 
     with self.argument_context('aks update') as c:
         c.argument('enable_cluster_autoscaler', options_list=[
@@ -469,6 +471,8 @@ def load_arguments(self, _):
         c.argument('enable_ultra_ssd', options_list=['--enable-ultra-ssd'], action='store_true')
         c.argument('enable_fips_image', action='store_true')
         c.argument('snapshot_id', validator=validate_snapshot_id)
+        c.argument('kubelet_config')
+        c.argument('linux_os_config')
 
     with self.argument_context('aks nodepool upgrade') as c:
         c.argument('snapshot_id', validator=validate_snapshot_id)

src/azure-cli/azure/cli/command_modules/acs/custom.py

@@ -59,7 +59,7 @@
 from azure.cli.core.profiles import ResourceType
 from azure.cli.core.commands.client_factory import get_mgmt_service_client, get_subscription_id
 from azure.cli.core.keys import is_valid_ssh_rsa_public_key
-from azure.cli.core.util import in_cloud_console, shell_safe_json_parse, truncate_text, sdk_no_wait
+from azure.cli.core.util import get_file_json, in_cloud_console, shell_safe_json_parse, truncate_text, sdk_no_wait
 from azure.cli.core.commands import LongRunningOperation
 from azure.graphrbac.models import (ApplicationCreateParameters,
                                     ApplicationUpdateParameters,
@@ -2007,6 +2007,8 @@ def aks_create(cmd, client, resource_group_name, name, ssh_key_value,  # pylint:
                gmsa_dns_server=None,
                gmsa_root_domain_name=None,
                snapshot_id=None,
+               kubelet_config=None,
+               linux_os_config=None,
                ):
     # DO NOT MOVE: get all the original parameters and save them as a dictionary
     raw_parameters = locals()
@@ -3137,6 +3139,8 @@ def aks_agentpool_add(cmd, client, resource_group_name, cluster_name, nodepool_n
                       spot_max_price=float('nan'),
                       tags=None,
                       labels=None,
+                      kubelet_config=None,
+                      linux_os_config=None,
                       max_surge=None,
                       mode="User",
                       enable_encryption_at_host=False,
@@ -3241,6 +3245,12 @@ def aks_agentpool_add(cmd, client, resource_group_name, cluster_name, nodepool_n
     if node_osdisk_type:
         agent_pool.os_disk_type = node_osdisk_type
 
+    if kubelet_config:
+        agent_pool.kubelet_config = _get_kubelet_config(kubelet_config)
+
+    if linux_os_config:
+        agent_pool.linux_os_config = _get_linux_os_config(linux_os_config)
+
     # custom headers
     aks_custom_headers = extract_comma_separated_string(
         aks_custom_headers,
@@ -4231,3 +4241,24 @@ def aks_snapshot_list(cmd, client, resource_group_name=None):  # pylint: disable
         return client.list()
 
     return client.list_by_resource_group(resource_group_name)
+
+
+def _get_kubelet_config(file_path):
+    if not os.path.isfile(file_path):
+        raise InvalidArgumentValueError("{} is not valid file, or not accessable.".format(file_path))
+    kubelet_config = get_file_json(file_path)
+    if not isinstance(kubelet_config, dict):
+        msg = "Error reading kubelet configuration at {}. Please see https://aka.ms/CustomNodeConfig for proper format."
+        raise InvalidArgumentValueError(msg.format(file_path))
+    return kubelet_config
+
+
+def _get_linux_os_config(file_path):
+    if not os.path.isfile(file_path):
+        raise InvalidArgumentValueError("{} is not valid file, or not accessable.".format(file_path))
+    os_config = get_file_json(file_path)
+    if not isinstance(os_config, dict):
+        msg = "Error reading Linux OS configuration at {}. \
+            Please see https://aka.ms/CustomNodeConfig for proper format."
+        raise InvalidArgumentValueError(msg.format(file_path))
+    return os_config

src/azure-cli/azure/cli/command_modules/acs/decorator.py

@@ -3,6 +3,7 @@
 # Licensed under the MIT License. See License.txt in the project root for license information.
 # --------------------------------------------------------------------------------------------
 
+import os
 import re
 import time
 from distutils.version import StrictVersion
@@ -51,7 +52,7 @@
 from azure.cli.core.commands import AzCliCommand
 from azure.cli.core.keys import is_valid_ssh_rsa_public_key
 from azure.cli.core.profiles import ResourceType
-from azure.cli.core.util import truncate_text
+from azure.cli.core.util import truncate_text, get_file_json
 from azure.core.exceptions import HttpResponseError
 from knack.log import get_logger
 from knack.prompting import NoTTYException, prompt, prompt_pass, prompt_y_n
@@ -69,6 +70,8 @@
 ResourceReference = TypeVar("ResourceReference")
 ManagedClusterAddonProfile = TypeVar("ManagedClusterAddonProfile")
 Snapshot = TypeVar("Snapshot")
+KubeletConfig = TypeVar("KubeletConfig")
+LinuxOSConfig = TypeVar("LinuxOSConfig")
 
 # TODO
 # add validation for all/some of the parameters involved in the getter of outbound_type/enable_addons
@@ -274,6 +277,16 @@ def __init__(
             resource_type=self.resource_type,
             operation_group="managed_clusters",
         )
+        self.KubeletConfig = self.__cmd.get_models(
+            "KubeletConfig",
+            resource_type=self.resource_type,
+            operation_group="managed_clusters",
+        )
+        self.LinuxOSConfig = self.__cmd.get_models(
+            "LinuxOSConfig",
+            resource_type=self.resource_type,
+            operation_group="managed_clusters",
+        )
         # init load balancer models
         self.init_lb_models()
 
@@ -4770,6 +4783,86 @@ def get_assignee_from_identity_or_sp_profile(self) -> Tuple[str, bool]:
             raise UnknownError('Cannot get the AKS cluster\'s service principal.')
         return assignee, is_service_principal
 
+    def get_kubelet_config(self) -> Union[dict, KubeletConfig, None]:
+        """Obtain the value of kubelet_config.
+
+        :return: dict, KubeletConfig or None
+        """
+        # read the original value passed by the command
+        kubelet_config = None
+        kubelet_config_file_path = self.raw_param.get("kubelet_config")
+        # validate user input
+        if kubelet_config_file_path:
+            if not os.path.isfile(kubelet_config_file_path):
+                raise InvalidArgumentValueError(
+                    "{} is not valid file, or not accessable.".format(
+                        kubelet_config_file_path
+                    )
+                )
+            kubelet_config = get_file_json(kubelet_config_file_path)
+            if not isinstance(kubelet_config, dict):
+                raise InvalidArgumentValueError(
+                    "Error reading kubelet configuration from {}. "
+                    "Please see https://aka.ms/CustomNodeConfig for correct format.".format(
+                        kubelet_config_file_path
+                    )
+                )
+
+        # try to read the property value corresponding to the parameter from the `mc` object
+        if self.mc and self.mc.agent_pool_profiles:
+            agent_pool_profile = safe_list_get(
+                self.mc.agent_pool_profiles, 0, None
+            )
+            if (
+                agent_pool_profile and
+                agent_pool_profile.kubelet_config is not None
+            ):
+                kubelet_config = agent_pool_profile.kubelet_config
+
+        # this parameter does not need dynamic completion
+        # this parameter does not need validation
+        return kubelet_config
+
+    def get_linux_os_config(self) -> Union[dict, LinuxOSConfig, None]:
+        """Obtain the value of linux_os_config.
+
+        :return: dict, LinuxOSConfig or None
+        """
+        # read the original value passed by the command
+        linux_os_config = None
+        linux_os_config_file_path = self.raw_param.get("linux_os_config")
+        # validate user input
+        if linux_os_config_file_path:
+            if not os.path.isfile(linux_os_config_file_path):
+                raise InvalidArgumentValueError(
+                    "{} is not valid file, or not accessable.".format(
+                        linux_os_config_file_path
+                    )
+                )
+            linux_os_config = get_file_json(linux_os_config_file_path)
+            if not isinstance(linux_os_config, dict):
+                raise InvalidArgumentValueError(
+                    "Error reading Linux OS configuration from {}. "
+                    "Please see https://aka.ms/CustomNodeConfig for correct format.".format(
+                        linux_os_config_file_path
+                    )
+                )
+
+        # try to read the property value corresponding to the parameter from the `mc` object
+        if self.mc and self.mc.agent_pool_profiles:
+            agent_pool_profile = safe_list_get(
+                self.mc.agent_pool_profiles, 0, None
+            )
+            if (
+                agent_pool_profile and
+                agent_pool_profile.linux_os_config is not None
+            ):
+                linux_os_config = agent_pool_profile.linux_os_config
+
+        # this parameter does not need dynamic completion
+        # this parameter does not need validation
+        return linux_os_config
+
 
 class AKSCreateDecorator:
     def __init__(
@@ -4860,6 +4953,8 @@ def set_up_agent_pool_profiles(self, mc: ManagedCluster) -> ManagedCluster:
             max_count=max_count,
             enable_auto_scaling=enable_auto_scaling,
             enable_fips=self.context.get_enable_fips_image(),
+            kubelet_config=self.context.get_kubelet_config(),
+            linux_os_config=self.context.get_linux_os_config(),
         )
 
         # snapshot creation data

src/azure-cli/azure/cli/command_modules/acs/tests/latest/data/kubeletconfig.json

@@ -0,0 +1,13 @@
+{
+    "cpuManagerPolicy": "static",
+    "cpuCfsQuota": true,
+    "cpuCfsQuotaPeriod": "200ms",
+    "imageGcHighThreshold": 90,
+    "imageGcLowThreshold": 70,
+    "topologyManagerPolicy": "best-effort",
+    "allowedUnsafeSysctls": [
+        "kernel.msg*",
+        "net.*"
+    ],
+    "failSwapOn": false
+}

src/azure-cli/azure/cli/command_modules/acs/tests/latest/data/linuxosconfig.json

@@ -0,0 +1,10 @@
+{
+    "transparentHugePageEnabled": "madvise",
+    "transparentHugePageDefrag": "defer+madvise",
+    "swapFileSizeMB": 1500,
+    "sysctls": {
+        "netCoreSomaxconn": 163849,
+        "netIpv4TcpTwReuse": true,
+        "netIpv4IpLocalPortRange": "32000 60000"
+    }
+}