[IoT Hub] Support for 2020-03-01 API and Network Isolation commands (#…

…13467)

* Add authenticationType, endpointUri, and entityPath parameters to routing endpoint creation

* Added AuthType and ContainerURI fileupload parameters on hub create

* Support for fileUpload authenticationType change on iot hub update command

* removed debug line

* Testing and formatting updates, added temporary template file for identity-based hub creation

* Moved identity-based hub tests to new test function to avoid hub confusion and speed up testing iteration

* Fix for Identity-Based file upload not utilizing connectionString

* Test recording updates

* WIP: Private-endpoint and private link resource implementation

* parameter fixes

* test recording updates

* whitespace / formatting fixes

* Added help commands for private-endpoint-connection and private-link-resources commands

* Added help example for identity-based routing endpoint

* Removed accidental change

* Linting and style updates, addressing PR feedback

* Rollback of accidental profile version update

* SDK and test updates

* Test template path/name fix and test update

* History update

* Test updates

* Address PR feedback

Moved private-endpoint-connection and private-link-resources into network module, updated tests, and addressed PR feedback

* Help update

* Address PR Feedback - add test recording processor to obscure access keys, updated tests

Co-authored-by: Ryan Kelly <[email protected]>

src/azure-cli-core/azure/cli/core/profiles/_shared.py

@@ -196,7 +196,7 @@ def default_api_version(self):
             'private_endpoint_connections': '2019-10-17-preview'
         }),
         ResourceType.MGMT_APPSERVICE: '2019-08-01',
-        ResourceType.MGMT_IOTHUB: '2019-07-01-preview',
+        ResourceType.MGMT_IOTHUB: '2020-03-01',
         ResourceType.MGMT_ARO: '2020-04-30'
     },
     '2019-03-01-hybrid': {

src/azure-cli/azure/cli/command_modules/iot/_help.py

@@ -685,6 +685,9 @@
         --endpoint-subscription-id {SubscriptionId} --connection-string {ConnectionString} \\
         --container-name {ContainerName} --batch-frequency 100 --chunk-size 100 \\
         --ff {iothub}-{partition}-{YYYY}-{MM}-{DD}-{HH}-{mm}
+  - name: Add a new identity-based EventHub endpoint named "EventHubIdentity"
+    text: >
+        az iot hub routing-endpoint create --resource-group MyResourceGroup --hub-name MyIotHub --endpoint-name EventHubIdentity --endpoint-type eventhub --endpoint-resource-group {ResourceGroup} --endpoint-subscription-id {SubscriptionId} --auth-type identityBased --endpoint-uri {EventHubEndpointUri} --entity-path {EntityPath}
 """
 
 helps['iot hub routing-endpoint delete'] = """

src/azure-cli/azure/cli/command_modules/iot/_params.py

@@ -20,6 +20,7 @@
                                                   RouteSourceType,
                                                   EncodingFormat,
                                                   RenewKeyType,
+                                                  AuthenticationType,
                                                   UserRole)
 from .custom import KeyType, SimpleAccessRights
 from ._validators import (validate_policy_permissions,
@@ -160,6 +161,13 @@ def load_arguments(self, _):  # pylint: disable=too-many-statements
         c.argument('fileupload_storage_connectionstring',
                    options_list=['--fileupload-storage-connectionstring', '--fcs'],
                    help='The connection string for the Azure Storage account to which files are uploaded.')
+        c.argument('fileupload_storage_authentication_type',
+                   options_list=['--fileupload-storage-auth-type', '--fsa'],
+                   help='The authentication type for the Azure Storage account to which files are uploaded.'
+                        'Possible values are keyBased and identityBased')
+        c.argument('fileupload_storage_container_uri',
+                   options_list=['--fileupload-storage-container-uri', '--fcu'],
+                   help='The container URI for the Azure Storage account to which files are uploaded.')
         c.argument('fileupload_storage_container_name',
                    options_list=['--fileupload-storage-container-name', '--fc'],
                    help='The name of the root container where you upload files. The container need not exist but'
@@ -207,6 +215,10 @@ def load_arguments(self, _):  # pylint: disable=too-many-statements
         c.argument('encoding', options_list=['--encoding'], arg_type=get_enum_type(EncodingFormat),
                    help='Encoding format for the container. The default is AVRO. '
                         'Note that this field is applicable only for blob container endpoints.')
+        c.argument('endpoint_uri', options_list=['--endpoint-uri'],
+                   help='The uri of the endpoint resource.')
+        c.argument('entity_path', options_list=['--entity-path'],
+                   help='The entity path of the endpoint resource.')
 
     with self.argument_context('iot hub routing-endpoint create') as c:
         c.argument('batch_frequency', options_list=['--batch-frequency', '-b'], type=int,
@@ -218,6 +230,8 @@ def load_arguments(self, _):  # pylint: disable=too-many-statements
                    help='File name format for the blob. The file name format must contain {iothub},'
                         ' {partition}, {YYYY}, {MM}, {DD}, {HH} and {mm} fields. All parameters are'
                         ' mandatory but can be reordered with or without delimiters.')
+        c.argument('authentication_type', options_list=['--auth-type'], arg_type=get_enum_type(AuthenticationType),
+                   help='Authentication type for the endpoint. The default is keyBased.')
 
     with self.argument_context('iot hub certificate') as c:
         c.argument('certificate_path', options_list=['--path', '-p'], type=file_type,

src/azure-cli/azure/cli/command_modules/iot/custom.py

@@ -45,7 +45,7 @@
 
 from azure.cli.command_modules.iot.mgmt_iot_hub_device.lib.iot_hub_device_client import IotHubDeviceClient
 from azure.cli.command_modules.iot.sas_token_auth import SasTokenAuthentication
-from azure.cli.command_modules.iot.shared import EndpointType, EncodingFormat, RenewKeyType
+from azure.cli.command_modules.iot.shared import EndpointType, EncodingFormat, RenewKeyType, AuthenticationType
 from ._constants import PNP_ENDPOINT
 from ._client_factory import resource_service_factory, get_pnp_client
 from ._utils import open_certificate, get_auth_header, generateKey
@@ -388,7 +388,9 @@ def iot_hub_create(cmd, client, hub_name, resource_group_name, location=None,
                    fileupload_notification_ttl=1,
                    fileupload_storage_connectionstring=None,
                    fileupload_storage_container_name=None,
-                   fileupload_sas_ttl=1):
+                   fileupload_sas_ttl=1,
+                   fileupload_storage_authentication_type=None,
+                   fileupload_storage_container_uri=None):
     from datetime import timedelta
     cli_ctx = cmd.cli_ctx
     if enable_fileupload_notifications:
@@ -398,6 +400,11 @@ def iot_hub_create(cmd, client, hub_name, resource_group_name, location=None,
         raise CLIError('Please mention storage container name.')
     if fileupload_storage_container_name and not fileupload_storage_connectionstring:
         raise CLIError('Please mention storage connection string.')
+    identity_based_file_upload = fileupload_storage_authentication_type and fileupload_storage_authentication_type.lower() == AuthenticationType.IdentityBased.value
+    if not identity_based_file_upload and not fileupload_storage_connectionstring and fileupload_storage_container_name:
+        raise CLIError('Key-based authentication requires a connection string.')
+    if identity_based_file_upload and not fileupload_storage_container_uri:
+        raise CLIError('Identity-based authentication requires a storage container uri (--fileupload-storage-container-uri, --fcu).')
     _check_name_availability(client.iot_hub_resource, hub_name)
     location = _ensure_location(cli_ctx, resource_group_name, location)
     sku = IotHubSkuInfo(name=sku, capacity=unit)
@@ -418,7 +425,9 @@ def iot_hub_create(cmd, client, hub_name, resource_group_name, location=None,
     storage_endpoint_dic['$default'] = StorageEndpointProperties(
         sas_ttl_as_iso8601=timedelta(hours=fileupload_sas_ttl),
         connection_string=fileupload_storage_connectionstring if fileupload_storage_connectionstring else '',
-        container_name=fileupload_storage_container_name if fileupload_storage_container_name else '')
+        container_name=fileupload_storage_container_name if fileupload_storage_container_name else '',
+        authentication_type=fileupload_storage_authentication_type if fileupload_storage_authentication_type else None,
+        container_uri=fileupload_storage_container_uri if fileupload_storage_container_uri else '')
 
     properties = IotHubProperties(event_hub_endpoints=event_hub_dic,
                                   messaging_endpoints=msg_endpoint_dic,
@@ -472,7 +481,9 @@ def update_iot_hub_custom(instance,
                           fileupload_notification_ttl=None,
                           fileupload_storage_connectionstring=None,
                           fileupload_storage_container_name=None,
-                          fileupload_sas_ttl=None):
+                          fileupload_sas_ttl=None,
+                          fileupload_storage_authentication_type=None,
+                          fileupload_storage_container_uri=None):
     from datetime import timedelta
     if sku is not None:
         instance.sku.name = sku
@@ -499,6 +510,15 @@ def update_iot_hub_custom(instance,
     if fileupload_notification_ttl is not None:
         ttl = timedelta(hours=fileupload_notification_ttl)
         instance.properties.messaging_endpoints['fileNotifications'].ttl_as_iso8601 = ttl
+
+    identity_based_file_upload = fileupload_storage_authentication_type and fileupload_storage_authentication_type.lower() == AuthenticationType.IdentityBased.value
+    if identity_based_file_upload:
+        instance.properties.storage_endpoints['$default'].authentication_type = AuthenticationType.IdentityBased
+        instance.properties.storage_endpoints['$default'].container_uri = fileupload_storage_container_uri
+    elif fileupload_storage_authentication_type is not None:
+        instance.properties.storage_endpoints['$default'].authentication_type = None
+        instance.properties.storage_endpoints['$default'].container_uri = None
+    # TODO - remove connection string and set containerURI once fileUpload SAS URL is enabled
     if fileupload_storage_connectionstring is not None and fileupload_storage_container_name is not None:
         instance.properties.storage_endpoints['$default'].connection_string = fileupload_storage_connectionstring
         instance.properties.storage_endpoints['$default'].container_name = fileupload_storage_container_name
@@ -672,11 +692,26 @@ def iot_hub_get_stats(client, hub_name, resource_group_name=None):
     return client.iot_hub_resource.get_stats(resource_group_name, hub_name)
 
 
+def validate_authentication_type_input(endpoint_type, connection_string=None, authentication_type=None, endpoint_uri=None, entity_path=None):
+    is_keyBased = (AuthenticationType.KeyBased.value == authentication_type.lower()) or (authentication_type is None)
+    has_connection_string = (connection_string is not None)
+    if is_keyBased and not has_connection_string:
+        raise CLIError("Please provide a connection string '--connection-string/-c'")
+
+    has_endpoint_uri = (endpoint_uri is not None)
+    has_endpoint_uri_and_path = (has_endpoint_uri) and (entity_path is not None)
+    if EndpointType.AzureStorageContainer.value == endpoint_type.lower() and not has_endpoint_uri:
+        raise CLIError("Please provide an endpoint uri '--endpoint-uri'")
+    if not has_endpoint_uri_and_path:
+        raise CLIError("Please provide an endpoint uri '--endpoint-uri' and entity path '--entity-path'")
+
+
 def iot_hub_routing_endpoint_create(cmd, client, hub_name, endpoint_name, endpoint_type,
                                     endpoint_resource_group, endpoint_subscription_id,
-                                    connection_string, container_name=None, encoding=None,
+                                    connection_string=None, container_name=None, encoding=None,
                                     resource_group_name=None, batch_frequency=300, chunk_size_window=300,
-                                    file_name_format='{iothub}/{partition}/{YYYY}/{MM}/{DD}/{HH}/{mm}'):
+                                    file_name_format='{iothub}/{partition}/{YYYY}/{MM}/{DD}/{HH}/{mm}',
+                                    authentication_type=None, endpoint_uri=None, entity_path=None):
     resource_group_name = _ensure_resource_group_name(client, resource_group_name, hub_name)
     hub = iot_hub_get(cmd, client, hub_name, resource_group_name)
     if EndpointType.EventHub.value == endpoint_type.lower():
@@ -685,7 +720,10 @@ def iot_hub_routing_endpoint_create(cmd, client, hub_name, endpoint_name, endpoi
                 connection_string=connection_string,
                 name=endpoint_name,
                 subscription_id=endpoint_subscription_id,
-                resource_group=endpoint_resource_group
+                resource_group=endpoint_resource_group,
+                authentication_type=authentication_type,
+                endpoint_uri=endpoint_uri,
+                entity_path=entity_path
             )
         )
     elif EndpointType.ServiceBusQueue.value == endpoint_type.lower():
@@ -694,7 +732,10 @@ def iot_hub_routing_endpoint_create(cmd, client, hub_name, endpoint_name, endpoi
                 connection_string=connection_string,
                 name=endpoint_name,
                 subscription_id=endpoint_subscription_id,
-                resource_group=endpoint_resource_group
+                resource_group=endpoint_resource_group,
+                authentication_type=authentication_type,
+                endpoint_uri=endpoint_uri,
+                entity_path=entity_path
             )
         )
     elif EndpointType.ServiceBusTopic.value == endpoint_type.lower():
@@ -703,7 +744,10 @@ def iot_hub_routing_endpoint_create(cmd, client, hub_name, endpoint_name, endpoi
                 connection_string=connection_string,
                 name=endpoint_name,
                 subscription_id=endpoint_subscription_id,
-                resource_group=endpoint_resource_group
+                resource_group=endpoint_resource_group,
+                authentication_type=authentication_type,
+                endpoint_uri=endpoint_uri,
+                entity_path=entity_path
             )
         )
     elif EndpointType.AzureStorageContainer.value == endpoint_type.lower():
@@ -719,7 +763,9 @@ def iot_hub_routing_endpoint_create(cmd, client, hub_name, endpoint_name, endpoi
                 encoding=encoding.lower() if encoding else EncodingFormat.AVRO.value,
                 file_name_format=file_name_format,
                 batch_frequency_in_seconds=batch_frequency,
-                max_chunk_size_in_bytes=(chunk_size_window * 1048576)
+                max_chunk_size_in_bytes=(chunk_size_window * 1048576),
+                authentication_type=authentication_type,
+                endpoint_uri=endpoint_uri
             )
         )
     return client.iot_hub_resource.create_or_update(resource_group_name, hub_name, hub, {'IF-MATCH': hub.etag})
@@ -891,8 +937,11 @@ def iot_message_enrichment_list(cmd, client, hub_name, resource_group_name=None)
 
 
 def iot_hub_devicestream_show(cmd, client, hub_name, resource_group_name=None):
+    from azure.cli.core.commands.client_factory import get_mgmt_service_client, ResourceType
     resource_group_name = _ensure_resource_group_name(client, resource_group_name, hub_name)
-    hub = iot_hub_get(cmd, client, hub_name, resource_group_name)
+    # DeviceStreams property is still in preview, so until GA we need to use an older API version (2019-07-01-preview)
+    client = get_mgmt_service_client(cmd.cli_ctx, ResourceType.MGMT_IOTHUB, api_version='2019-07-01-preview')
+    hub = client.iot_hub_resource.get(resource_group_name, hub_name)
     return hub.properties.device_streams
 
 

src/azure-cli/azure/cli/command_modules/iot/shared.py

@@ -62,3 +62,12 @@ class RenewKeyType(Enum):
     Primary = 'primary'
     Secondary = 'secondary'
     Swap = 'swap'
+
+
+# pylint: disable=too-few-public-methods
+class AuthenticationType(Enum):
+    """
+    Type of the Authentication for the routing endpoint.
+    """
+    KeyBased = 'keybased'
+    IdentityBased = 'identitybased'

src/azure-cli/azure/cli/command_modules/iot/tests/latest/recording_processors.py

@@ -0,0 +1,46 @@
+# --------------------------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for license information.
+# --------------------------------------------------------------------------------------------
+
+from azure_devtools.scenario_tests import RecordingProcessor
+from azure_devtools.scenario_tests.utilities import is_text_payload
+
+MOCK_KEY = 'mock_key'
+
+
+class KeyReplacer(RecordingProcessor):
+
+    def process_request(self, request):
+        if is_text_payload(request) and isinstance(request.body, bytes):
+            request.body = self._replace_byte_keys(request.body)
+        elif is_text_payload(request) and isinstance(request.body, str):
+            request.body = self._replace_string_keys(request.body)
+        return request
+
+    def process_response(self, response):
+        if is_text_payload(response) and response['body']['string']:
+            response['body']['string'] = self._replace_string_keys(response['body']['string'])
+        return response
+
+    # pylint: disable=no-self-use
+    def _replace_string_keys(self, val):
+        import re
+        if 'primaryKey' in val:
+            val = re.sub(r'"primaryKey":( ?)"([^"]+)"', r'"primaryKey":"{}"'
+                         .format(MOCK_KEY), val, flags=re.IGNORECASE)
+        if 'secondaryKey' in val:
+            val = re.sub(r'"secondaryKey":( ?)"([^"]+)"', r'"secondaryKey":"{}"'
+                         .format(MOCK_KEY), val, flags=re.IGNORECASE)
+        return val
+
+    # pylint: disable=no-self-use
+    def _replace_byte_keys(self, val):
+        import re
+        if b'primaryKey' in val:
+            val = re.sub(b'"primaryKey":( ?)"([^"]+)"', '"primaryKey":"{}"'
+                         .format(MOCK_KEY).encode(), val, flags=re.IGNORECASE)
+        if b'secondaryKey' in val:
+            val = re.sub(b'"secondaryKey":( ?)"([^"]+)"', '"secondaryKey":"{}"'
+                         .format(MOCK_KEY).encode(), val, flags=re.IGNORECASE)
+        return val