Azure · needuv · Feb 15, 2023 · Feb 15, 2023 · Feb 15, 2023 · Feb 15, 2023
@@ -93,7 +93,7 @@ def upload(
             file_size, _ = get_directory_size(source, ignore_file=ignore_file)
             file_size_in_mb = file_size / 10**6
             cloud = _get_cloud_details()
-            cloud_endpoint = cloud['storage_endpoint']  # make sure proper cloud endpoint is used
+            cloud_endpoint = cloud["storage_endpoint"]  # make sure proper cloud endpoint is used
             full_storage_url = f"https://{self.account_name}.blob.{cloud_endpoint}/{self.container}/{dest}"
             if file_size_in_mb > 100:
                 module_logger.warning(FILE_SIZE_WARNING.format(source=source, destination=full_storage_url))
@@ -231,9 +231,9 @@ def download(
                 # check if total size of download has exceeded 100 MB
                 # make sure proper cloud endpoint is used
                 cloud = _get_cloud_details()
-                cloud_endpoint = cloud['storage_endpoint']
+                cloud_endpoint = cloud["storage_endpoint"]
                 full_storage_url = f"https://{self.account_name}.blob.{cloud_endpoint}/{self.container}/{starts_with}"
-                download_size_in_mb += (blob_content.size / 10**6)
+                download_size_in_mb += blob_content.size / 10**6
                 if download_size_in_mb > 100:
                     module_logger.warning(FILE_SIZE_WARNING.format(source=full_storage_url, destination=destination))
 

@@ -85,7 +85,7 @@ def upload(
             file_size_in_mb = file_size / 10**6
 
             cloud = _get_cloud_details()
-            cloud_endpoint = cloud['storage_endpoint']  # make sure proper cloud endpoint is used
+            cloud_endpoint = cloud["storage_endpoint"]  # make sure proper cloud endpoint is used
             full_storage_url = f"https://{self.account_name}.dfs.{cloud_endpoint}/{self.file_system}/{dest}"
             if file_size_in_mb > 100:
                 module_logger.warning(FILE_SIZE_WARNING.format(source=source, destination=full_storage_url))
@@ -185,9 +185,9 @@ def download(self, starts_with: str, destination: str = Path.home()) -> None:
 
                 # check if total size of download has exceeded 100 MB
                 cloud = _get_cloud_details()
-                cloud_endpoint = cloud['storage_endpoint']  # make sure proper cloud endpoint is used
+                cloud_endpoint = cloud["storage_endpoint"]  # make sure proper cloud endpoint is used
                 full_storage_url = f"https://{self.account_name}.dfs.{cloud_endpoint}/{self.file_system}/{starts_with}"
-                download_size_in_mb += (file_client.get_file_properties().size / 10**6)
+                download_size_in_mb += file_client.get_file_properties().size / 10**6
                 if download_size_in_mb > 100:
                     module_logger.warning(FILE_SIZE_WARNING.format(source=full_storage_url, destination=destination))
 

@@ -15,7 +15,6 @@
 from azure.mgmt.core import ARMPipelineClient
 
 
-
 module_logger = logging.getLogger(__name__)
 
 
@@ -63,10 +62,11 @@ class EndpointURLS:  # pylint: disable=too-few-public-methods,no-init
 
 _requests_pipeline = None
 
+
 def _get_cloud(cloud: str):
     if cloud in _environments:
         return _environments[cloud]
-    arm_url = os.environ.get(ArmConstants.METADATA_URL_ENV_NAME,ArmConstants.DEFAULT_URL)
+    arm_url = os.environ.get(ArmConstants.METADATA_URL_ENV_NAME, ArmConstants.DEFAULT_URL)
     arm_clouds = _get_clouds_by_metadata_url(arm_url)
     try:
         new_cloud = arm_clouds[cloud]
@@ -75,6 +75,7 @@ def _get_cloud(cloud: str):
     except KeyError:
         raise Exception('Unknown cloud environment "{0}".'.format(cloud))
 
+
 def _get_default_cloud_name():
     """Return AzureCloud as the default cloud."""
     return os.getenv(AZUREML_CLOUD_ENV_NAME, AzureEnvironments.ENV_DEFAULT)
@@ -209,53 +210,56 @@ def _resource_to_scopes(resource):
     scope = resource + "/.default"
     return [scope]
 
+
 def _get_registry_discovery_url(cloud, cloud_suffix=""):
     """Get or generate the registry discovery url
 
-        :param cloud: configuration of the cloud to get the registry_discovery_url from
-        :param cloud_suffix: the suffix to use for the cloud, in the case that the registry_discovery_url
-            must be generated
-        :return: string of discovery url
+    :param cloud: configuration of the cloud to get the registry_discovery_url from
+    :param cloud_suffix: the suffix to use for the cloud, in the case that the registry_discovery_url
+        must be generated
+    :return: string of discovery url
     """
     cloud_name = cloud["name"]
     if cloud_name in _environments:
         return _environments[cloud_name].registry_url
 
     registry_discovery_region = os.environ.get(
-        ArmConstants.REGISTRY_DISCOVERY_REGION_ENV_NAME,
-        ArmConstants.REGISTRY_DISCOVERY_DEFAULT_REGION
+        ArmConstants.REGISTRY_DISCOVERY_REGION_ENV_NAME, ArmConstants.REGISTRY_DISCOVERY_DEFAULT_REGION
     )
     registry_discovery_region_default = "https://{}{}.api.azureml.{}/".format(
-        cloud_name.lower(),
-        registry_discovery_region,
-        cloud_suffix
+        cloud_name.lower(), registry_discovery_region, cloud_suffix
     )
     return os.environ.get(ArmConstants.REGISTRY_ENV_URL, registry_discovery_region_default)
 
+
 def _get_clouds_by_metadata_url(metadata_url):
     """Get all the clouds by the specified metadata url
 
-        :return: list of the clouds
+    :return: list of the clouds
     """
     try:
-        module_logger.debug('Start : Loading cloud metadata from the url specified by %s', metadata_url)
+        module_logger.debug("Start : Loading cloud metadata from the url specified by %s", metadata_url)
         client = ARMPipelineClient(base_url=metadata_url, policies=[])
         HttpRequest("GET", metadata_url)
         with client.send_request(HttpRequest("GET", metadata_url)) as meta_response:
             arm_cloud_dict = meta_response.json()
             cli_cloud_dict = _convert_arm_to_cli(arm_cloud_dict)
-            module_logger.debug('Finish : Loading cloud metadata from the url specified by %s', metadata_url)
+            module_logger.debug("Finish : Loading cloud metadata from the url specified by %s", metadata_url)
             return cli_cloud_dict
     except Exception as ex:  # pylint: disable=broad-except
-        module_logger.warning("Error: Azure ML was unable to load cloud metadata from the url specified by %s. %s. "
-                        "This may be due to a misconfiguration of networking controls. Azure Machine Learning Python "
-                        "SDK requires outbound access to Azure Resource Manager. Please contact your networking team "
-                        "to configure outbound access to Azure Resource Manager on both Network Security Group and "
-                        "Firewall. For more details on required configurations, see "
-                        "https://docs.microsoft.com/azure/machine-learning/how-to-access-azureml-behind-firewall.",
-                        metadata_url, ex)
+        module_logger.warning(
+            "Error: Azure ML was unable to load cloud metadata from the url specified by %s. %s. "
+            "This may be due to a misconfiguration of networking controls. Azure Machine Learning Python "
+            "SDK requires outbound access to Azure Resource Manager. Please contact your networking team "
+            "to configure outbound access to Azure Resource Manager on both Network Security Group and "
+            "Firewall. For more details on required configurations, see "
+            "https://docs.microsoft.com/azure/machine-learning/how-to-access-azureml-behind-firewall.",
+            metadata_url,
+            ex,
+        )
         return {}
 
+
 def _convert_arm_to_cli(arm_cloud_metadata):
     cli_cloud_metadata_dict = {}
     if isinstance(arm_cloud_metadata, dict):
@@ -265,15 +269,15 @@ def _convert_arm_to_cli(arm_cloud_metadata):
         try:
             cloud_name = cloud["name"]
             portal_endpoint = cloud["portal"]
-            cloud_suffix = ".".join(portal_endpoint.split('.')[2:]).replace("/", "")
+            cloud_suffix = ".".join(portal_endpoint.split(".")[2:]).replace("/", "")
             registry_discovery_url = _get_registry_discovery_url(cloud, cloud_suffix)
             cli_cloud_metadata_dict[cloud_name] = {
                 EndpointURLS.AZURE_PORTAL_ENDPOINT: cloud["portal"],
                 EndpointURLS.RESOURCE_MANAGER_ENDPOINT: cloud["resourceManager"],
                 EndpointURLS.ACTIVE_DIRECTORY_ENDPOINT: cloud["authentication"]["loginEndpoint"],
                 EndpointURLS.AML_RESOURCE_ID: "https://ml.azure.{}".format(cloud_suffix),
                 EndpointURLS.STORAGE_ENDPOINT: cloud["suffixes"]["storage"],
-                EndpointURLS.REGISTRY_DISCOVERY_ENDPOINT: registry_discovery_url
+                EndpointURLS.REGISTRY_DISCOVERY_ENDPOINT: registry_discovery_url,
             }
         except KeyError as ex:
             module_logger.warning("Property on cloud not found in arm cloud metadata: %s", ex)

@@ -119,6 +119,7 @@ def _serialize(self, obj, *, many: bool = False):
     def add_param_overrides(self, data, **kwargs):
         source_path = self.context.pop(SOURCE_PATH_CONTEXT_KEY, None)
         if isinstance(data, dict) and source_path and os.path.isfile(source_path):
+
             def should_node_overwritten(_root, _parts):
                 parts = _parts.copy()
                 parts.pop()
@@ -135,9 +136,7 @@ def should_node_overwritten(_root, _parts):
                     ("inputs.*.enum", should_node_overwritten),
                 ]:
                     for dot_key in get_valid_dot_keys_with_wildcard(
-                        origin_data,
-                        dot_key_wildcard,
-                        validate_func=condition_func
+                        origin_data, dot_key_wildcard, validate_func=condition_func
                     ):
                         pydash.set_(data, dot_key, pydash.get(origin_data, dot_key))
         return super().add_param_overrides(data, **kwargs)

@@ -18,7 +18,7 @@
     "float",
     "Float",
     "double",
-    "Double"
+    "Double",
 ]
 
 

@@ -4,6 +4,4 @@
 
 from ._yaml_utils import yaml_safe_load_with_base_resolver
 
-__all__ = [
-    "yaml_safe_load_with_base_resolver"
-]
+__all__ = ["yaml_safe_load_with_base_resolver"]
@@ -11,6 +11,7 @@
 
 def _map_internal_output_type(_meta):
     """Map component output type to valid pipeline output type."""
+
     def _map_primitive_type(_type):
         """Convert double and float to number type."""
         _type = _type.lower()

@@ -18,6 +18,7 @@ class _SafeLoaderWithBaseLoader(strictyaml.ruamel.SafeLoader):
     from the inheritance list. Instead, we overwrite add_version_implicit_resolver method to make
     _SafeLoaderWithBaseLoader._version_implicit_resolver empty. Then the resolver will act like a BaseResolver.
     """
+
     def fetch_comment(self, comment):
         pass
 

@@ -95,11 +95,7 @@ def _copy(src: Path, dst: Path, *, ignore_file=None) -> None:
             # for same folder, the expected behavior is merging
             # ignore will be also applied during this process
             for name in src.glob("*"):
-                _AdditionalIncludes._copy(
-                    name,
-                    dst / name.name,
-                    ignore_file=ignore_file.merge(name)
-                )
+                _AdditionalIncludes._copy(name, dst / name.name, ignore_file=ignore_file.merge(name))
 
     @staticmethod
     def _is_folder_to_compress(path: Path) -> bool:
@@ -182,9 +178,7 @@ def resolve(self) -> None:
                 skip_ignore_file=True,
             )
             self._copy(
-                Path(self._code_path),
-                tmp_folder_path / Path(self._code_path).name,
-                ignore_file=root_ignore_file
+                Path(self._code_path), tmp_folder_path / Path(self._code_path).name, ignore_file=root_ignore_file
             )
         else:
             # current implementation of ignore file is based on absolute path, so it cannot be shared

@@ -23,7 +23,7 @@ class DataCollectorSchema(metaclass=PatchedSchemaMeta):
     collections = fields.Mapping(fields.Str, NestedField(DeploymentCollectionSchema))
     rolling_rate = StringTransformedEnum(
         required=False,
-        allowed_values=[ RollingRate.MINUTE, RollingRate.DAY, RollingRate.HOUR],
+        allowed_values=[RollingRate.MINUTE, RollingRate.DAY, RollingRate.HOUR],
     )
     destination = NestedField(DestinationSchema)
     sampling_rate = fields.Float()

@@ -15,10 +15,7 @@
 
 
 class DeploymentCollectionSchema(metaclass=PatchedSchemaMeta):
-    enabled = StringTransformedEnum(
-        required= True,
-        allowed_values=[Boolean.TRUE, Boolean.FALSE]
-    )
+    enabled = StringTransformedEnum(required=True, allowed_values=[Boolean.TRUE, Boolean.FALSE])
     data = NestedField(DataAssetSchema)
 
     # pylint: disable=unused-argument,no-self-use

@@ -14,10 +14,7 @@
 
 
 class PayloadResponseSchema(metaclass=PatchedSchemaMeta):
-    enabled = StringTransformedEnum(
-        required= True,
-        allowed_values=[Boolean.TRUE, Boolean.FALSE]
-    )
+    enabled = StringTransformedEnum(required=True, allowed_values=[Boolean.TRUE, Boolean.FALSE])
 
     # pylint: disable=unused-argument,no-self-use
     @post_load

@@ -18,10 +18,7 @@
 
 class EndpointSchema(PathAwareSchema):
     id = fields.Str()
-    name = fields.Str(
-            required=True,
-            validate=validate.Regexp(EndpointConfigurations.NAME_REGEX_PATTERN)
-        )
+    name = fields.Str(required=True, validate=validate.Regexp(EndpointConfigurations.NAME_REGEX_PATTERN))
     description = fields.Str(metadata={"description": "Description of the inference endpoint."})
     tags = fields.Dict()
     provisioning_state = fields.Str(metadata={"description": "Provisioning state for the endpoint."})

@@ -75,9 +75,7 @@ class _BaseEnvironmentSchema(AssetSchema):
     )
     build = NestedField(
         BuildContextSchema,
-        metadata={
-            "description": "Docker build context to create the environment. Mutually exclusive with image"
-        },
+        metadata={"description": "Docker build context to create the environment. Mutually exclusive with image"},
     )
     image = fields.Str()
     conda_file = UnionField([fields.Raw(), fields.Str()])
@@ -101,9 +99,7 @@ def pre_load(self, data, **kwargs):
         # validates that "channels" and "dependencies" are not included in the data creation.
         # These properties should only be on environment conda files not in the environment creation file
         if "channels" in data or "dependencies" in data:
-            environmentMessage = CREATE_ENVIRONMENT_ERROR_MESSAGE.format(
-                YAMLRefDocLinks.ENVIRONMENT
-            )
+            environmentMessage = CREATE_ENVIRONMENT_ERROR_MESSAGE.format(YAMLRefDocLinks.ENVIRONMENT)
             raise ValidationError(environmentMessage)
         return data
 

@@ -9,11 +9,18 @@
 from .import_component import AnonymousImportComponentSchema, ImportComponentFileRefField, ImportComponentSchema
 from .parallel_component import AnonymousParallelComponentSchema, ParallelComponentFileRefField, ParallelComponentSchema
 from .spark_component import AnonymousSparkComponentSchema, SparkComponentFileRefField, SparkComponentSchema
-from .data_transfer_component import AnonymousDataTransferCopyComponentSchema, DataTransferCopyComponentFileRefField, \
-    DataTransferCopyComponentSchema, AnonymousDataTransferImportComponentSchema, \
-    DataTransferImportComponentFileRefField, DataTransferImportComponentSchema, \
-    AnonymousDataTransferExportComponentSchema, DataTransferExportComponentFileRefField, \
-    DataTransferExportComponentSchema
+from .data_transfer_component import (
+    AnonymousDataTransferCopyComponentSchema,
+    DataTransferCopyComponentFileRefField,
+    DataTransferCopyComponentSchema,
+    AnonymousDataTransferImportComponentSchema,
+    DataTransferImportComponentFileRefField,
+    DataTransferImportComponentSchema,
+    AnonymousDataTransferExportComponentSchema,
+    DataTransferExportComponentFileRefField,
+    DataTransferExportComponentSchema,
+)
+
 __all__ = [
     "ComponentSchema",
     "CommandComponentSchema",
-Original file line number
+Diff line change
@@ Expand Up @@
         from the inheritance list. Instead, we overwrite add_version_implicit_resolver method to make
         _SafeLoaderWithBaseLoader._version_implicit_resolver empty. Then the resolver will act like a BaseResolver.
         """
         def fetch_comment(self, comment):
             pass
@@ Expand Down @@