Merge from base

.azure-pipelines/templates/azdev_setup.yml

@@ -5,16 +5,15 @@ parameters:
   - name: CLIExtensionRepoPath
     type: string
     default: ''
-
 steps:
   - bash: |
       set -ev
 
       python -m venv env
       chmod +x env/bin/activate
       . env/bin/activate
-
-      pip install azdev==0.1.29
+      python -m pip install -U pip
+      pip install azdev==0.1.30
       azdev --version
 
       if [ -z "$CLI_EXT_REPO_PATH" ]; then

.github/CODEOWNERS

@@ -12,7 +12,7 @@
 /scripts/live_test @qwordy
 /src/azure-cli-testsdk/ @jsntcy @jiasli @kairu-ms @qwordy
 /src/azure-cli-core/ @jiasli @Juliehzl @fengzhou-msft @evelyn-ys @jsntcy @houk-ms
-/src/azure-cli-core/azure/cli/core/extension/ @msyyc
+/src/azure-cli-core/azure/cli/core/extension/ @msyyc @fengzhou-msft
 /src/azure-cli/azure/cli/command_modules/acr/ @djyou @fengzhou-msft @yungezz
 /src/azure-cli/azure/cli/command_modules/acs/ @rjtsdl @fengzhou-msft
 /src/azure-cli/azure/cli/command_modules/advisor/ @Prasanna-Padmanabhan
@@ -33,6 +33,7 @@
 /src/azure-cli/azure/cli/command_modules/eventhubs/ @v-ajnava
 /src/azure-cli/azure/cli/command_modules/eventgrid/ @Juliehzl @evelyn-ys @VidyaKukke
 /src/azure-cli/azure/cli/command_modules/extension/ @fengzhou-msft @msyyc
+/src/azure-cli/azure/cli/command_modules/feedback/ @fengzhou-msft @jiasli
 /src/azure-cli/azure/cli/command_modules/hdinsight/ @aim-for-better @Juliehzl @kairu-ms
 /src/azure-cli/azure/cli/command_modules/iot/ @digimaun
 /src/azure-cli/azure/cli/command_modules/keyvault/ @fengzhou-msft @yungezz @houk-ms

build_scripts/windows/scripts/build.cmd

@@ -161,7 +161,7 @@ for /d /r %BUILDING_DIR%\Lib\site-packages\azure\mgmt %%d in (aio) do (
 :: Remove dist-info
 echo remove dist-info
 pushd %BUILDING_DIR%\Lib\site-packages
-for /d %%d in ("*.dist-info") do (
+for /d %%d in ("azure*.dist-info") do (
     if exist %%d rmdir /s /q "%%d"
 )
 popd

linter_exclusions.yml

@@ -1621,6 +1621,9 @@ eventgrid event-subscription create:
     preferred_batch_size_in_kilobytes:
       rule_exclusions:
       - option_length_too_long
+    delivery_attribute_mapping:
+      rule_exclusions:
+      - option_length_too_long
 eventgrid event-subscription show:
   parameters:
     include_full_endpoint_url:
@@ -1640,6 +1643,9 @@ eventgrid event-subscription update:
     endpoint_type:
       rule_exclusions:
       - no_parameter_defaults_for_update_commands
+    delivery_attribute_mapping:
+      rule_exclusions:
+      - option_length_too_long
 eventgrid partner namespace create:
   parameters:
     partner_registration_id:
@@ -1650,12 +1656,13 @@ eventgrid partner namespace event-channel create:
     activation_expiration_date:
       rule_exclusions:
       - option_length_too_long
-    desination_topic_name:
+    destination_topic_name:
       rule_exclusions:
       - option_length_too_long
     destination_resource_group_name:
       rule_exclusions:
       - option_length_too_long
+      - parameter_should_not_end_in_resource_group
     destination_subscription_id:
       rule_exclusions:
       - option_length_too_long
@@ -1721,6 +1728,9 @@ eventgrid partner topic event-subscription create:
     preferred_batch_size_in_kilobytes:
       rule_exclusions:
       - option_length_too_long
+    delivery_attribute_mapping:
+      rule_exclusions:
+      - option_length_too_long
 eventgrid partner topic event-subscription show:
   parameters:
     include_full_endpoint_url:
@@ -1731,6 +1741,9 @@ eventgrid partner topic event-subscription update:
     endpoint_type:
       rule_exclusions:
       - no_parameter_defaults_for_update_commands
+    delivery_attribute_mapping:
+      rule_exclusions:
+      - option_length_too_long
 eventgrid system-topic event-subscription create:
   parameters:
     azure_active_directory_application_id_or_uri:
@@ -1751,6 +1764,9 @@ eventgrid system-topic event-subscription create:
     preferred_batch_size_in_kilobytes:
       rule_exclusions:
       - option_length_too_long
+    delivery_attribute_mapping:
+      rule_exclusions:
+      - option_length_too_long
 eventgrid system-topic event-subscription show:
   parameters:
     include_full_endpoint_url:
@@ -1761,6 +1777,9 @@ eventgrid system-topic event-subscription update:
     endpoint_type:
       rule_exclusions:
       - no_parameter_defaults_for_update_commands
+    delivery_attribute_mapping:
+      rule_exclusions:
+      - option_length_too_long
 eventgrid topic create:
   parameters:
     input_mapping_default_values:

scripts/ci/credscan/CredScanSuppressions.json

@@ -12,6 +12,7 @@
     {
       "file": [
         "src\\azure-cli\\azure\\cli\\command_modules\\appconfig\\tests\\latest\\recordings\\test_appconfig_to_appconfig_import_export.yaml",
+        "src\\azure-cli\\azure\\cli\\command_modules\\appconfig\\tests\\latest\\recordings\\test_azconfig_aad_auth.yaml",
         "src\\azure-cli\\azure\\cli\\command_modules\\appconfig\\tests\\latest\\recordings\\test_azconfig_credential.yaml",
         "src\\azure-cli\\azure\\cli\\command_modules\\appconfig\\tests\\latest\\recordings\\test_azconfig_feature.yaml",
         "src\\azure-cli\\azure\\cli\\command_modules\\appconfig\\tests\\latest\\recordings\\test_azconfig_feature_filter.yaml",

scripts/ci/test_extensions.sh

@@ -18,15 +18,18 @@ export AZURE_CLI_DIAGNOSTICS_TELEMETRY=
 output=$(az extension list-available --query [].name -otsv)
 exit_code=0
 
-for ext in $output; do
+# azure-cli-ml: https://github.com/Azure/azure-cli-extensions/issues/826
+# azure-iot: https://github.com/Azure/azure-cli/pull/17456
+block_list='azure-cli-ml azure-iot'
 
-    # TODO: Remove when ML extension is compatible with CLI 2.0.69 core
-    # https://github.com/Azure/azure-cli-extensions/issues/826
-    if [ $ext == 'azure-cli-ml' ]; then
+for ext in $output; do
+    echo
+    # Use regex to detect if $ext is in $block_list
+    if [[ $block_list =~ $ext ]]; then
+        echo "Skip extension: $ext"
         continue
     fi
 
-    echo
     echo "Verifying extension:" $ext
     az extension add -n $ext
     if [ $? != 0 ]

src/azure-cli-core/azure/cli/core/__init__.py

@@ -111,12 +111,12 @@ def refresh_request_id(self):
         import uuid
         self.data['headers']['x-ms-client-request-id'] = str(uuid.uuid1())
 
-    def get_progress_controller(self, det=False):
+    def get_progress_controller(self, det=False, spinner=None):
         import azure.cli.core.commands.progress as progress
         if not self.progress_controller:
             self.progress_controller = progress.ProgressHook()
 
-        self.progress_controller.init_progress(progress.get_progress_view(det))
+        self.progress_controller.init_progress(progress.get_progress_view(det, spinner=spinner))
         return self.progress_controller
 
     def get_cli_version(self):

src/azure-cli-core/azure/cli/core/azclierror.py

@@ -70,16 +70,16 @@ def print_error(self):
             if self.exception_trace:
                 logger.exception(self.exception_trace)
 
-            # print recommendations to action
-            if self.recommendations:
-                for recommendation in self.recommendations:
-                    print(recommendation, file=sys.stderr)
-
-            if self.aladdin_recommendations:
-                print('\nTRY THIS:', file=sys.stderr)
-                for recommendation, description in self.aladdin_recommendations:
-                    print_styled_text(recommendation, file=sys.stderr)
-                    print_styled_text(description, file=sys.stderr)
+        # print recommendations to action
+        if self.recommendations:
+            for recommendation in self.recommendations:
+                print(recommendation, file=sys.stderr)
+
+        if self.aladdin_recommendations:
+            print('\nTRY THIS:', file=sys.stderr)
+            for recommendation, description in self.aladdin_recommendations:
+                print_styled_text(recommendation, file=sys.stderr)
+                print_styled_text(description, file=sys.stderr)
 
     def send_telemetry(self):
         telemetry.set_error_type(self.__class__.__name__)

src/azure-cli-core/azure/cli/core/azlogging.py

@@ -39,7 +39,7 @@
 
 
 class AzCliLogging(CLILogging):
-    _COMMAND_METADATA_LOGGER = 'az_command_data_logger'
+    COMMAND_METADATA_LOGGER = 'az_command_data_logger'
 
     def __init__(self, name, cli_ctx=None):
         super(AzCliLogging, self).__init__(name, cli_ctx)
@@ -86,16 +86,13 @@ def _delete_old_logs(log_dir):
             self = cli_ctx.logging
             args = kwargs['args']
 
-            cmd_logger = logging.getLogger(AzCliLogging._COMMAND_METADATA_LOGGER)
+            metadata_logger = logging.getLogger(AzCliLogging.COMMAND_METADATA_LOGGER)
 
-            # overwrite CLILogging._is_file_log_enabled() from knack
+            # Overwrite the default of knack.log.CLILogging._is_file_log_enabled() to True
             self.file_log_enabled = cli_ctx.config.getboolean('logging', 'enable_log_file', fallback=True)
 
             if self.file_log_enabled:
-                self._init_command_logfile_handlers(cmd_logger, args)  # pylint: disable=protected-access
-                get_logger(__name__).debug("metadata file logging enabled - writing logs to '%s'.",
-                                           self.command_log_dir)
-
+                self._init_command_logfile_handlers(metadata_logger, args)  # pylint: disable=protected-access
                 _delete_old_logs(self.command_log_dir)
 
     def _init_command_logfile_handlers(self, command_metadata_logger, args):
@@ -113,12 +110,15 @@ def _init_command_logfile_handlers(self, command_metadata_logger, args):
         log_name = "{}.{}.{}.{}.{}".format(date_str, time_str, command_str, os.getpid(), "log")
 
         log_file_path = os.path.join(self.command_log_dir, log_name)
+        get_logger(__name__).debug("metadata file logging enabled - writing logs to '%s'.", log_file_path)
 
         logfile_handler = logging.FileHandler(log_file_path)
 
         lfmt = logging.Formatter(_CMD_LOG_LINE_PREFIX + ' %(process)d | %(asctime)s | %(levelname)s | %(name)s | %(message)s')  # pylint: disable=line-too-long
         logfile_handler.setFormatter(lfmt)
         logfile_handler.setLevel(logging.DEBUG)
+        # command_metadata_logger should always accept all levels regardless of the root logger level.
+        command_metadata_logger.setLevel(logging.DEBUG)
         command_metadata_logger.addHandler(logfile_handler)
 
         self.command_logger_handler = logfile_handler
@@ -201,15 +201,23 @@ def end_cmd_metadata_logging(self, exit_code):  # leave it non '-' prefix to not
 
 
 class CommandLoggerContext:
+    """A context manager during which error logs are also written to az_command_data_logger for
+    `az feedback` usage.
+    """
     def __init__(self, module_logger):
-        self.logger = module_logger
-        self.hdlr = logging.getLogger(AzCliLogging._COMMAND_METADATA_LOGGER)  # pylint: disable=protected-access
+        metadata_logger = logging.getLogger(AzCliLogging.COMMAND_METADATA_LOGGER)
+        original_error = module_logger.error
+
+        # Duplicate error logging to metadata logger
+        def error_duplicated(*args, **kwargs):
+            original_error(*args, **kwargs)
+            metadata_logger.error(*args, **kwargs)
+        from unittest import mock
+        self.mock_cm = mock.patch.object(module_logger, 'error', error_duplicated)
 
     def __enter__(self):
-        if self.hdlr:
-            self.logger.addHandler(self.hdlr)  # add command metadata handler
+        self.mock_cm.__enter__()
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
-        if self.hdlr:
-            self.logger.removeHandler(self.hdlr)
+        self.mock_cm.__exit__(exc_type, exc_val, exc_tb)

src/azure-cli-core/azure/cli/core/commands/__init__.py

@@ -27,7 +27,7 @@
     get_command_type_kwarg, read_file_content, get_arg_list, poller_classes)
 from azure.cli.core.local_context import LocalContextAction
 import azure.cli.core.telemetry as telemetry
-
+from azure.cli.core.commands.progress import IndeterminateProgressBar
 
 from knack.arguments import CLICommandArgument
 from knack.commands import CLICommand, CommandGroup, PREVIEW_EXPERIMENTAL_CONFLICT_ERROR
@@ -866,14 +866,16 @@ def _validate_arg_level(self, ns, **_):  # pylint: disable=no-self-use
 
 
 class LongRunningOperation:  # pylint: disable=too-few-public-methods
-    def __init__(self, cli_ctx, start_msg='', finish_msg='', poller_done_interval_ms=1000.0):
+    def __init__(self, cli_ctx, start_msg='', finish_msg='', poller_done_interval_ms=500.0,
+                 progress_bar=None):
 
         self.cli_ctx = cli_ctx
         self.start_msg = start_msg
         self.finish_msg = finish_msg
         self.poller_done_interval_ms = poller_done_interval_ms
         self.deploy_dict = {}
         self.last_progress_report = datetime.datetime.now()
+        self.progress_bar = progress_bar if progress_bar is not None else IndeterminateProgressBar(cli_ctx)
 
     def _delay(self):
         time.sleep(self.poller_done_interval_ms / 1000.0)
@@ -945,16 +947,17 @@ def __call__(self, poller):
         from msrest.exceptions import ClientException
 
         correlation_message = ''
-        self.cli_ctx.get_progress_controller().begin()
+        self.progress_bar.begin()
         correlation_id = None
 
         cli_logger = get_logger()  # get CLI logger which has the level set through command lines
         is_verbose = any(handler.level <= logs.INFO for handler in cli_logger.handlers)
+
         telemetry.poll_start()
         poll_flag = False
         while not poller.done():
             poll_flag = True
-            self.cli_ctx.get_progress_controller().add(message='Running')
+
             try:
                 # pylint: disable=protected-access
                 correlation_id = json.loads(
@@ -972,22 +975,24 @@ def __call__(self, poller):
                 except Exception as ex:  # pylint: disable=broad-except
                     logger.warning('%s during progress reporting: %s', getattr(type(ex), '__name__', type(ex)), ex)
             try:
+                self.progress_bar.update_progress()
                 self._delay()
             except KeyboardInterrupt:
-                self.cli_ctx.get_progress_controller().stop()
+                self.progress_bar.stop()
                 logger.error('Long-running operation wait cancelled.  %s', correlation_message)
                 raise
 
         try:
             result = poller.result()
         except ClientException as client_exception:
             from azure.cli.core.commands.arm import handle_long_running_operation_exception
-            self.cli_ctx.get_progress_controller().stop()
+            self.progress_bar.stop()
             handle_long_running_operation_exception(client_exception)
 
-        self.cli_ctx.get_progress_controller().end()
+        self.progress_bar.end()
         if poll_flag:
             telemetry.poll_end()
+
         return result
 
 

src/azure-cli-core/azure/cli/core/commands/client_factory.py

@@ -253,3 +253,6 @@ def _add_headers(request):
             pass
 
     return _add_headers
+
+
+prepare_client_kwargs_track2 = _prepare_client_kwargs_track2

src/azure-cli-core/azure/cli/core/commands/progress.py

@@ -103,10 +103,10 @@ def is_running(self):
 
 class IndeterminateStandardOut(ProgressViewBase):
     """ custom output for progress reporting """
-    def __init__(self, out=None):
+    def __init__(self, out=None, spinner=None):
         super(IndeterminateStandardOut, self).__init__(
             out if out else sys.stderr)
-        self.spinner = None
+        self.spinner = spinner
 
     def write(self, args):
         """
@@ -166,8 +166,33 @@ def flush(self):
         self.out.flush()
 
 
-def get_progress_view(determinant=False, outstream=sys.stderr):
+def get_progress_view(determinant=False, outstream=sys.stderr, spinner=None):
     """ gets your view """
     if determinant:
         return DeterminateStandardOut(out=outstream)
-    return IndeterminateStandardOut(out=outstream)
+    return IndeterminateStandardOut(out=outstream, spinner=spinner)
+
+
+class IndeterminateProgressBar:
+    """ Define progress bar update view """
+    def __init__(self, cli_ctx, message="Running"):
+        self.cli_ctx = cli_ctx
+        self.message = message
+        self.hook = self.cli_ctx.get_progress_controller(
+            det=False,
+            spinner=humanfriendly.Spinner(  # pylint: disable=no-member
+                label='Running',
+                stream=sys.stderr,
+                hide_cursor=False))
+
+    def begin(self):
+        self.hook.begin()
+
+    def stop(self):
+        self.hook.stop()
+
+    def update_progress(self):
+        self.hook.add(message=self.message)
+
+    def end(self):
+        self.hook.end()