Merge branch 'dev' into drop-python-2.7

# Conflicts: # src/azure-cli/requirements.py2.Darwin.txt # src/azure-cli/requirements.py2.Linux.txt # src/azure-cli/requirements.py2.windows.txt
Azure · fengzhou-msft · Feb 12, 2020 · Nov 22, 2019 · Feb 12, 2020 · Feb 12, 2020
commit f38cf2f912c7efebcdc0c8ccbbeef4059f8aa210
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -1,9 +1,15 @@
 
 
+**History Notes:**  
+(Fill in the following template if multiple notes are needed, otherwise PR title will be used for history note.)
+
+[Component Name 1] (BREAKING CHANGE:) (az command:) make some customer-facing change.  
+[Component Name 2] (BREAKING CHANGE:) (az command:) make some customer-facing change.
+
 ---
 
 This checklist is used to make sure that common guidelines for a pull request are followed.
 
-- [ ] The PR has modified HISTORY.rst describing any customer-facing, functional changes. Note that this does not include changes only to help content. (see [Modifying change log](https://github.com/Azure/azure-cli/tree/master/doc/authoring_command_modules#modify-change-log)).
+- [ ] The PR title and description has followed the guideline in [Submitting Pull Requests](https://github.com/Azure/azure-cli/tree/dev/doc/authoring_command_modules#submitting-pull-requests).
 
 - [ ] I adhere to the [Command Guidelines](https://github.com/Azure/azure-cli/blob/dev/doc/command_guidelines.md).
diff --git a/Dockerfile b/Dockerfile
@@ -34,7 +34,7 @@ LABEL maintainer="Microsoft" \
 # pip wheel - required for CLI packaging
 # jmespath-terminal - we include jpterm as a useful tool
 # libintl and icu-libs - required by azure devops artifact (az extension add --name azure-devops)
-RUN apk add --no-cache bash openssh ca-certificates jq curl openssl git zip \
+RUN apk add --no-cache bash openssh ca-certificates jq curl openssl perl git zip \
  && apk add --no-cache --virtual .build-deps gcc make openssl-dev libffi-dev musl-dev linux-headers \
  && apk add --no-cache libintl icu-libs libc6-compat \
  && update-ca-certificates

diff --git a/azure-cli.pyproj b/azure-cli.pyproj
@@ -654,10 +654,15 @@
     <Compile Include="command_modules\azure-cli-role\azure_bdist_wheel.py" />
     <Compile Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\commands.py" />
     <Compile Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\custom.py" />
-    <Compile Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\tests\latest\test_sf_commands.py" />
+    <Compile Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\operations\applications.py" />
+    <Compile Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\tests\latest\test_util.py" />
+    <Compile Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\tests\latest\test_sf_application.py" />
+    <Compile Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\tests\latest\test_sf_cluster.py" />
+    <Compile Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\_arm_deployment_utils.py" />
     <Compile Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\_client_factory.py" />
     <Compile Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\_help.py" />
     <Compile Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\_params.py" />
+    <Compile Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\_validators.py" />
     <Compile Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\__init__.py" />
     <Compile Include="command_modules\azure-cli-servicefabric\setup.py" />
     <Compile Include="command_modules\azure-cli-sql\azure\cli\command_modules\sql\commands.py" />
@@ -1065,8 +1070,10 @@
     <Folder Include="command_modules\azure-cli-servicefabric\azure\cli\" />
     <Folder Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\" />
     <Folder Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\" />
+    <Folder Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\operations\" />
     <Folder Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\template\" />
     <Folder Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\template\linux\" />
+    <Folder Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\template\service\" />
     <Folder Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\template\windows\" />
     <Folder Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\tests\" />
     <Folder Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\tests\latest\" />
@@ -1337,6 +1344,8 @@
     <Content Include="command_modules\azure-cli-role\HISTORY.rst" />
     <Content Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\template\linux\parameter.json" />
     <Content Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\template\linux\template.json" />
+    <Content Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\template\service\parameter.json" />
+    <Content Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\template\service\template.json" />
     <Content Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\template\windows\parameter.json" />
     <Content Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\template\windows\template.json" />
     <Content Include="command_modules\azure-cli-servicefabric\azure\cli\command_modules\servicefabric\tests\policy.json" />

diff --git a/azure-cli2017.pyproj b/azure-cli2017.pyproj
@@ -842,10 +842,15 @@
     <Compile Include="azure-cli\azure\cli\command_modules\servicebus\__init__.py" />
     <Compile Include="azure-cli\azure\cli\command_modules\servicefabric\commands.py" />
     <Compile Include="azure-cli\azure\cli\command_modules\servicefabric\custom.py" />
-    <Compile Include="azure-cli\azure\cli\command_modules\servicefabric\tests\latest\test_sf_commands.py" />
+    <Compile Include="azure-cli\azure\cli\command_modules\servicefabric\operations\applications.py" />
+    <Compile Include="azure-cli\azure\cli\command_modules\servicefabric\tests\latest\test_util.py" />
+    <Compile Include="azure-cli\azure\cli\command_modules\servicefabric\tests\latest\test_sf_application.py" />
+    <Compile Include="azure-cli\azure\cli\command_modules\servicefabric\tests\latest\test_sf_cluster.py" />
+    <Compile Include="azure-cli\azure\cli\command_modules\servicefabric\_arm_deployment_utils.py" />
     <Compile Include="azure-cli\azure\cli\command_modules\servicefabric\_client_factory.py" />
     <Compile Include="azure-cli\azure\cli\command_modules\servicefabric\_help.py" />
     <Compile Include="azure-cli\azure\cli\command_modules\servicefabric\_params.py" />
+    <Compile Include="azure-cli\azure\cli\command_modules\servicefabric\_validators.py" />
     <Compile Include="azure-cli\azure\cli\command_modules\servicefabric\__init__.py" />
     <Compile Include="azure-cli\azure\cli\command_modules\signalr\commands.py" />
     <Compile Include="azure-cli\azure\cli\command_modules\signalr\custom.py" />
@@ -1167,9 +1172,13 @@
     <Folder Include="azure-cli\azure\cli\command_modules\servicebus\tests\" />
     <Folder Include="azure-cli\azure\cli\command_modules\servicebus\tests\latest\" />
     <Folder Include="azure-cli\azure\cli\command_modules\servicefabric\" />
+    <Folder Include="azure-cli\azure\cli\command_modules\servicefabric\operations\" />
     <Folder Include="azure-cli\azure\cli\command_modules\servicefabric\template\" />
     <Folder Include="azure-cli\azure\cli\command_modules\servicefabric\template\linux\" />
+    <Folder Include="azure-cli\azure\cli\command_modules\servicefabric\template\service\" />
     <Folder Include="azure-cli\azure\cli\command_modules\servicefabric\template\windows\" />
+    <Folder Include="azure-cli\azure\cli\command_modules\servicefabric\tests\" />
+    <Folder Include="azure-cli\azure\cli\command_modules\servicefabric\tests\latest\" />
     <Folder Include="azure-cli\azure\cli\command_modules\signalr\" />
     <Folder Include="azure-cli\azure\cli\command_modules\signalr\tests\" />
     <Folder Include="azure-cli\azure\cli\command_modules\signalr\tests\latest\" />
@@ -1421,6 +1430,8 @@
     <Content Include="azure-cli\azure\cli\command_modules\role\tests\latest\cert.pem" />
     <Content Include="azure-cli\azure\cli\command_modules\servicefabric\template\linux\parameter.json" />
     <Content Include="azure-cli\azure\cli\command_modules\servicefabric\template\linux\template.json" />
+    <Content Include="azure-cli\azure\cli\command_modules\servicefabric\template\service\parameter.json" />
+    <Content Include="azure-cli\azure\cli\command_modules\servicefabric\template\service\template.json" />
     <Content Include="azure-cli\azure\cli\command_modules\servicefabric\template\windows\parameter.json" />
     <Content Include="azure-cli\azure\cli\command_modules\servicefabric\template\windows\template.json" />
     <Content Include="azure-cli\azure\cli\command_modules\servicefabric\tests\policy.json" />

diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -8,6 +8,20 @@ trigger:
     - '*'
 
 jobs:
+- job: CheckPullRequestTitle
+  displayName: "Check the Format of Pull Request Title"
+  condition: and(succeeded(), in(variables['System.PullRequest.TargetBranch'], 'dev', 'release'))
+
+  pool:
+    vmImage: 'ubuntu-16.04'
+  steps:
+    - bash: |
+        echo "Check Title of Pull Request: #$(System.PullRequest.PullRequestNumber)"
+        title=$(curl https://api.github.com/repos/Azure/azure-cli/pulls/$(System.PullRequest.PullRequestNumber) | jq -r '.title')
+        [[ $title =~ ^(\[.*\]|\{.*\}).* ]] && exit 0
+        echo "Pull Request title should follow https://aka.ms/submitAzPR"
+        exit 1
+
 - job: RejectPullRequestToMasterBranch
   displayName: "Reject Pull Request To Master Branch"
   condition: and(succeeded(), eq(variables['System.PullRequest.TargetBranch'], 'master'))
@@ -19,33 +33,25 @@ jobs:
         echo "Reject pull request directly to master branch"
         exit 1
 
-#- job: CredScan
-#  displayName: "Credential Scan"
-#
-#  pool:
-#    vmImage: "windows-2019"
-#  steps:
-#  - task: CredScan@2
-#    inputs:
-#      toolMajorVersion: 'V2'
-#      suppressionsFile: ./scripts/ci/credscan/CredScanSuppressions.json
-#
-#  - task: PostAnalysis@1
-#    inputs:
-#      AllTools: false
-#      APIScan: false
-#      BinSkim: false
-#      CodesignValidation: false
-#      CredScan: true
-#      FortifySCA: false
-#      FxCop: false
-#      ModernCop: false
-#      PoliCheck: false
-#      RoslynAnalyzers: false
-#      SDLNativeRules: false
-#      Semmle: false
-#      TSLint: false
-#      ToolLogsNotFoundAction: 'Standard'
+- job: CredScan
+  displayName: "Credential Scan"
+  pool:
+    vmImage: "windows-2019"
+  steps:
+    - task: ms-codeanalysis.vss-microsoft-security-code-analysis.build-task-credscan.CredScan@2
+      displayName: 'CredScan'
+      inputs:
+        toolVersion: 'Latest'
+        suppressionsFile: './scripts/ci/credscan/CredScanSuppressions.json'
+    - task: ms-codeanalysis.vss-microsoft-security-code-analysis.build-task-postanalysis.PostAnalysis@1
+      displayName: 'Post Analysis'
+      inputs:
+        AllTools: false
+        BinSkim: false
+        CredScan: true
+        RoslynAnalyzers: false
+        TSLint: false
+        ToolLogsNotFoundAction: 'Standard'
 
 - job: ExtractMetadata
   displayName: Extract Metadata

diff --git a/doc/authoring_command_modules/README.md b/doc/authoring_command_modules/README.md
@@ -199,28 +199,39 @@ azdev style --module <module> [--pylint] [--pep8]
 Submitting Pull Requests
 ------------------------
 
-### Modify Change Log
+### Format PR Title
 
-Modify the `HISTORY.rst` for any customer-facing changes. If a module has changed at all since a previous release so that a version bump is required, it is fine to add a generic entry that says "* Minor fixes.".
+History notes are auto-generated based on commit messages and descriptions starting from [S165](https://github.com/Azure/azure-cli/milestone/82). The commmit message for the squashed merge commit defaults to the PR title. Starting from 01/30/2020, we require all the PR titles to follow the below format:
+1. [**Mandatory**] Each commit message must start with `[Component Name]` or `{Component Name}`. 
+    * `Component Name` shall be replaced by the real ones such as `Storage`, `Compute`. It could be the name of a command module, but in title case with necessary spaces for better readability, such as `API Management`, `Managed Service`. Other possible component names include but are not limited to: `Packaging`, `Misc.`, `Aladdin`.
+    * `[]` means this change is customer-facing and the message will be put into `HISTORY.rst`. `{}` means this change is not customer-facing and the message will **NOT** be included in `HISTORY.rst`.
+    * If the component name is `Core`, the message will be written in `src/azure-cli-core/HISTORY.rst`. Otherwise, the message will be written in `src/azure-cli/HISTORY.rst`.
+2. [**Mandatory**] If it's a breaking change, the second part should be `BREAKING CHANGE` followed by a colon. In the case of hotfix, put `Hotfix` in this part. For other cases, this part should be empty.
+3. [**Recommendation**] If the change can be mapped into a command, then the next part could be the command name, followed by a colon.
+4. [**Recommendation**] Use the right verb with present-tense in original form to descibe what is done:
+    * **Add** for new features.
+    * **Change** for changes in existing functionality.
+    * **Deprecate** for once-stable features removed in upcoming releases.
+    * **Remove** for deprecated features removed in this release.
+    * **Fix** for any bug fixes.
 
-This will be the release notes for the next release.
+An example title of customer-facing change PR:
 
-e.g.:  
-```
-.. :changelog:
+>[Storage] BREAKING CHANGE: az storage remove: remove --auth-mode argument
 
-Release History
-===============
+An example title of non-customer-facing change PR:
 
-0.0.3
-+++++
-* This is my customer-facing change.
+>{Aladdin} Add help example for dns
 
-0.0.2
-+++++
-* Minor fixes.
+### Format PR Description
 
-0.0.1
-+++++
-* Initial release
-```
+If you would like to write multiple history notes for one PR, please write the notes under `History Notes` section in the PR description,  following the same format described above. In this case, the PR title should be a summary of all the changes in this PR and will not be put into `HISTORY.rst`.
+
+Other than that, you can put any reasonable information in the description above the `History Notes` part.
+
+### Only Modify HISTORY.rst for Hotfix
+As described above, history notes will be auto-generated and inserted into `HISTORY.rst` during release. When submitting a normal PR, you don't need to modify `HISTORY.rst` manually. In the case of hotfix, you should create a hotfix branch based on release branch and submit a PR to merge hotfix into release. In this PR, if you have customer-facing changes, you need to modify `HISTORY.rst` to add history notes. Later on you also need to merge the hotfix branch into dev branch and the second part of the PR title should be `Hotfix`. The auto-generation of history notes for the next release will ignore the commit that contains `Hotfix`.
+
+An example title of hotfix change PR:
+
+>{Packaging} Hotfix: fix dependency error
diff --git a/doc/extensions/README.md b/doc/extensions/README.md
@@ -25,6 +25,16 @@ What an Extension is not
 - If you're looking to build your own CLI using `az` patterns, see [knack](https://github.com/Microsoft/knack).
 
 
+How to find and install an Extension
+------------------------
+
+- List all available extensions: `az extension list-available`
+
+- Install an extension: `az extension install --name <extension-name>`
+
+More details on usage in [Extensions for Azure CLI 2.0](https://docs.microsoft.com/cli/azure/azure-cli-extensions-overview?view=azure-cli-latest#install-extensions)
+
+
 Doc Sections
 ------------
 

diff --git a/scripts/ci/credscan/CredScanSuppressions.json b/scripts/ci/credscan/CredScanSuppressions.json
@@ -81,6 +81,10 @@
       "placeholder": "+XLy+MVZ+aTeOnVzN2kLeB16O+kSxmz6g3rS6fAf6rw=",
       "_justification": "[IoT] hard code access key"
     },
+    {
+      "placeholder": "RTNrGy6n%2Fs2uLLZFuVHBIJtdxIJP1LuKfhKjHwwbu7A%3D",
+      "_justification": "[IoT] false alarm of signature in test_sas_token_auth.py"
+    },
     {
       "file": [
         "src\\azure-cli\\azure\\cli\\command_modules\\servicebus\\tests\\latest\\recordings\\test_sb_alias.yaml",
@@ -122,6 +126,10 @@
       "placeholder": "admin123",
       "_justification": "[VM] admin user name for test_vm_defaults.py"
     },
+    {
+      "placeholder": "verySecret!",
+      "_justification": "[VM] one-off password in test"
+    },
     {
       "placeholder": "testpassword",
       "_justification": "[DMS] one-off password in test"
@@ -142,6 +150,10 @@
       "placeholder": "SecretPassword123",
       "_justification": "[lab]/[rdbms] one-off password in test"
     },
+    {
+      "placeholder": "AZURE_CLIENT_SECRET",
+      "_justification": "[rdbms] false alarm about environment variable name"
+    },
     {
       "placeholder": "pass1234",
       "_justification": "[KeyVault] one-off password in test"
@@ -222,6 +234,14 @@
       ],
       "_justification": "[SQL] one-off password used for test"
     },
+    {
+      "placeholder": "SqlPassword",
+      "_justification": "[SQL] False alarm about ClientAuthenticationType"
+    },
+    {
+      "placeholder": "ADPassword",
+      "_justification": "[SQL] False alarm about ClientAuthenticationType"
+    },
     {
       "file": [
         "src\\azure-cli\\azure\\cli\\command_modules\\monitor\\tests\\latest\\recordings\\test_monitor_log_analytics_workspace_default.yaml"
@@ -270,7 +290,14 @@
         "src\\azure-cli\\azure\\cli\\command_modules\\keyvault\\tests\\latest\\mydomain.test.encrypted.pem",
         "src\\azure-cli\\azure\\cli\\command_modules\\keyvault\\tests\\latest\\import_pfx.pfx",
         "src\\azure-cli\\azure\\cli\\command_modules\\keyvault\\tests\\latest\\import_pem_plain.pem",
-        "src\\azure-cli\\azure\\cli\\command_modules\\keyvault\\tests\\latest\\ec521pw.pem"
+        "src\\azure-cli\\azure\\cli\\command_modules\\keyvault\\tests\\latest\\ec521pw.pem",
+        "src\\azure-cli\\azure\\cli\\command_modules\\keyvault\\tests\\latest\\keys\\ec-p256.pem",
+        "src\\azure-cli\\azure\\cli\\command_modules\\keyvault\\tests\\latest\\keys\\ec-p256k.pem",
+        "src\\azure-cli\\azure\\cli\\command_modules\\keyvault\\tests\\latest\\keys\\ec-p521.pem",
+        "src\\azure-cli\\azure\\cli\\command_modules\\keyvault\\tests\\latest\\keys\\ec-p384.pem",
+        "src\\azure-cli\\azure\\cli\\command_modules\\keyvault\\tests\\latest\\keys\\rsa-2048.pem",
+        "src\\azure-cli\\azure\\cli\\command_modules\\keyvault\\tests\\latest\\keys\\rsa-3072.pem",
+        "src\\azure-cli\\azure\\cli\\command_modules\\keyvault\\tests\\latest\\keys\\rsa-4096.pem"
       ],
       "_justification": "[KeyVault] Test certs"
     },
@@ -294,6 +321,10 @@
       "placeholder": "Docker!",
       "_justification": "[AppService] hard code password in custom.py of service which must be took down in the future"
     },
+    {
+      "placeholder": "myPwd",
+      "_justification": "[AppService] hard code MySQL password in _help.py"
+    },
     {
       "placeholder": "5+36OCtbIwfy8g5glC4bQQrFsfmMc3iD",
       "_justification": "[Container] hard code password in test_container_commands.py"
@@ -306,6 +337,10 @@
         "src\\azure-cli\\azure\\cli\\command_modules\\batch\\tests\\latest\\data\\batch-pool-create.json"
       ],
       "_justification": "unclear file contains password, need to remove in the future"
+    },
+    {
+      "placeholder": "f6Nx8en4sIJQryYFVVj%2B5BdU7bho96jAgOzLO40Twkg%3D",
+      "_justification": "[DeploymentManager] false alarm of signature in _help.py"
     }
   ]
 }