Skip to content

[HDInsight] Support kafka rest proxy feature #7844

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
fengzhou-msft merged 3 commits into from
Dec 4, 2019
Merged

[HDInsight] Support kafka rest proxy feature #7844

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
46de1ec
1. Add KafkaRestProperties feature
Nov 15, 2019
4cc08ae
Add Kafka Rest Proxy example
Nov 25, 2019
6ff955b
1. Fix syntax error in example
Nov 26, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
1. Fix syntax error in example 
2. Add response body to preview example
3. Add response body to stable example
4. fix prettiercheck
  • Loading branch information
Zhenyu Zhou committed Dec 2, 2019
commit 6ff955bdafd257ad4fae47b549f64ca1f6e4183c
6 changes: 3 additions & 3 deletions ...on/hdinsight/resource-manager/Microsoft.HDInsight/preview/2015-03-01-preview/cluster.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@
"Create HDInsight cluster with Autoscale configuration": {
"$ref": "./examples/CreateHDInsightClusterWithAutoscaleConfig.json"
},
"Create Kafka cluster with Kafak Rest Proxy": {
"Create Kafka cluster with Kafka Rest Proxy": {
"$ref": "./examples/CreateKafkaClusterWithKafkaRestProxy.json"
}
},
Expand Down Expand Up @@ -559,7 +559,7 @@
"ClientGroupInfo": {
Copy link
Contributor

@KrisBash KrisBash Dec 4, 2019

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's strongly preferred to add new configurable properties in a new API version. The issue is when a user updates the new properties, and another user does a GET->PUT with an older SDK. The new properties would be omitted from the request body from the downlevel SDK

Copy link
Member Author

@aim-for-better aim-for-better Dec 4, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's strongly preferred to add new configurable properties in a new API version. The issue is when a user updates the new properties, and another user does a GET->PUT with an older SDK. The new properties would be omitted from the request body from the downlevel SDK

Thanks for your comment. This new properties is not mandatory. And I agree with your opinion. But now we don't have enough time to do this. Adding new api version also needs rp's support.

"description": "The information of AAD security group.",
"properties": {
"groupName":{
"groupName": {
"type": "string",
"description": "The AAD security group name."
},
Expand All @@ -576,7 +576,7 @@
"$ref": "#/definitions/ClientGroupInfo",
"description": "The information of AAD security group."
}
}
}
},
"SecurityProfile": {
"description": "The security profile which contains Ssh public key for the HDInsight cluster.",
Expand Down
79 changes: 79 additions & 0 deletions ...nsight/preview/2015-03-01-preview/examples/CreateHDInsightClusterWithAutoscaleConfig.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,6 +117,85 @@
"200": {
"headers": {
"location": "https://management.azure.com/subscriptions/subid/providers/Microsoft.HDInsight/pathToOperationResult"
},
"body": {
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/zzy-test-rg/providers/Microsoft.HDInsight/clusters/cluster1",
"name": "cluster1",
"type": "Microsoft.HDInsight/clusters",
"location": "East US",
"etag": "fdf2a6e8-ce83-42cc-8c2d-0ceb11a370ff",
"properties": {
"clusterVersion": "4.0.1000.1",
"osType": "Linux",
"clusterDefinition": {
"blueprint": "https://blueprints.azurehdinsight.net/hadoop-4.0.1000.1.1910270459.json",
"kind": "HADOOP",
"componentVersion": {
"Hadoop": "3.1"
}
},
"computeProfile": {
"roles": [
{
"name": "headnode",
"targetInstanceCount": 2,
"hardwareProfile": {
"vmSize": "standard_d12_v2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser"
}
}
},
{
"name": "workernode",
"targetInstanceCount": 4,
"hardwareProfile": {
"vmSize": "standard_d4_v2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser"
}
}
},
{
"name": "zookeepernode",
"targetInstanceCount": 3,
"hardwareProfile": {
"vmSize": "standard_a2_v2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser"
}
}
}
]
},
"provisioningState": "Succeeded",
"clusterState": "Running",
"createdDate": "2019-11-18T12:25:43.48",
"quotaInfo": {
"coresUsed": 40
},
"connectivityEndpoints": [
{
"name": "SSH",
"protocol": "TCP",
"location": "cluster1-ssh.azurehdinsight.net",
"port": 22
},
{
"name": "HTTPS",
"protocol": "TCP",
"location": "cluster1.azurehdinsight.net",
"port": 443
}
],
"tier": "Standard"
}
}
}
}
Expand Down
187 changes: 149 additions & 38 deletions ...t.HDInsight/preview/2015-03-01-preview/examples/CreateKafkaClusterWithKafkaRestProxy.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,17 +15,17 @@
"Kafka": "2.1"
},
"configurations": {
"gateway": {
"restAuthCredential.isEnabled": "true",
"restAuthCredential.username": "admin4031",
"restAuthCredential.password": "Password1!9681"
}
"gateway": {
"restAuthCredential.isEnabled": true,
"restAuthCredential.username": "admin",
"restAuthCredential.password": "**********"
}
}
},
"kafkaRestProperties": {
"clientGroupInfo": {
"groupName": "KafakaRestProperties123",
"groupId": "7bef90fa-0aa3-4bb4-b4d2-2ae7c14cfe41"
"groupName": "Kafka security group name",
"groupId": "00000000-0000-0000-0000-111111111111"
}
},
"computeProfile": {
Expand All @@ -34,79 +34,190 @@
"name": "headnode",
"targetInstanceCount": 2,
"hardwareProfile": {
"vmSize": "Large"
"vmSize": "Large"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser936",
"password": "Password1!9640"
}
"linuxOperatingSystemProfile": {
"username": "sshuser",
"password": "**********"
}
}
},
{
"name": "workernode",
"targetInstanceCount": 3,
"hardwareProfile": {
"vmSize": "Large"
"vmSize": "Large"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser936",
"password": "Password1!9640"
}
"linuxOperatingSystemProfile": {
"username": "sshuser",
"password": "**********"
}
},
"dataDisksGroups": [
{
"disksPerNode": 8
}
{
"disksPerNode": 8
}
]
},
{
"name": "zookeepernode",
"targetInstanceCount": 3,
"hardwareProfile": {
"vmSize": "Small"
"vmSize": "Small"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser936",
"password": "Password1!9640"
}
"linuxOperatingSystemProfile": {
"username": "sshuser",
"password": "**********"
}
}
},
{
"name": "kafkamanagementnode",
"targetInstanceCount": 2,
"hardwareProfile": {
"vmSize": "Standard_D4_v2"
"vmSize": "Standard_D4_v2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "kafkauser",
"password": "Test123!"
}
"linuxOperatingSystemProfile": {
"username": "kafkauser",
"password": "**********"
}
}
}
]
},
"storageProfile": {
"storageaccounts": [
{
"name": "hdicsharpstorage7236.blob.core.windows.net",
"isDefault": true,
"container": "default8689",
"key": "4QAZ49utFF/WtrBMtnAZSdvrhmF+CKG5UXYUJLSSczYCZz0RX5wCWymqE8t0VxxkCt+ssBTWgte6q3hXBNNdrA=="
}
{
"name": "mystorage.blob.core.windows.net",
"isDefault": true,
"container": "containername",
"key": "storagekey"
}
]
}
}
}
},
"response": {
"responses": {
"200": {
"headers": {
"location": "https://management.azure.com/subscriptions/subid/providers/Microsoft.HDInsight/pathToOperationResult"
},
"body": {
"id": "/subscriptions/00000000-0000-0000-0000-000000000000/resourceGroups/rg1/providers/Microsoft.HDInsight/clusters/cluster1",
"name": "cluster1",
"type": "Microsoft.HDInsight/clusters",
"location": "South Central US",
"etag": "e1266b83-9bda-4797-a906-1bf82c8eb09a",
"tags": null,
"properties": {
"clusterVersion": "4.0.1000.1",
"osType": "Linux",
"clusterDefinition": {
"blueprint": "https://blueprints.azurehdinsight.net/kafka-4.0.1000.1.1911212244.json",
"kind": "KAFKA",
"componentVersion": {
"Kafka": "2.1"
}
},
"computeProfile": {
"roles": [
{
"name": "headnode",
"targetInstanceCount": 2,
"hardwareProfile": {
"vmSize": "standard_d3_v2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser"
}
}
},
{
"name": "workernode",
"targetInstanceCount": 4,
"hardwareProfile": {
"vmSize": "standard_d3_v2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser"
}
},
"dataDisksGroups": [
{
"disksPerNode": 2,
"storageAccountType": "Standard_LRS",
"diskSizeGB": 1023
}
]
},
{
"name": "kafkamanagementnode",
"targetInstanceCount": 2,
"hardwareProfile": {
"vmSize": "standard_d4_v2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser"
}
}
},
{
"name": "zookeepernode",
"targetInstanceCount": 3,
"hardwareProfile": {
"vmSize": "standard_a4_v2"
},
"osProfile": {
"linuxOperatingSystemProfile": {
"username": "sshuser"
}
}
}
]
},
"provisioningState": "Succeeded",
"clusterState": "Running",
"createdDate": "2019-11-25T03:43:23.663",
"quotaInfo": {
"coresUsed": 52
},
"connectivityEndpoints": [
{
"name": "SSH",
"protocol": "TCP",
"location": "cluster1-ssh.azurehdinsight.net",
"port": 22
},
{
"name": "HTTPS",
"protocol": "TCP",
"location": "cluster1.azurehdinsight.net",
"port": 443
},
{
"name": "KafkaRestProxyPublicEndpoint",
"protocol": "TCP",
"location": "cluster1-kafkarest.azurehdinsight.net",
"port": 443
}
],
"tier": "Standard",
"kafkaRestProperties": {
"clientGroupInfo": {
"groupName": "security group name",
"groupId": "00000000-0000-0000-0000-000000000000"
}
}
}
}
}
}
}
}
Loading