Skip to content

Commit 112f347

Browse files
allxiaoallxiao
authored
Execute deployment on slave when the job is started from slave (jenkinsci#8)
* Schedule the build to slave node if the job is running on slave * Complete unit tests * Send AI event based on the deployment state
1 parent a011043 commit 112f347

File tree

22 files changed

+1001
-294
lines changed

22 files changed

+1001
-294
lines changed

‎src/main/java/com/microsoft/jenkins/kubernetes/KubernetesClientWrapper.java‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,12 @@
77
package com.microsoft.jenkins.kubernetes;
88

99
import com.google.common.annotations.VisibleForTesting;
10+
import com.microsoft.jenkins.kubernetes.credentials.ResolvedDockerRegistryEndpoint;
1011
import com.microsoft.jenkins.kubernetes.util.CommonUtils;
1112
import com.microsoft.jenkins.kubernetes.util.Constants;
1213
import com.microsoft.jenkins.kubernetes.util.DockerConfigBuilder;
1314
import hudson.EnvVars;
1415
import hudson.FilePath;
15-
import hudson.model.Item;
1616
import hudson.util.VariableResolver;
1717
import io.fabric8.kubernetes.api.model.HasMetadata;
1818
import io.fabric8.kubernetes.api.model.Job;
@@ -32,7 +32,6 @@
3232
import io.fabric8.kubernetes.client.DefaultKubernetesClient;
3333
import io.fabric8.kubernetes.client.KubernetesClient;
3434
import org.apache.commons.lang.StringUtils;
35-
import org.jenkinsci.plugins.docker.commons.credentials.DockerRegistryEndpoint;
3635

3736
import java.io.IOException;
3837
import java.io.PrintStream;
@@ -163,7 +162,6 @@ public void apply(FilePath[] configFiles) throws IOException, InterruptedExcepti
163162
* <p>
164163
* This can be used by the Pods later to pull images from the private container registry.
165164
*
166-
* @param context the current job context, generally this should be {@code getRun().getParent()}
167165
* @param kubernetesNamespace The namespace in which the Secret should be created / updated
168166
* @param secretName The name of the Secret
169167
* @param credentials All the configured credentials
@@ -173,14 +171,13 @@ public void apply(FilePath[] configFiles) throws IOException, InterruptedExcepti
173171
* </a>
174172
*/
175173
public void createOrReplaceSecrets(
176-
Item context,
177174
String kubernetesNamespace,
178175
String secretName,
179-
List<DockerRegistryEndpoint> credentials) throws IOException {
176+
List<ResolvedDockerRegistryEndpoint> credentials) throws IOException {
180177
log(Messages.KubernetesClientWrapper_prepareSecretsWithName(secretName));
181178

182179
DockerConfigBuilder dockerConfigBuilder = new DockerConfigBuilder(credentials);
183-
String dockercfg = dockerConfigBuilder.buildDockercfgBase64(context);
180+
String dockercfg = dockerConfigBuilder.buildDockercfgBase64();
184181

185182
Map<String, String> data = new HashMap<>();
186183
data.put(".dockercfg", dockercfg);

‎src/main/java/com/microsoft/jenkins/kubernetes/KubernetesDeployContext.java‎

Lines changed: 26 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -15,20 +15,24 @@
1515
import com.microsoft.jenkins.azurecommons.command.SimpleBuildStepExecution;
1616
import com.microsoft.jenkins.azurecommons.remote.SSHClient;
1717
import com.microsoft.jenkins.kubernetes.command.DeploymentCommand;
18+
import com.microsoft.jenkins.kubernetes.credentials.ClientWrapperFactory;
1819
import com.microsoft.jenkins.kubernetes.credentials.ConfigFileCredentials;
1920
import com.microsoft.jenkins.kubernetes.credentials.KubernetesCredentialsType;
21+
import com.microsoft.jenkins.kubernetes.credentials.ResolvedDockerRegistryEndpoint;
2022
import com.microsoft.jenkins.kubernetes.credentials.SSHCredentials;
2123
import com.microsoft.jenkins.kubernetes.credentials.TextCredentials;
2224
import com.microsoft.jenkins.kubernetes.util.Constants;
2325
import hudson.Extension;
2426
import hudson.FilePath;
2527
import hudson.Launcher;
28+
import hudson.model.Item;
2629
import hudson.model.Run;
2730
import hudson.model.TaskListener;
2831
import hudson.util.FormValidation;
2932
import hudson.util.ListBoxModel;
3033
import org.apache.commons.lang3.StringUtils;
3134
import org.jenkinsci.plugins.docker.commons.credentials.DockerRegistryEndpoint;
35+
import org.jenkinsci.plugins.docker.commons.credentials.DockerRegistryToken;
3236
import org.jenkinsci.plugins.workflow.steps.StepContext;
3337
import org.jenkinsci.plugins.workflow.steps.StepDescriptor;
3438
import org.jenkinsci.plugins.workflow.steps.StepExecution;
@@ -164,14 +168,14 @@ public void setEnableConfigSubstitution(boolean enableConfigSubstitution) {
164168
this.enableConfigSubstitution = enableConfigSubstitution;
165169
}
166170

167-
@Override
168171
public List<DockerRegistryEndpoint> getDockerCredentials() {
169172
if (dockerCredentials == null) {
170173
return ImmutableList.of();
171174
}
172-
return dockerCredentials;
175+
return ImmutableList.copyOf(dockerCredentials);
173176
}
174177

178+
175179
@DataBoundSetter
176180
public void setDockerCredentials(List<DockerRegistryEndpoint> dockerCredentials) {
177181
List<DockerRegistryEndpoint> endpoints = new ArrayList<>();
@@ -191,30 +195,34 @@ public void setDockerCredentials(List<DockerRegistryEndpoint> dockerCredentials)
191195
registryUrl = "http://" + registryUrl;
192196
}
193197
}
194-
endpoint = new DockerRegistryEndpoint(registryUrl, credentialsId);
195-
endpoints.add(endpoint);
198+
endpoints.add(new DockerRegistryEndpoint(registryUrl, credentialsId));
196199
}
197200
this.dockerCredentials = endpoints;
198201
}
199202

200-
public KubernetesClientWrapper buildKubernetesClientWrapper(FilePath workspace) throws Exception {
203+
@Override
204+
public List<ResolvedDockerRegistryEndpoint> resolveEndpoints(Item context) throws IOException {
205+
List<ResolvedDockerRegistryEndpoint> endpoints = new ArrayList<>();
206+
List<DockerRegistryEndpoint> configured = getDockerCredentials();
207+
for (DockerRegistryEndpoint endpoint : configured) {
208+
DockerRegistryToken token = endpoint.getToken(context);
209+
if (token == null) {
210+
throw new IllegalArgumentException("No credentials found for " + endpoint);
211+
}
212+
endpoints.add(new ResolvedDockerRegistryEndpoint(endpoint.getEffectiveUrl(), token));
213+
}
214+
return endpoints;
215+
}
216+
217+
@Override
218+
public ClientWrapperFactory clientFactory() {
201219
switch (getCredentialsTypeEnum()) {
202220
case SSH:
203-
FilePath tempConfig = getSsh().getConfigFilePath(workspace);
204-
try {
205-
return new KubernetesClientWrapper(tempConfig.getRemote());
206-
} finally {
207-
tempConfig.delete();
208-
}
221+
return getSsh().buildClientWrapperFactory();
209222
case KubeConfig:
210-
return new KubernetesClientWrapper(getKubeConfig().getConfigFilePath(workspace).getRemote());
223+
return getKubeConfig().buildClientWrapperFactory();
211224
case Text:
212-
TextCredentials text = getTextCredentials();
213-
return new KubernetesClientWrapper(
214-
text.getServerUrl(),
215-
text.getCertificateAuthorityData(),
216-
text.getClientCertificateData(),
217-
text.getClientKeyData());
225+
return getTextCredentials().buildClientWrapperFactory();
218226
default:
219227
throw new IllegalStateException(
220228
Messages.KubernetesDeployContext_unknownCredentialsType(getCredentialsTypeEnum()));

‎src/main/java/com/microsoft/jenkins/kubernetes/command/DeploymentCommand.java‎

Lines changed: 99 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -16,95 +16,149 @@
1616
import com.microsoft.jenkins.kubernetes.KubernetesCDPlugin;
1717
import com.microsoft.jenkins.kubernetes.KubernetesClientWrapper;
1818
import com.microsoft.jenkins.kubernetes.Messages;
19+
import com.microsoft.jenkins.kubernetes.credentials.ClientWrapperFactory;
20+
import com.microsoft.jenkins.kubernetes.credentials.ResolvedDockerRegistryEndpoint;
1921
import com.microsoft.jenkins.kubernetes.util.Constants;
2022
import hudson.EnvVars;
2123
import hudson.FilePath;
2224
import hudson.model.Item;
25+
import hudson.model.TaskListener;
2326
import hudson.util.VariableResolver;
27+
import io.fabric8.kubernetes.client.KubernetesClient;
28+
import jenkins.security.MasterToSlaveCallable;
2429
import org.apache.commons.lang3.StringUtils;
25-
import org.jenkinsci.plugins.docker.commons.credentials.DockerRegistryEndpoint;
2630

31+
import java.io.IOException;
32+
import java.io.Serializable;
2733
import java.net.URL;
34+
import java.util.HashMap;
2835
import java.util.List;
36+
import java.util.Map;
2937

3038
import static com.google.common.base.Preconditions.checkState;
3139

32-
public class DeploymentCommand implements ICommand<DeploymentCommand.IDeploymentCommand> {
40+
/**
41+
* Command to deploy Kubernetes configurations.
42+
* <p>
43+
* Mark it as serializable so that the inner Callable can be serialized correctly.
44+
*/
45+
public class DeploymentCommand implements ICommand<DeploymentCommand.IDeploymentCommand>, Serializable {
3346
@Override
3447
public void execute(IDeploymentCommand context) {
3548
JobContext jobContext = context.getJobContext();
36-
FilePath workspace = jobContext.getWorkspace();
37-
Item jobItem = jobContext.getRun().getParent();
38-
EnvVars envVars = context.getEnvVars();
39-
String secretNamespace = context.getSecretNamespace();
40-
String configPaths = context.getConfigs();
4149

42-
KubernetesClientWrapper wrapper = null;
50+
// all the final variables below are serializable, which will be captured in the below MasterToSlaveCallable
51+
// and execute on the slave if the job is scheduled on slave.
52+
final TaskListener taskListener = jobContext.getTaskListener();
53+
final String secretNameCfg = context.getSecretName();
54+
final String secretNamespace = context.getSecretNamespace();
55+
final String configPaths = context.getConfigs();
56+
final FilePath workspace = jobContext.getWorkspace();
57+
final String defaultSecretName = jobContext.getRun().getDisplayName();
58+
final EnvVars envVars = context.getEnvVars();
59+
final boolean enableSubstitution = context.isEnableConfigSubstitution();
60+
final ClientWrapperFactory clientFactory = context.clientFactory();
61+
62+
TaskResult taskResult = null;
4363
try {
44-
checkState(StringUtils.isNotBlank(secretNamespace), Messages.DeploymentCommand_blankNamespace());
45-
checkState(StringUtils.isNotBlank(configPaths), Messages.DeploymentCommand_blankConfigFiles());
46-
47-
wrapper = context.buildKubernetesClientWrapper(workspace).withLogger(jobContext.logger());
48-
49-
FilePath[] configFiles = workspace.list(configPaths);
50-
if (configFiles.length == 0) {
51-
context.logError(Messages.DeploymentCommand_noMatchingConfigFiles(configPaths));
52-
return;
53-
}
54-
55-
List<DockerRegistryEndpoint> dockerCredentials = context.getDockerCredentials();
56-
if (!dockerCredentials.isEmpty()) {
57-
String secretName = KubernetesClientWrapper.prepareSecretName(
58-
context.getSecretName(), jobContext.getRun().getDisplayName(), envVars);
59-
60-
wrapper.createOrReplaceSecrets(jobItem, secretNamespace, secretName, dockerCredentials);
61-
62-
context.logStatus(Messages.DeploymentCommand_injectSecretName(
63-
Constants.KUBERNETES_SECRET_NAME_PROP, secretName));
64-
EnvironmentInjector.inject(
65-
jobContext.getRun(), envVars, Constants.KUBERNETES_SECRET_NAME_PROP, secretName);
64+
final List<ResolvedDockerRegistryEndpoint> dockerRegistryEndpoints =
65+
context.resolveEndpoints(jobContext.getRun().getParent());
66+
taskResult = workspace.act(new MasterToSlaveCallable<TaskResult, Exception>() {
67+
@Override
68+
public TaskResult call() throws Exception {
69+
TaskResult result = new TaskResult();
70+
71+
checkState(StringUtils.isNotBlank(secretNamespace), Messages.DeploymentCommand_blankNamespace());
72+
checkState(StringUtils.isNotBlank(configPaths), Messages.DeploymentCommand_blankConfigFiles());
73+
74+
KubernetesClientWrapper wrapper =
75+
clientFactory.buildClient(workspace).withLogger(taskListener.getLogger());
76+
result.masterHost = getMasterHost(wrapper);
77+
78+
FilePath[] configFiles = workspace.list(configPaths);
79+
if (configFiles.length == 0) {
80+
String message = Messages.DeploymentCommand_noMatchingConfigFiles(configPaths);
81+
taskListener.error(message);
82+
result.commandState = CommandState.HasError;
83+
throw new IllegalStateException(message);
84+
}
85+
86+
if (!dockerRegistryEndpoints.isEmpty()) {
87+
String secretName =
88+
KubernetesClientWrapper.prepareSecretName(secretNameCfg, defaultSecretName, envVars);
89+
90+
wrapper.createOrReplaceSecrets(secretNamespace, secretName, dockerRegistryEndpoints);
91+
92+
taskListener.getLogger().println(Messages.DeploymentCommand_injectSecretName(
93+
Constants.KUBERNETES_SECRET_NAME_PROP, secretName));
94+
envVars.put(Constants.KUBERNETES_SECRET_NAME_PROP, secretName);
95+
result.extraEnvVars.put(Constants.KUBERNETES_SECRET_NAME_PROP, secretName);
96+
}
97+
98+
if (enableSubstitution) {
99+
wrapper.withVariableResolver(new VariableResolver.ByMap<>(envVars));
100+
}
101+
102+
wrapper.apply(configFiles);
103+
104+
result.commandState = CommandState.Success;
105+
106+
return result;
107+
}
108+
});
109+
for (Map.Entry<String, String> entry : taskResult.extraEnvVars.entrySet()) {
110+
EnvironmentInjector.inject(jobContext.getRun(), envVars, entry.getKey(), entry.getValue());
66111
}
67112

68-
if (context.isEnableConfigSubstitution()) {
69-
wrapper.withVariableResolver(new VariableResolver.ByMap<>(envVars));
113+
context.setCommandState(taskResult.commandState);
114+
if (taskResult.commandState.isError()) {
115+
KubernetesCDPlugin.sendEvent(Constants.AI_KUBERNETES, "DeployFailed",
116+
Constants.AI_K8S_MASTER, AppInsightsUtils.hash(taskResult.masterHost));
117+
} else {
118+
KubernetesCDPlugin.sendEvent(Constants.AI_KUBERNETES, "Deployed",
119+
Constants.AI_K8S_MASTER, AppInsightsUtils.hash(taskResult.masterHost));
70120
}
71-
72-
wrapper.apply(configFiles);
73-
74-
context.setCommandState(CommandState.Success);
75-
76-
KubernetesCDPlugin.sendEvent(Constants.AI_KUBERNETES, "Deployed",
77-
Constants.AI_K8S_MASTER, AppInsightsUtils.hash(getMasterHost(wrapper)));
78121
} catch (Exception e) {
79122
if (e instanceof InterruptedException) {
80123
Thread.currentThread().interrupt();
81124
}
82125
context.logError(e);
83126
KubernetesCDPlugin.sendEvent(Constants.AI_KUBERNETES, "DeployFailed",
84-
Constants.AI_K8S_MASTER, AppInsightsUtils.hash(getMasterHost(wrapper)),
127+
Constants.AI_K8S_MASTER, AppInsightsUtils.hash(taskResult == null ? null : taskResult.masterHost),
85128
Constants.AI_MESSAGE, e.getMessage());
86129
}
87130
}
88131

89132
@VisibleForTesting
90133
String getMasterHost(KubernetesClientWrapper wrapper) {
91134
if (wrapper != null) {
92-
URL masterURL = wrapper.getClient().getMasterUrl();
93-
if (masterURL != null) {
94-
return masterURL.getHost();
135+
KubernetesClient client = wrapper.getClient();
136+
if (client != null) {
137+
URL masterURL = client.getMasterUrl();
138+
if (masterURL != null) {
139+
return masterURL.getHost();
140+
}
95141
}
96142
}
97143
return "Unknown";
98144
}
99145

146+
public static class TaskResult implements Serializable {
147+
private static final long serialVersionUID = 1L;
148+
149+
private CommandState commandState = CommandState.Unknown;
150+
private String masterHost;
151+
private final Map<String, String> extraEnvVars = new HashMap<>();
152+
}
153+
100154
public interface IDeploymentCommand extends IBaseCommandData {
101-
KubernetesClientWrapper buildKubernetesClientWrapper(FilePath workspace) throws Exception;
155+
ClientWrapperFactory clientFactory();
102156

103157
String getSecretNamespace();
104158

105159
String getSecretName();
106160

107-
List<DockerRegistryEndpoint> getDockerCredentials();
161+
List<ResolvedDockerRegistryEndpoint> resolveEndpoints(Item context) throws IOException;
108162

109163
String getConfigs();
110164

‎src/main/java/com/microsoft/jenkins/kubernetes/credentials/ClientWrapperFactory.java‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
/*
2+
* Copyright (c) Microsoft Corporation. All rights reserved.
3+
* Licensed under the MIT License. See License.txt in the project root for
4+
* license information.
5+
*/
6+
7+
package com.microsoft.jenkins.kubernetes.credentials;
8+
9+
import com.microsoft.jenkins.kubernetes.KubernetesClientWrapper;
10+
import hudson.FilePath;
11+
12+
import java.io.Serializable;
13+
14+
/**
15+
* The serializable factory that produces a {@link KubernetesClientWrapper}.
16+
* <p>
17+
* The implementation should be serializable. It will be passed to the remote node when required, and build the
18+
* client wrapper there.
19+
*/
20+
public interface ClientWrapperFactory extends Serializable {
21+
KubernetesClientWrapper buildClient(FilePath workspace) throws Exception;
22+
23+
/**
24+
* The builder that builds {@link ClientWrapperFactory}.
25+
*/
26+
interface Builder {
27+
ClientWrapperFactory buildClientWrapperFactory();
28+
}
29+
}

0 commit comments

Comments
 (0)