Skip to content

Commit dca8820

Browse files
baisui1981baisui1981
committed
add aliyunOSS Storage Maven
1 parent 2cb8aa9 commit dca8820

File tree

9 files changed

+885
-19
lines changed

9 files changed

+885
-19
lines changed

‎AliyunOSSStorageWagon/pom.xml‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<parent>
6+
<artifactId>cloud-storage-maven</artifactId>
7+
<groupId>com.gkatzioura.maven.cloud</groupId>
8+
<version>2.3</version>
9+
</parent>
10+
<modelVersion>4.0.0</modelVersion>
11+
12+
<groupId>com.qlangtech.maven.cloud</groupId>
13+
<!-- <artifactId>AliyunOSSStorageWagon</artifactId>-->
14+
<artifactId>oss-storage-wagon</artifactId>
15+
<packaging>maven-plugin</packaging>
16+
17+
<properties>
18+
<httpcore>4.4.10</httpcore>
19+
<powermock.version>1.7.1</powermock.version>
20+
</properties>
21+
22+
<dependencies>
23+
24+
<dependency>
25+
<groupId>com.aliyun.oss</groupId>
26+
<artifactId>aliyun-sdk-oss</artifactId>
27+
<version>3.8.0</version>
28+
</dependency>
29+
30+
<dependency>
31+
<groupId>org.apache.maven.wagon</groupId>
32+
<artifactId>wagon-provider-api</artifactId>
33+
<version>${wagon.version}</version>
34+
<scope>provided</scope>
35+
</dependency>
36+
<dependency>
37+
<groupId>org.apache.maven</groupId>
38+
<artifactId>maven-plugin-api</artifactId>
39+
<version>${maven.plugin.api.version}</version>
40+
<scope>provided</scope>
41+
</dependency>
42+
<!-- <dependency>-->
43+
<!-- <groupId>com.amazonaws</groupId>-->
44+
<!-- <artifactId>aws-java-sdk-s3</artifactId>-->
45+
<!-- <version>${aws.version}</version>-->
46+
<!-- </dependency>-->
47+
<!-- <dependency>-->
48+
<!-- <groupId>com.amazonaws</groupId>-->
49+
<!-- <artifactId>aws-java-sdk-sts</artifactId>-->
50+
<!-- <version>${aws.version}</version>-->
51+
<!-- </dependency>-->
52+
<dependency>
53+
<groupId>com.gkatzioura.maven.cloud</groupId>
54+
<artifactId>cloud-storage-core</artifactId>
55+
<version>${project.parent.version}</version>
56+
</dependency>
57+
<dependency>
58+
<groupId>commons-io</groupId>
59+
<artifactId>commons-io</artifactId>
60+
<version>${commons-io.version}</version>
61+
</dependency>
62+
<dependency>
63+
<groupId>org.apache.maven.plugin-tools</groupId>
64+
<artifactId>maven-plugin-annotations</artifactId>
65+
<version>${maven.plugin.annotations.version}</version>
66+
<scope>provided</scope>
67+
</dependency>
68+
<dependency>
69+
<groupId>org.apache.httpcomponents</groupId>
70+
<artifactId>httpcore</artifactId>
71+
<version>${httpcore}</version>
72+
</dependency>
73+
<dependency>
74+
<groupId>junit</groupId>
75+
<artifactId>junit</artifactId>
76+
<version>${junit.version}</version>
77+
<scope>test</scope>
78+
</dependency>
79+
<dependency>
80+
<groupId>org.powermock</groupId>
81+
<artifactId>powermock-module-junit4</artifactId>
82+
<version>${powermock.version}</version>
83+
<scope>test</scope>
84+
</dependency>
85+
<dependency>
86+
<groupId>org.powermock</groupId>
87+
<artifactId>powermock-api-mockito2</artifactId>
88+
<version>${powermock.version}</version>
89+
<scope>test</scope>
90+
</dependency>
91+
<dependency>
92+
<groupId>org.apache.maven.wagon</groupId>
93+
<artifactId>wagon-provider-test</artifactId>
94+
<version>${wagon.version}</version>
95+
<scope>test</scope>
96+
</dependency>
97+
</dependencies>
98+
99+
<build>
100+
<plugins>
101+
<plugin>
102+
<groupId>org.apache.maven.plugins</groupId>
103+
<artifactId>maven-plugin-plugin</artifactId>
104+
<version>3.3</version>
105+
<executions>
106+
<execution>
107+
<id>default-descriptor</id>
108+
<phase>process-classes</phase>
109+
</execution>
110+
</executions>
111+
</plugin>
112+
</plugins>
113+
</build>
114+
115+
</project>

‎AliyunOSSStorageWagon/src/main/java/com/gkatzioura/maven/cloud/oss/OSSStorageRepository.java‎

Lines changed: 212 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,212 @@
1+
/*
2+
* Copyright 2018 Emmanouil Gkatziouras
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package com.gkatzioura.maven.cloud.oss;
18+
19+
import org.apache.commons.io.IOUtils;
20+
import org.apache.maven.wagon.ResourceDoesNotExistException;
21+
import org.apache.maven.wagon.TransferFailedException;
22+
import org.apache.maven.wagon.authentication.AuthenticationException;
23+
import org.apache.maven.wagon.authentication.AuthenticationInfo;
24+
25+
import java.io.File;
26+
import java.io.IOException;
27+
import java.io.InputStream;
28+
import java.io.OutputStream;
29+
import java.util.ArrayList;
30+
import java.util.List;
31+
import java.util.Objects;
32+
import java.util.logging.Level;
33+
import java.util.logging.Logger;
34+
35+
import com.aliyun.oss.OSS;
36+
import com.aliyun.oss.OSSException;
37+
import com.aliyun.oss.model.ListObjectsRequest;
38+
import com.aliyun.oss.model.OSSObject;
39+
import com.aliyun.oss.model.ObjectListing;
40+
import com.aliyun.oss.model.ObjectMetadata;
41+
import com.aliyun.oss.model.PutObjectRequest;
42+
import com.gkatzioura.maven.cloud.resolver.KeyResolver;
43+
import com.gkatzioura.maven.cloud.transfer.TransferProgress;
44+
import com.gkatzioura.maven.cloud.transfer.TransferProgressFileInputStream;
45+
import com.gkatzioura.maven.cloud.transfer.TransferProgressFileOutputStream;
46+
import com.gkatzioura.maven.cloud.wagon.PublicReadProperty;
47+
48+
public class OSSStorageRepository {
49+
50+
//private final String bucket;
51+
private final String baseDirectory;
52+
53+
private final KeyResolver keyResolver = new KeyResolver();
54+
private OssConfig ossCfg;
55+
private OSS aliyunOSS;
56+
private PublicReadProperty publicReadProperty;
57+
58+
private static final Logger LOGGER = Logger.getLogger(OSSStorageRepository.class.getName());
59+
60+
public OSSStorageRepository() {
61+
// this.bucket = bucket;
62+
this.baseDirectory = "";
63+
this.publicReadProperty = new PublicReadProperty(false);
64+
}
65+
66+
public OSSStorageRepository(PublicReadProperty publicReadProperty) {
67+
68+
this.baseDirectory = "";
69+
this.publicReadProperty = publicReadProperty;
70+
}
71+
72+
public OSSStorageRepository(String baseDirectory) {
73+
74+
this.baseDirectory = baseDirectory;
75+
this.publicReadProperty = new PublicReadProperty(false);
76+
}
77+
78+
public OSSStorageRepository(String baseDirectory, PublicReadProperty publicReadProperty) {
79+
80+
this.baseDirectory = baseDirectory;
81+
this.publicReadProperty = publicReadProperty;
82+
}
83+
84+
85+
public void connect(AuthenticationInfo authenticationInfo, String region) throws AuthenticationException {
86+
this.ossCfg = TaskConfig.getInstance().getOss();
87+
Objects.requireNonNull(ossCfg, "ossCfg can not be null");
88+
this.aliyunOSS = TaskConfig.getInstance().getOSSClient(); //S3Connect.connect(authenticationInfo, region, endpoint, pathStyle);
89+
}
90+
91+
public void copy(String resourceName, File destination, TransferProgress transferProgress) throws TransferFailedException, ResourceDoesNotExistException {
92+
93+
final String key = resolveKey(resourceName);
94+
95+
try {
96+
97+
final OSSObject ossObj;
98+
try {
99+
ossObj = aliyunOSS.getObject(this.ossCfg.getBucket(), key);
100+
} catch (OSSException e) {
101+
throw new ResourceDoesNotExistException("Resource does not exist");
102+
}
103+
destination.getParentFile().mkdirs();//make sure the folder exists or the outputStream will fail.
104+
try (OutputStream outputStream = new TransferProgressFileOutputStream(destination, transferProgress);
105+
InputStream inputStream = ossObj.getObjectContent()) {
106+
IOUtils.copy(inputStream, outputStream);
107+
}
108+
} catch (IOException e) {
109+
LOGGER.log(Level.SEVERE, "Could not transfer file", e);
110+
throw new TransferFailedException("Could not download resource " + key);
111+
}
112+
}
113+
114+
public void put(File file, String destination, TransferProgress transferProgress) throws TransferFailedException {
115+
116+
final String key = resolveKey(destination);
117+
118+
try {
119+
try (InputStream inputStream = new TransferProgressFileInputStream(file, transferProgress)) {
120+
PutObjectRequest putObjectRequest = new PutObjectRequest(this.ossCfg.getBucket(), key, inputStream, createContentLengthMetadata(file));
121+
applyPublicRead(putObjectRequest);
122+
aliyunOSS.putObject(putObjectRequest);
123+
}
124+
} catch (IOException e) {
125+
LOGGER.log(Level.SEVERE, "Could not transfer file ", e);
126+
throw new TransferFailedException("Could not transfer file " + file.getName());
127+
}
128+
}
129+
130+
private ObjectMetadata createContentLengthMetadata(File file) {
131+
ObjectMetadata metadata = new ObjectMetadata();
132+
metadata.setContentLength(file.length());
133+
return metadata;
134+
}
135+
136+
public boolean newResourceAvailable(String resourceName, long timeStamp) throws ResourceDoesNotExistException {
137+
138+
final String key = resolveKey(resourceName);
139+
140+
LOGGER.log(Level.FINER, String.format("Checking if new key %s exists", key));
141+
142+
// try {
143+
ObjectMetadata objectMetadata = aliyunOSS.getObjectMetadata(this.ossCfg.getBucket(), key);
144+
145+
long updated = objectMetadata.getLastModified().getTime();
146+
return updated > timeStamp;
147+
// } catch (AmazonS3Exception e) {
148+
// LOGGER.log(Level.SEVERE, String.format("Could not retrieve %s", key), e);
149+
// throw new ResourceDoesNotExistException("Could not retrieve key " + key);
150+
// }
151+
}
152+
153+
154+
public List<String> list(String path) {
155+
156+
String key = resolveKey(path);
157+
158+
ListObjectsRequest listRequest = (ListObjectsRequest) new ListObjectsRequest()
159+
.withBucketName(this.ossCfg.getBucket()).withKey(key);
160+
161+
ObjectListing objectListing = aliyunOSS.listObjects(listRequest);
162+
// .withPrefix(key));
163+
List<String> objects = new ArrayList<>();
164+
retrieveAllObjects(objectListing, objects);
165+
return objects;
166+
}
167+
168+
private void applyPublicRead(PutObjectRequest putObjectRequest) {
169+
if (publicReadProperty.get()) {
170+
// LOGGER.info("Public read was set to true");
171+
//putObjectRequest.withCannedAcl(CannedAccessControlList.PublicRead);
172+
}
173+
}
174+
175+
private void retrieveAllObjects(ObjectListing objectListing, List<String> objects) {
176+
177+
objectListing.getObjectSummaries().forEach(os -> objects.add(os.getKey()));
178+
179+
if (objectListing.isTruncated()) {
180+
// ObjectListing nextObjectListing = amazonS3.listNextBatchOfObjects(objectListing);
181+
// retrieveAllObjects(nextObjectListing, objects);
182+
}
183+
}
184+
185+
public boolean exists(String resourceName) {
186+
187+
final String key = resolveKey(resourceName);
188+
189+
try {
190+
aliyunOSS.getObjectMetadata(this.ossCfg.getBucket(), key);
191+
return true;
192+
} catch (OSSException e) {
193+
return false;
194+
}
195+
}
196+
197+
public void disconnect() {
198+
this.aliyunOSS = null;
199+
this.ossCfg = null;
200+
}
201+
202+
private String resolveKey(String path) {
203+
return keyResolver.resolve(baseDirectory, path);
204+
}
205+
206+
207+
public String getBaseDirectory() {
208+
return baseDirectory;
209+
}
210+
211+
212+
}

0 commit comments

Comments
 (0)