Skip to content

Commit 3352e3a

Browse files
kuhekuhe
authored
test(lib-storage): example of how to mock requests for MPU (#7378)
1 parent 1f6666e commit 3352e3a

File tree

3 files changed

+314
-1
lines changed

3 files changed

+314
-1
lines changed

‎lib/lib-storage/package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,8 +15,10 @@
1515
"clean": "rimraf ./dist-* && rimraf *.tsbuildinfo",
1616
"extract:docs": "api-extractor run --local",
1717
"test": "yarn g:vitest run",
18-
"test:e2e": "yarn g:vitest run -c vitest.config.e2e.mts --mode development",
1918
"test:watch": "yarn g:vitest watch",
19+
"test:integration": "yarn g:vitest run -c vitest.config.integ.mts",
20+
"test:integration:watch": "yarn g:vitest watch -c vitest.config.integ.mts",
21+
"test:e2e": "yarn g:vitest run -c vitest.config.e2e.mts --mode development",
2022
"test:e2e:watch": "yarn g:vitest watch -c vitest.config.e2e.mts"
2123
},
2224
"engines": {

‎lib/lib-storage/src/lib-storage.integ.spec.ts‎

Lines changed: 303 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,303 @@
1+
import { requireRequestsFrom } from "@aws-sdk/aws-util-test/src";
2+
import { S3 } from "@aws-sdk/client-s3";
3+
import { Upload } from "@aws-sdk/lib-storage";
4+
import { HttpResponse } from "@smithy/protocol-http";
5+
import { randomBytes } from "crypto";
6+
import { Readable } from "node:stream";
7+
import { describe, expect, test as it } from "vitest";
8+
9+
describe("lib storage integration test", () => {
10+
it("verifies CompleteMultipartUpload response is properly mapped to Upload response for large files", async () => {
11+
const client = new S3({
12+
region: "us-west-2",
13+
credentials: {
14+
accessKeyId: "INTEG",
15+
secretAccessKey: "INTEG",
16+
},
17+
});
18+
19+
const commandOutputs: Record<string, any[]> = {};
20+
21+
client.middlewareStack.add((next, context) => async (args) => {
22+
const r = await next(args);
23+
commandOutputs[context.commandName!] = commandOutputs[context.commandName!] ?? [];
24+
commandOutputs[context.commandName!].push(r.output);
25+
return r;
26+
});
27+
28+
requireRequestsFrom(client)
29+
.toMatch({
30+
hostname: /amazon/,
31+
})
32+
.respondWith(
33+
new HttpResponse({
34+
statusCode: 200,
35+
headers: {
36+
"x-amz-id-2":
37+
"bKbXrk1IXbqfupvsn8gGtkGi33Nszcq9iwiah4xGeydCedkuKWeht6xnBkn0sCBhVDyDs0Xa4ecdbnxtyzMGqc17Cv6Se7P8",
38+
"x-amz-request-id": "MC075MYM6KAT5AQE",
39+
date: "Fri, 26 Sep 2025 17:27:23 GMT",
40+
"x-amz-server-side-encryption": "AES256",
41+
"x-amz-checksum-algorithm": "CRC32",
42+
"x-amz-checksum-type": "COMPOSITE",
43+
"transfer-encoding": "chunked",
44+
server: "AmazonS3",
45+
},
46+
body: Readable.from(
47+
Buffer.from(
48+
`<?xml version="1.0" encoding="UTF-8"?>
49+
<InitiateMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
50+
<Bucket>
51+
sdkreleasev3integtestreso-integtestbucketa93771ae-zh5lrv1xnwjx
52+
</Bucket>
53+
<Key>
54+
MPU-1758907641953
55+
</Key>
56+
<UploadId>
57+
FPyQ2V.AnlVZcN3GUqieu5Ael9CllYWycVH0slQyiS9wYjDeUKS0okoMm.jbbbmMbNln.K8HtPzbwjCChgCUH9B94b6MyrD72_auD23tEpXhmel40UtdL.7w_RiNAc1xkyr8ooIKRsGyDE9J.WIH0Q--
58+
</UploadId>
59+
</InitiateMultipartUploadResult>`
60+
)
61+
),
62+
}),
63+
new HttpResponse({
64+
statusCode: 200,
65+
headers: {
66+
"x-amz-id-2":
67+
"1ezf9iJF3YvPWo3SF2UCrgFGBXltd25g9bHUA9W4k58PJ/W03OZ13nIOEmGE+NCRCbmuERJ4lvML5zGQU0JCw44sOelC9sRb",
68+
"x-amz-request-id": "MC01ZVGGQ8Z7950C",
69+
date: "Fri, 26 Sep 2025 17:27:23 GMT",
70+
etag: '"eb3760d36bc660b509833238c0799b58"',
71+
"x-amz-checksum-crc32": "Ikyd7A==",
72+
"x-amz-server-side-encryption": "AES256",
73+
"content-length": "0",
74+
server: "AmazonS3",
75+
},
76+
}),
77+
new HttpResponse({
78+
statusCode: 200,
79+
headers: {
80+
"x-amz-id-2": "JwgQ5LZ9Sx3fj6G46KOjfx7HI2XvK18Nx6iCOQPH+/UFjbFPju3hlZ7Gq8BIW6g2IiyI8cM3v1LPh+Me8KmCZQ==",
81+
"x-amz-request-id": "MC0DH0R666ESRWE3",
82+
date: "Fri, 26 Sep 2025 17:27:23 GMT",
83+
etag: '"76d0701ab8175448d01476321416bf01"',
84+
"x-amz-checksum-crc32": "JTOG+w==",
85+
"x-amz-server-side-encryption": "AES256",
86+
"content-length": "0",
87+
server: "AmazonS3",
88+
},
89+
}),
90+
new HttpResponse({
91+
statusCode: 200,
92+
headers: {
93+
"x-amz-id-2": "0G3bPmuvsW9FMp4OCpbRUxtldh81E3PxbvhUuXsCtasMMpYVfKlxvYkWD9wekOxD/C0xay5ttt/d7MXxz79JBw==",
94+
"x-amz-request-id": "HY3KQKTR6ZKMGZ6E",
95+
date: "Fri, 26 Sep 2025 17:27:24 GMT",
96+
"x-amz-version-id": "PVmXZ_B1Qs3bTot7SY6w_.aiH3TpVbQ6",
97+
"x-amz-server-side-encryption": "AES256",
98+
"content-type": "application/xml",
99+
"transfer-encoding": "chunked",
100+
server: "AmazonS3",
101+
},
102+
body: Readable.from(
103+
Buffer.from(
104+
`<?xml version="1.0" encoding="UTF-8"?>
105+
<CompleteMultipartUploadResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
106+
<Location>
107+
https://bucket.s3.us-west-2.amazonaws.com/MPU-1758907641953
108+
</Location>
109+
<Bucket>
110+
bucket
111+
</Bucket>
112+
<Key>
113+
MPU-1758907641953
114+
</Key>
115+
<ETag>
116+
"e4cd0558ba33b2b33aa64f158deae527-2"
117+
</ETag>
118+
<ChecksumCRC32>
119+
53GakA==-2
120+
</ChecksumCRC32>
121+
<ChecksumType>
122+
COMPOSITE
123+
</ChecksumType>
124+
</CompleteMultipartUploadResult>`
125+
)
126+
),
127+
})
128+
);
129+
130+
const uploadOutput = await new Upload({
131+
client,
132+
params: {
133+
Bucket: "bucket",
134+
Key: "key",
135+
Body: randomBytes(6 * 1024 * 1024),
136+
},
137+
}).done();
138+
139+
/**
140+
* (Because the XML was given without trimming).
141+
*/
142+
function trimObject(item: any): any {
143+
if (typeof item === "string") {
144+
return item.trim();
145+
}
146+
if (Array.isArray(item)) {
147+
return item.map(trimObject);
148+
}
149+
if (item && typeof item === "object") {
150+
for (const key in item) {
151+
item[key] = trimObject(item[key]);
152+
}
153+
}
154+
return item;
155+
}
156+
trimObject(commandOutputs);
157+
158+
expect(commandOutputs).toEqual({
159+
CreateMultipartUploadCommand: [
160+
{
161+
$metadata: {
162+
httpStatusCode: 200,
163+
requestId: "MC075MYM6KAT5AQE",
164+
extendedRequestId:
165+
"bKbXrk1IXbqfupvsn8gGtkGi33Nszcq9iwiah4xGeydCedkuKWeht6xnBkn0sCBhVDyDs0Xa4ecdbnxtyzMGqc17Cv6Se7P8",
166+
attempts: 1,
167+
totalRetryDelay: 0,
168+
},
169+
ServerSideEncryption: "AES256",
170+
ChecksumAlgorithm: "CRC32",
171+
ChecksumType: "COMPOSITE",
172+
Bucket: "sdkreleasev3integtestreso-integtestbucketa93771ae-zh5lrv1xnwjx",
173+
Key: "MPU-1758907641953",
174+
UploadId:
175+
"FPyQ2V.AnlVZcN3GUqieu5Ael9CllYWycVH0slQyiS9wYjDeUKS0okoMm.jbbbmMbNln.K8HtPzbwjCChgCUH9B94b6MyrD72_auD23tEpXhmel40UtdL.7w_RiNAc1xkyr8ooIKRsGyDE9J.WIH0Q--",
176+
},
177+
],
178+
UploadPartCommand: [
179+
{
180+
$metadata: {
181+
httpStatusCode: 200,
182+
requestId: "MC01ZVGGQ8Z7950C",
183+
extendedRequestId:
184+
"1ezf9iJF3YvPWo3SF2UCrgFGBXltd25g9bHUA9W4k58PJ/W03OZ13nIOEmGE+NCRCbmuERJ4lvML5zGQU0JCw44sOelC9sRb",
185+
attempts: 1,
186+
totalRetryDelay: 0,
187+
},
188+
ServerSideEncryption: "AES256",
189+
ETag: '"eb3760d36bc660b509833238c0799b58"',
190+
ChecksumCRC32: "Ikyd7A==",
191+
},
192+
{
193+
$metadata: {
194+
httpStatusCode: 200,
195+
requestId: "MC0DH0R666ESRWE3",
196+
extendedRequestId:
197+
"JwgQ5LZ9Sx3fj6G46KOjfx7HI2XvK18Nx6iCOQPH+/UFjbFPju3hlZ7Gq8BIW6g2IiyI8cM3v1LPh+Me8KmCZQ==",
198+
attempts: 1,
199+
totalRetryDelay: 0,
200+
},
201+
ServerSideEncryption: "AES256",
202+
ETag: '"76d0701ab8175448d01476321416bf01"',
203+
ChecksumCRC32: "JTOG+w==",
204+
},
205+
],
206+
CompleteMultipartUploadCommand: [
207+
{
208+
$metadata: {
209+
httpStatusCode: 200,
210+
requestId: "HY3KQKTR6ZKMGZ6E",
211+
extendedRequestId:
212+
"0G3bPmuvsW9FMp4OCpbRUxtldh81E3PxbvhUuXsCtasMMpYVfKlxvYkWD9wekOxD/C0xay5ttt/d7MXxz79JBw==",
213+
attempts: 1,
214+
totalRetryDelay: 0,
215+
},
216+
ServerSideEncryption: "AES256",
217+
VersionId: "PVmXZ_B1Qs3bTot7SY6w_.aiH3TpVbQ6",
218+
Bucket: "bucket",
219+
ChecksumCRC32: "53GakA==-2",
220+
ChecksumType: "COMPOSITE",
221+
ETag: '"e4cd0558ba33b2b33aa64f158deae527-2"',
222+
Key: "MPU-1758907641953",
223+
Location: "https://bucket.s3.us-west-2.amazonaws.com/MPU-1758907641953",
224+
},
225+
],
226+
});
227+
228+
expect(uploadOutput).toMatchObject(commandOutputs.CompleteMultipartUploadCommand[0]);
229+
});
230+
231+
it("verifies PutObject response is properly mapped to Upload response for small files", async () => {
232+
const client = new S3({
233+
region: "us-west-2",
234+
credentials: {
235+
accessKeyId: "INTEG",
236+
secretAccessKey: "INTEG",
237+
},
238+
});
239+
240+
const commandOutputs: Record<string, any[]> = {};
241+
242+
client.middlewareStack.add((next, context) => async (args) => {
243+
const r = await next(args);
244+
commandOutputs[context.commandName!] = commandOutputs[context.commandName!] ?? [];
245+
commandOutputs[context.commandName!].push(r.output);
246+
return r;
247+
});
248+
249+
requireRequestsFrom(client)
250+
.toMatch({
251+
hostname: /amazon/,
252+
})
253+
.respondWith(
254+
new HttpResponse({
255+
statusCode: 200,
256+
headers: {
257+
"x-amz-id-2":
258+
"abc123def456ghi789jkl012mno345pqr678stu901vwx234yzA567BCD890EFG123HIJ456KLM789NOP012QRS345TUV",
259+
"x-amz-request-id": "ABCD1234EFGH5678",
260+
date: "Fri, 26 Sep 2025 17:30:00 GMT",
261+
etag: '"d41d8cd98f00b204e9800998ecf8427e"',
262+
"x-amz-checksum-crc32": "AAAAAA==",
263+
"x-amz-checksum-type": "CRC32",
264+
"x-amz-server-side-encryption": "AES256",
265+
"x-amz-version-id": "null",
266+
"content-length": "0",
267+
server: "AmazonS3",
268+
},
269+
})
270+
);
271+
272+
const uploadOutput = await new Upload({
273+
client,
274+
params: {
275+
Bucket: "bucket",
276+
Key: "small-file-key",
277+
Body: randomBytes(1024), // 1KB - small enough to use PutObject
278+
},
279+
}).done();
280+
281+
expect(commandOutputs).toEqual({
282+
PutObjectCommand: [
283+
{
284+
$metadata: {
285+
httpStatusCode: 200,
286+
requestId: "ABCD1234EFGH5678",
287+
extendedRequestId:
288+
"abc123def456ghi789jkl012mno345pqr678stu901vwx234yzA567BCD890EFG123HIJ456KLM789NOP012QRS345TUV",
289+
attempts: 1,
290+
totalRetryDelay: 0,
291+
},
292+
ETag: '"d41d8cd98f00b204e9800998ecf8427e"',
293+
ChecksumCRC32: "AAAAAA==",
294+
ChecksumType: "CRC32",
295+
ServerSideEncryption: "AES256",
296+
VersionId: "null",
297+
},
298+
],
299+
});
300+
301+
expect(uploadOutput).toMatchObject(commandOutputs.PutObjectCommand[0]);
302+
});
303+
}, 60_000);

‎lib/lib-storage/vitest.config.integ.mts‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
import { defineConfig } from "vitest/config";
2+
3+
export default defineConfig({
4+
test: {
5+
include: ["**/*.integ.spec.ts"],
6+
environment: "node",
7+
},
8+
});

0 commit comments

Comments
 (0)