Skip to content
Draft
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
f70735f
feat(lib-storage): add S3 Transfer Manager
lukachad Jun 11, 2025
49fd801
feat(lib-storage): concurrent requests
kuhe Jul 21, 2025
1938251
feat(lib-storage): spec updates
kuhe Jul 21, 2025
c4bf56a
Merge pull request #4 from lukachad/revised-sep-requirements
kuhe Jul 21, 2025
879000e
feat: addressed pr comments
lukachad Jul 21, 2025
94c1e1b
feat: addresses change requests for range and part validation
lukachad Jul 22, 2025
a77718f
Merge pull request #3 from lukachad/concurrency
lukachad Jul 22, 2025
deb53b5
feat: validate part and range test cases
lukachad Jul 22, 2025
16264c0
feat: creating tests for join-streams, made browser specific
lukachad Jul 22, 2025
e558dea
feat: changed join-streams path in browser.spec
lukachad Jul 23, 2025
c726213
feat(lib-storage): readme for S3TransferManager
lukachad Jul 24, 2025
3d26103
feat(lib-storage): more expect statements in multipartdownload tests …
lukachad Jul 24, 2025
c2d6863
feat(lib-storage): added try catch and dispatches for transferFailedE…
lukachad Jul 25, 2025
832e6d7
feat: fixed unhandled promise rejection issue, added unit and e2e tes…
lukachad Jul 28, 2025
3655319
feat: part and range handles 0 byte objects, added helper function fo…
lukachad Jul 28, 2025
2559e6a
feat: custom range download tests and fixes
lukachad Jul 28, 2025
53c730f
feat: fixing unhandled promise error
lukachad Jul 29, 2025
53ef260
feat: cr revisions
lukachad Jul 29, 2025
cf4964f
chore: debug
kuhe Jul 29, 2025
d44f0cc
feat: promise.all temp fix for promise unhandled rejection issue
lukachad Jul 29, 2025
164fcee
chore: update join-streams browser to be consistent with join-streams…
lukachad Jul 30, 2025
f367640
chore: code annotations S3TM functions
lukachad Jul 30, 2025
4450b16
chore: readme updates for s3transfermanager
lukachad Jul 30, 2025
927c394
chore: removed dependencies and unused internal event handler
lukachad Jul 30, 2025
41ee715
chore: import cleanup and removed dependencies
lukachad Jul 30, 2025
21dffe4
Merge pull request #5 from lukachad/testing
lukachad Jul 30, 2025
49022b9
Merge branch 'transfer-manager' into debugging
lukachad Jul 30, 2025
987d3ad
Merge pull request #6 from lukachad/debugging
lukachad Jul 30, 2025
82cc0b5
chore: reademe updates
lukachad Jul 30, 2025
21fdc36
Merge branch 'transfer-manager' into add-documentation
lukachad Jul 30, 2025
8daad36
chore: cr changes for readme and added download() examples
lukachad Jul 30, 2025
060045f
chore: deleted redundant example code file
lukachad Jul 30, 2025
4fc0d2f
chore: cr nits
lukachad Jul 30, 2025
bb26df3
Merge pull request #7 from lukachad/add-documentation
lukachad Jul 30, 2025
63040cc
chore: fixed yarn lockfile
lukachad Jul 31, 2025
1bd3255
chore: deleted unused file
lukachad Jul 31, 2025
2a68516
chore: improved async iterable error handling with minimal performanc…
lukachad Aug 4, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
chore: debug
  • Loading branch information
kuhe committed Jul 29, 2025
commit cf4964fb446b4a8f6b38fd8f1336f1812eeddda5
Original file line number Diff line number Diff line change
@@ -1,16 +1,11 @@
import {
GetObjectCommandOutput,
ListBucketInventoryConfigurationsOutputFilterSensitiveLog,
PutObjectCommand,
S3,
} from "@aws-sdk/client-s3";
import internal from "stream";
import { GetObjectCommandOutput, S3 } from "@aws-sdk/client-s3";
import { Readable } from "node:stream";
import { beforeAll, describe, expect, test as it } from "vitest";

import { getIntegTestResources } from "../../../../tests/e2e/get-integ-test-resources";
import { Upload } from "../Upload";
import { internalEventHandler, S3TransferManager } from "./S3TransferManager";
import type { IS3TransferManager, S3TransferManagerConfig } from "./types";
import type { S3TransferManagerConfig } from "./types";

describe(S3TransferManager.name, () => {
const chunk = "01234567";
Expand Down
12 changes: 6 additions & 6 deletions lib/lib-storage/src/s3-transfer-manager/join-streams.ts
Original file line number Diff line number Diff line change
Expand Up @@ -32,17 +32,17 @@ export async function joinStreams(
* @internal
*/
export async function* iterateStreams(
streams: Promise<StreamingBlobPayloadOutputTypes>[],
promises: Promise<StreamingBlobPayloadOutputTypes>[],
eventListeners?: JoinStreamIterationEvents
): AsyncIterable<StreamingBlobPayloadOutputTypes, void, void> {
let bytesTransferred = 0;
let index = 0;
for (const streamPromise of streams) {
let stream: Awaited<(typeof streams)[0]>;
for (const streamPromise of promises) {
let stream: Awaited<(typeof promises)[0]>;
try {
stream = await streamPromise;
} catch (e) {
await destroy(streams);
await destroy(promises);
eventListeners?.onFailure?.(e, index);
throw e;
}
Expand Down Expand Up @@ -82,9 +82,9 @@ export async function* iterateStreams(
/**
* @internal
*/
async function destroy(streams: Promise<StreamingBlobPayloadOutputTypes>[]): Promise<void> {
async function destroy(promises: Promise<StreamingBlobPayloadOutputTypes>[]): Promise<void> {
await Promise.all(
streams.map(async (streamPromise) => {
promises.map(async (streamPromise) => {
return streamPromise
.then((stream) => {
if (stream instanceof Readable) {
Expand Down