Skip to content

Commit 8fd1a09

Browse files
SylvainSenechalSylvainSenechal
committed
test ingestion
1 parent 2b08f5c commit 8fd1a09

File tree

8 files changed

+151
-73
lines changed

8 files changed

+151
-73
lines changed

‎.github/workflows/end2end.yaml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -662,9 +662,9 @@ jobs:
662662
- name: Configure E2E test environment
663663
run: bash configure-e2e-ctst.sh
664664
working-directory: ./.github/scripts/end2end
665-
# - name: Run CTST end to end tests
666-
# run: bash run-e2e-ctst.sh "" "" "" "" --tags 'not @PRA'
667-
# working-directory: ./.github/scripts/end2end
665+
- name: Run CTST end to end tests
666+
run: bash run-e2e-ctst.sh "" "" "" "" --tags 'not @PRA'
667+
working-directory: ./.github/scripts/end2end
668668
- name: Debug wait
669669
uses: ./.github/actions/debug-wait
670670
timeout-minutes: 60

‎solution/deps.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ backbeat:
66
dashboard: backbeat/backbeat-dashboards
77
image: backbeat
88
policy: backbeat/backbeat-policies
9-
tag: 9.1.136
9+
tag: 9.1.140
1010
envsubst: BACKBEAT_TAG
1111
busybox:
1212
image: busybox

‎tests/zenko_tests/node_tests/backbeat/BackbeatAPIUtility.js‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,11 +72,13 @@ class BackbeatAPIUtility {
7272
let path = '/_/backbeat/api/ingestion/resume';
7373
let requestBody;
7474
if (locationName && (schedule === false)) {
75-
path = `${path}/${locationName}`;
75+
path = `${path}/${locationName}test`;
7676
} else if (locationName && hoursScheduled) {
7777
path = `${path}/${locationName}/schedule`;
7878
requestBody = JSON.stringify({ hours: hoursScheduled });
7979
}
80+
console.log("WWWWW 103", path)
81+
8082
return makeUpdateRequest(path, (err, res) => {
8183
assert.ifError(err);
8284
return getResponseBody(res, cb);

‎tests/zenko_tests/node_tests/backbeat/IngestionUtility.js‎

Lines changed: 76 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ class IngestionUtility extends ReplicationUtility {
2121
}
2222

2323
getSourceObject(bucketName, objName, versionId, cb) {
24+
console.log(`getSourceObject: getting ${objName} from ${bucketName} with version ${versionId}`);
2425
this.ringS3C.send(new GetObjectCommand({
2526
Bucket: bucketName,
2627
Key: objName,
@@ -35,12 +36,17 @@ class IngestionUtility extends ReplicationUtility {
3536
}
3637
data.Body = Buffer.concat(chunks);
3738
}
39+
console.log(`getSourceObject: successfully got ${objName}`);
3840
cb(null, data);
3941
})
40-
.catch(cb);
42+
.catch(err => {
43+
console.log(`getSourceObject: error getting ${objName}: ${err}`);
44+
cb(err);
45+
});
4146
}
4247

4348
getDestObject(bucketName, objName, versionId, cb) {
49+
console.log(`getDestObject: getting ${objName} from ${bucketName} with version ${versionId}`);
4450
this.s3.send(new GetObjectCommand({
4551
Bucket: bucketName,
4652
Key: objName,
@@ -55,9 +61,13 @@ class IngestionUtility extends ReplicationUtility {
5561
}
5662
data.Body = Buffer.concat(chunks);
5763
}
64+
console.log(`getDestObject: successfully got ${objName}`);
5865
cb(null, data);
5966
})
60-
.catch(cb);
67+
.catch(err => {
68+
console.log(`getDestObject: error getting ${objName}: ${err}`);
69+
cb(err);
70+
});
6171
}
6272

6373
createIngestionBucket(bucketName, locationName, cb) {
@@ -78,6 +88,8 @@ class IngestionUtility extends ReplicationUtility {
7888
// For this reason, we are waiting 10 seconds to make sure ingestion processes are up-to-date.
7989
setTimeout(() => {
8090
backbeatAPIUtils.resumeIngestion(locationName, false, null, (err, body) => {
91+
console.log("WWWWW 101", err)
92+
console.log("WWWWW 102", body)
8193
if (err) {
8294
return cb(err);
8395
}
@@ -108,22 +120,30 @@ class IngestionUtility extends ReplicationUtility {
108120
waitUntilIngested(bucketName, key, versionId, cb) {
109121
let status = false;
110122
return async.doWhilst(
111-
callback => this.s3.send(new HeadObjectCommand({
112-
Bucket: bucketName,
113-
Key: key,
114-
VersionId: versionId,
115-
}))
116-
.then(() => {
117-
status = true;
118-
return callback();
119-
})
120-
.catch(err => {
121-
if (err.name !== 'NotFound') {
122-
return callback(err);
123-
}
124-
return setTimeout(callback, 2000);
125-
}),
126-
() => !status,
123+
callback => {
124+
console.log('waitUntilIngested: sending HeadObject command');
125+
this.s3.send(new HeadObjectCommand({
126+
Bucket: bucketName,
127+
Key: key,
128+
VersionId: versionId,
129+
}))
130+
.then(() => {
131+
console.log('waitUntilIngested: object found, status is true');
132+
status = true;
133+
return callback();
134+
})
135+
.catch(err => {
136+
console.log(`waitUntilIngested: got error: ${err.name}`);
137+
if (err.name !== 'NotFound') {
138+
return callback(err);
139+
}
140+
return setTimeout(callback, 2000);
141+
});
142+
},
143+
() => {
144+
console.log(`waitUntilIngested: checking status: ${status}`);
145+
return !status;
146+
},
127147
cb,
128148
);
129149
}
@@ -166,14 +186,47 @@ class IngestionUtility extends ReplicationUtility {
166186
}
167187

168188
compareObjectsRINGS3C(srcBucket, destBucket, key, versionId, optionalFields, cb) {
189+
console.log(`compareObjectsRINGS3C: starting comparison for key ${key} in bucket ${destBucket} with version ${versionId}`);
169190
return async.series([
170-
next => this.waitUntilIngested(destBucket, key, versionId, next),
171-
next => this.getSourceObject(srcBucket, key, versionId, next),
172-
next => this.getDestObject(destBucket, key, versionId, next),
191+
next => {
192+
console.log('compareObjectsRINGS3C: step 1 -> waitUntilIngested');
193+
this.waitUntilIngested(destBucket, key, versionId, err => {
194+
if (err) {
195+
console.error('compareObjectsRINGS3C: error in waitUntilIngested', err);
196+
return next(err);
197+
}
198+
console.log('compareObjectsRINGS3C: waitUntilIngested successfully completed');
199+
return next();
200+
});
201+
},
202+
next => {
203+
console.log('compareObjectsRINGS3C: step 2 -> getSourceObject');
204+
this.getSourceObject(srcBucket, key, versionId, (err, data) => {
205+
if (err) {
206+
console.error('compareObjectsRINGS3C: error in getSourceObject', err);
207+
return next(err);
208+
}
209+
console.log('compareObjectsRINGS3C: getSourceObject successfully returned data');
210+
return next(null, data);
211+
});
212+
},
213+
next => {
214+
console.log('compareObjectsRINGS3C: step 3 -> getDestObject');
215+
this.getDestObject(destBucket, key, versionId, (err, data) => {
216+
if (err) {
217+
console.error('compareObjectsRINGS3C: error in getDestObject', err);
218+
return next(err);
219+
}
220+
console.log('compareObjectsRINGS3C: getDestObject successfully returned data');
221+
return next(null, data);
222+
});
223+
},
173224
], (err, data) => {
174225
if (err) {
226+
console.log(`compareObjectsRINGS3C: error after series: ${err}`);
175227
return cb(err);
176228
}
229+
console.log('compareObjectsRINGS3C: successfully completed series, proceeding with assertions');
177230
const srcData = data[1];
178231
const destData = data[2];
179232
assert.strictEqual(
@@ -190,6 +243,7 @@ class IngestionUtility extends ReplicationUtility {
190243
destData.LastModified.toString(),
191244
);
192245
if (optionalFields) {
246+
console.log('compareObjectsRINGS3C: asserting optional fields');
193247
optionalFields.forEach(field => {
194248
if (field === 'Metadata') {
195249
assert.strictEqual(srcData.customKey, destData.customKey);
@@ -198,6 +252,7 @@ class IngestionUtility extends ReplicationUtility {
198252
}
199253
});
200254
}
255+
console.log('compareObjectsRINGS3C: finished comparison');
201256
return cb();
202257
});
203258
}

0 commit comments

Comments
 (0)