Skip to content

Commit 9730086

Browse files
alexcasalbonialexcasalboni
authored
Merge pull request alexcasalboni#98 from alexcasalboni/97-bugfix-weighted-payload-allocation
Bugfix: weighted payload allocation (uneven division)
2 parents db76ed4 + a44aea2 commit 9730086

File tree

6 files changed

+313
-64
lines changed

6 files changed

+313
-64
lines changed

‎lambda/executor.js‎

Lines changed: 1 addition & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ module.exports.handler = async(event, context) => {
3333
let results;
3434

3535
// pre-generate an array of N payloads
36-
const payloads = generatePayloads(num, payload);
36+
const payloads = utils.generatePayloads(num, payload);
3737

3838
if (enableParallel) {
3939
results = await runInParallel(num, lambdaARN, lambdaAlias, payloads, preProcessorARN, postProcessorARN);
@@ -74,55 +74,6 @@ const extractDataFromInput = (event) => {
7474
};
7575
};
7676

77-
const generatePayloads = (num, payloadInput) => {
78-
if (Array.isArray(payloadInput)) {
79-
// if array, generate a list of payloads based on weights
80-
81-
// fail if empty list or missing weight/payload
82-
if (payloadInput.length === 0 || payloadInput.some(p => !p.weight || !p.payload)) {
83-
throw new Error('Invalid weighted payload structure');
84-
}
85-
86-
if (num < payloadInput.length) {
87-
throw new Error(`You have ${payloadInput.length} payloads and only "num"=${num}. Please increase "num".`);
88-
}
89-
90-
// we use relative weights (not %), so here we compute the total weight
91-
const total = payloadInput.map(p => p.weight).reduce((a, b) => a + b, 0);
92-
93-
// generate an array of num items (to be filled)
94-
const payloads = utils.range(num);
95-
96-
// iterate over weighted payloads and fill the array based on relative weight
97-
let done = 0;
98-
for (let p of payloadInput) {
99-
const howMany = Math.floor(p.weight * num / total);
100-
if (howMany < 1) {
101-
throw new Error('Invalid payload weight (num is too small)');
102-
}
103-
payloads.fill(convertPayload(p.payload), done, done + howMany);
104-
done += howMany;
105-
}
106-
107-
return payloads;
108-
109-
} else {
110-
// if not an array, always use the same payload (still generate a list)
111-
const payloads = utils.range(num);
112-
payloads.fill(convertPayload(payloadInput), 0, num);
113-
return payloads;
114-
}
115-
};
116-
117-
const convertPayload = (payload) => {
118-
// optionally convert everything into string
119-
if (typeof payload !== 'string' && typeof payload !== 'undefined') {
120-
console.log('Converting payload to string from ', typeof payload);
121-
payload = JSON.stringify(payload);
122-
}
123-
return payload;
124-
};
125-
12677
const runInParallel = async(num, lambdaARN, lambdaAlias, payloads, preARN, postARN) => {
12778
const results = [];
12879
// run all invocations in parallel ...

‎lambda/utils.js‎

Lines changed: 62 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -214,7 +214,7 @@ module.exports.invokeLambdaWithProcessors = async(lambdaARN, alias, payload, pre
214214
* Invoke a given Lambda Function:Alias with payload and return its logs.
215215
*/
216216
module.exports.invokeLambda = (lambdaARN, alias, payload) => {
217-
console.log(`Invoking function ${lambdaARN}:${alias || "$LATEST"} with payload ${JSON.stringify(payload)}`);
217+
console.log(`Invoking function ${lambdaARN}:${alias || '$LATEST'} with payload ${JSON.stringify(payload)}`);
218218
const params = {
219219
FunctionName: lambdaARN,
220220
Qualifier: alias,
@@ -225,6 +225,67 @@ module.exports.invokeLambda = (lambdaARN, alias, payload) => {
225225
return lambda.invoke(params).promise();
226226
};
227227

228+
229+
/**
230+
* Generate a list of `num` payloads (repeated or weighted)
231+
*/
232+
module.exports.generatePayloads = (num, payloadInput) => {
233+
if (Array.isArray(payloadInput)) {
234+
// if array, generate a list of payloads based on weights
235+
236+
// fail if empty list or missing weight/payload
237+
if (payloadInput.length === 0 || payloadInput.some(p => !p.weight || !p.payload)) {
238+
throw new Error('Invalid weighted payload structure');
239+
}
240+
241+
if (num < payloadInput.length) {
242+
throw new Error(`You have ${payloadInput.length} payloads and only "num"=${num}. Please increase "num".`);
243+
}
244+
245+
// we use relative weights (not %), so here we compute the total weight
246+
const total = payloadInput.map(p => p.weight).reduce((a, b) => a + b, 0);
247+
248+
// generate an array of num items (to be filled)
249+
const payloads = utils.range(num);
250+
251+
// iterate over weighted payloads and fill the array based on relative weight
252+
let done = 0;
253+
for (let p of payloadInput) {
254+
var howMany = Math.floor(p.weight * num / total);
255+
if (howMany < 1) {
256+
throw new Error('Invalid payload weight (num is too small)');
257+
}
258+
// if it's an uneven division, make sure the last item fills the remaining gap
259+
const howManyWillBeLeft = num - done - howMany;
260+
if (howManyWillBeLeft > 0 && howManyWillBeLeft < howMany) {
261+
howMany += howManyWillBeLeft;
262+
}
263+
// finally fill the list with howMany items
264+
payloads.fill(utils.convertPayload(p.payload), done, done + howMany);
265+
done += howMany;
266+
}
267+
268+
return payloads;
269+
} else {
270+
// if not an array, always use the same payload (still generate a list)
271+
const payloads = utils.range(num);
272+
payloads.fill(utils.convertPayload(payloadInput), 0, num);
273+
return payloads;
274+
}
275+
};
276+
277+
/**
278+
* Convert payload to string, if it's not a string already
279+
*/
280+
module.exports.convertPayload = (payload) => {
281+
// optionally convert everything into string
282+
if (typeof payload !== 'string' && typeof payload !== 'undefined') {
283+
console.log('Converting payload to string from ', typeof payload);
284+
payload = JSON.stringify(payload);
285+
}
286+
return payload;
287+
};
288+
228289
/**
229290
* Compute average price and returns with average duration.
230291
*/

‎package-lock.json‎

Lines changed: 8 additions & 8 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎package.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,8 @@
2929
},
3030
"dependencies": {},
3131
"devDependencies": {
32-
"aws-sdk": "^2.455.0",
33-
"aws-sdk-mock": "^1.6.1",
32+
"aws-sdk": "^2.697.0",
33+
"aws-sdk-mock": "^1.7.0",
3434
"coveralls": "^3.0.3",
3535
"eslint": "^6.1.0",
3636
"eslint-config-strongloop": "^2.1.0",

‎test/unit/test-lambda.js‎

Lines changed: 25 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -474,9 +474,9 @@ describe('Lambda Functions', async() => {
474474

475475
it('should invoke the given cb, when done (weighted payload)', async() => {
476476
const weightedPayload = [
477-
{ name: 'A', payload: {test: 'A'}, weight: 10 },
478-
{ name: 'B', payload: {test: 'B'}, weight: 30 },
479-
{ name: 'C', payload: {test: 'C'}, weight: 60 },
477+
{ payload: {test: 'A'}, weight: 10 },
478+
{ payload: {test: 'B'}, weight: 30 },
479+
{ payload: {test: 'C'}, weight: 60 },
480480
];
481481
await invokeForSuccess(handler, {
482482
value: '128',
@@ -526,6 +526,28 @@ describe('Lambda Functions', async() => {
526526
expect(counters.C).to.be(60);
527527
});
528528

529+
it('should invoke the given cb, when done (weighted payload 3)', async() => {
530+
const weightedPayload = [
531+
{ payload: {test: 'A'}, weight: 1 },
532+
{ payload: {test: 'B'}, weight: 1 },
533+
{ payload: {test: 'C'}, weight: 1 },
534+
];
535+
await invokeForSuccess(handler, {
536+
value: '128',
537+
input: {
538+
lambdaARN: 'arnOK',
539+
num: 10,
540+
payload: weightedPayload,
541+
},
542+
});
543+
544+
expect(invokeLambdaPayloads.length).to.be(10);
545+
invokeLambdaPayloads.forEach(payload => {
546+
expect(payload).to.be.a('string');
547+
});
548+
549+
});
550+
529551
it('should explode if count(payloads) < num', async() => {
530552
const weightedPayload = [
531553
{ payload: {test: 'A'}, weight: 5 },

0 commit comments

Comments
 (0)