# Node.js put records to Kinesis with infinite retry strategy.

My application is really easy. It's going to add 500 records at a time to AWS Kinesis and if there's an error occur I'll just add that back to the queue and it's going to be retry. It's going well but I feel like there's a bug somewhere that I don't see.

'use strict';

const _ = require('lodash');
const moment = require('moment');

const config = require('../config');
const logger = require('../logger');
const numbers = require('../helpers/numbers');

const streamName = config.kinesis.streamName;

let records = [];
module.exports = (kinesis) => {
let sendRecords = () => {
Records: records,
StreamName: streamName
};

if (records.length >= 500) {
const pushingToKinesis = records.splice(0, 500);
} else {
records = []
}

if (err) {
logger.error(err);
}

const failedRecord = _.get(data, 'FailedRecordCount', 0);

if (failedRecord > 0) {

logger.warn(There are \${data.FailedRecordCount} failed);

data.Records.forEach((record, index) => {
if (_.has(record, 'ErrorCode') || _.has(record, 'ErrorMessage')) {
logger.debug(record);

}
});
}
});
};

let putRecord = (record) => {
Data: JSON.stringify(record),
PartitionKey: String(numbers.random() * 100000)
};

if (records.length >= config.kinesis.maxRecords) {
sendRecords();
}
};

return {
putRecord: putRecord,
recordCount: () => {
return records.length;
},
clearRecord: () => {
records = [];
},
init: () => {
setInterval(() => {
if (!_.isEmpty(records)) {
sendRecords();
}
}, 200);
}
};
};

• "I feel like there's a bug somewhere I don't see." - Do you mean that you're worried about overlooked issues or are you actually getting bad data from this script? – SuperBiasedMan Jul 5 '16 at 13:34
• Yes. I'm worried that I might have overlooked something. Am I in the wrong place? – toy Jul 5 '16 at 18:37

There is no obvious bug but;

• You are defining functions both within the return structure and outside of it, which makes it hard to read/parse, just keep it clean and define everything outside of it
• You are sending records in a function called putRecord, that is not good.
• 500 should probably be a constant retrieved with config, same thing for 200
• Using numbers.random() sounds like a terrible idea, you should use a library that generates GUID's (https://www.npmjs.com/package/guid)
• Why assign config.kinesis.streamName to streamName, just assign it straight in the structure. It is one less line of code, and the reader does not have to wonder where it comes from
• I would have re-ordered the 500 check and the assignment:

records  = records.splice(0, 500);

Records: records,
StreamName: streamName
};


let payloadRecords = {
Records: records,
StreamName: streamName
};

if (records.length >= 500) {
const pushingToKinesis = records.splice(0, 500);
} else {
records = []
}


personally I would even go for

//Make sure we never send more than 500 records, splice keeps the rest in records