From fa8a3efa23843d9355e7f811174871ccf23e57cf Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Wed, 30 Sep 2015 16:25:11 -0700 Subject: [PATCH 01/46] Update tests to pass with a default token The incorrect index error will only happen if an token is locked down to a specific index(es). --- test/test_send.js | 32 ++++++++++++-------------------- 1 file changed, 12 insertions(+), 20 deletions(-) diff --git a/test/test_send.js b/test/test_send.js index b15c9e1..dc66f46 100644 --- a/test/test_send.js +++ b/test/test_send.js @@ -41,11 +41,12 @@ var noDataBody = { code: 5 }; -var incorrectIndexBody = { - text: "Incorrect index", - code: 7, - "invalid-event-number": 1 -}; +// TODO: add a test that gets this response +// var incorrectIndexBody = { +// text: "Incorrect index", +// code: 7, +// "invalid-event-number": 1 +// }; // Backup console.log so we can restore it later var ___log = console.log; @@ -269,8 +270,8 @@ describe("SplunkLogger send", function() { done(); }); }); - // TODO: test successfully sending to another index - it("should error with valid token, sending to a different index", function(done) { + // TODO: test unsuccessfully sending to another index with specific index token settings + it("should succeed with valid token, sending to a different index", function(done) { var config = { token: configurationFile.token }; @@ -287,25 +288,16 @@ describe("SplunkLogger send", function() { } }; - var run = false; - - logger.error = function(err, errContext) { - run = true; - assert.ok(err); - assert.strictEqual(err.message, incorrectIndexBody.text); - assert.strictEqual(err.code, incorrectIndexBody.code); - assert.ok(errContext); - assert.strictEqual(errContext, context); + logger.error = function() { + assert.ok(false); }; logger.send(context, function(err, resp, body) { assert.ok(!err); - assert.ok(run); assert.strictEqual(resp.headers["content-type"], "application/json; charset=UTF-8"); assert.strictEqual(resp.body, body); - assert.strictEqual(body.text, incorrectIndexBody.text); - assert.strictEqual(body.code, incorrectIndexBody.code); - assert.strictEqual(body["invalid-event-number"], incorrectIndexBody["invalid-event-number"]); + assert.strictEqual(body.text, successBody.text); + assert.strictEqual(body.code, successBody.code); done(); }); }); From ae084516d32b1213587aeb77e82d791e041d7395 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Tue, 6 Oct 2015 23:18:54 -0700 Subject: [PATCH 02/46] Add utils.whilst + tests, rename utils.chain tests --- test/test_utils.js | 110 +++++++++++++++++++++++++++++++++++++++++---- utils.js | 25 +++++++++++ 2 files changed, 126 insertions(+), 9 deletions(-) diff --git a/test/test_utils.js b/test/test_utils.js index 5bde605..acd2844 100644 --- a/test/test_utils.js +++ b/test/test_utils.js @@ -161,9 +161,8 @@ describe("Utils", function() { testToArray(1, 2, 3, 4, 5); }); }); - // TODO: rename these tests to the "should..." format describe("chain", function () { - it("single success", function(done) { + it("should succeed with 3 callbacks, passing a single argument through the chain", function(done) { utils.chain([ function(callback) { callback(null, 1); @@ -181,7 +180,7 @@ describe("Utils", function() { } ); }); - it("flat single success", function(done) { + it("should succeed with flat callbacks, passing a single argument through the chain", function(done) { utils.chain( function(callback) { callback(null, 1); @@ -199,7 +198,7 @@ describe("Utils", function() { } ); }); - it("flat multiple success", function(done) { + it("should succeed with flat callbacks, passing multiple arguments through the chain", function(done) { utils.chain( function(callback) { callback(null, 1, 2); @@ -218,7 +217,7 @@ describe("Utils", function() { } ); }); - it("flat add args success", function(done) { + it("should succeed with flat callbacks, appending an argument in the middle of the chain", function(done) { utils.chain( function(callback) { callback(null, 1, 2); @@ -237,7 +236,7 @@ describe("Utils", function() { } ); }); - it("error", function(done) { + it("should surface error from middle of the chain", function(done) { utils.chain([ function(callback) { callback(null, 1, 2); @@ -257,7 +256,7 @@ describe("Utils", function() { } ); }); - it("no tasks", function(done) { + it("should be noop without task callbacks", function(done) { utils.chain([], function(err, val1, val2) { assert.ok(!err); @@ -267,10 +266,10 @@ describe("Utils", function() { } ); }); - it("no args", function() { + it("should be noop without args", function() { utils.chain(); }); - it("no final callback", function(done) { + it("should surface error from first callback in the chain", function(done) { utils.chain([ function(callback) { callback("err"); @@ -282,4 +281,97 @@ describe("Utils", function() { ); }); }); + describe("whilst", function() { + it("should succeed with short counting loop", function(done) { + var i = 0; + utils.whilst( + function() { return i++ < 3; }, + function(callback) { + setTimeout(function() { callback(); }, 0); + }, + function(err) { + assert.ok(!err); + assert.strictEqual(i, 4); + done(); + } + ); + }); + it("should succeed with long counting loop", function(done) { + var i = 0; + utils.whilst( + function() { return i++ < 1000; }, + function(callback) { + setTimeout(function() { callback(); }, 0); + }, + function(err) { + assert.ok(!err); + assert.strictEqual(i, 1001); + done(); + } + ); + }); + it("should pass error to callback function", function(done) { + var i = 0; + utils.whilst( + function() { return i++ < 1000; }, + function(callback) { + setTimeout(function() { callback(i === 1000 ? 1 : null); }, 0); + }, + function(err) { + assert.ok(err); + assert.strictEqual(err, 1); + // Don't execute condition function 1 extra time like above + assert.strictEqual(i, 1000); + done(); + } + ); + }); + it("should never enter loop body when condition loop returns false (default)", function(done) { + var i = false; + utils.whilst( + undefined, + function(callback) { i = true; callback(); }, + function(err) { + assert.ok(!err); + assert.strictEqual(i, false); + done(); + } + ); + }); + it("should be noop with noop loop body", function(done) { + var i = true; + utils.whilst( + function() { + if (i) { + i = false; + return true; + } + else { + return i; + } + }, + undefined, + function (err) { + assert.ok(!err); + done(); + } + ); + }); + it("should succeed with short counting loop, without done callback", function(done) { + var i = 0; + utils.whilst( + function() { return i++ < 3; }, + function(callback) { + setTimeout(function() { callback(); }, 0); + } + ); + + setTimeout(function(){ + if (i !== 4) { + assert.ok(false, "test timed out"); + } + done(); + }, 10); + }); + }); }); \ No newline at end of file diff --git a/utils.js b/utils.js index eeb32b8..5360625 100644 --- a/utils.js +++ b/utils.js @@ -128,4 +128,29 @@ utils.chain = function(tasks, callback) { } }; +/** + * Asynchronous while loop. + */ +utils.whilst = function (condition, body, callback) { + condition = condition || function() { return false; }; + body = body || function(done){ done(); }; + callback = callback || function() {}; + + var wrappedCallback = function(err) { + if (err) { + callback(err); + } + else { + utils.whilst(condition, body, callback); + } + }; + + if (condition()) { + body(wrappedCallback); + } + else { + callback(null); + } +}; + module.exports = utils; \ No newline at end of file From 1a5581099dd926a9f90ac7f34ac83ff41907ec90 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Thu, 8 Oct 2015 19:56:21 -0700 Subject: [PATCH 03/46] Add utils.expBackoff for exponential backoff timeout --- test/test_utils.js | 80 +++++++++++++++++++++++++++++++++++++++++++++- utils.js | 44 +++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/test/test_utils.js b/test/test_utils.js index acd2844..1fa2ef2 100644 --- a/test/test_utils.js +++ b/test/test_utils.js @@ -44,7 +44,6 @@ describe("Utils", function() { var otherFound = utils.formatTime(other); assert.strictEqual(otherFound, otherExpected); }); - // TODO: change these to be strictequal? it("should correctly handle Strings", function() { // Test time in seconds var stringTime = "1372187084"; @@ -374,4 +373,83 @@ describe("Utils", function() { }, 10); }); }); + describe("expBackoff", function() { + it("should error with bad param", function(done) { + utils.expBackoff(null, function(err, timeout) { + assert.ok(err); + assert.strictEqual(err.message, "Must send opts as an object."); + assert.ok(!timeout); + done(); + }); + }); + it("should error with missing opts.attempt", function(done) { + utils.expBackoff({foo: 123}, function(err, timeout) { + assert.ok(err); + assert.strictEqual(err.message, "Must set opts.attempt."); + assert.ok(!timeout); + done(); + }); + }); + it("should have backoff in [20, 40]ms, attempt = 1", function(done) { + utils.expBackoff({attempt: 1}, function(err, timeout) { + assert.ok(!err); + assert.ok(20 <= timeout && timeout <= 40); + done(); + }); + }); + it("should have backoff in [40, 80]ms, attempt = 2", function(done) { + utils.expBackoff({attempt: 2}, function(err, timeout) { + assert.ok(!err); + assert.ok(40 <= timeout && timeout <= 80); + done(); + }); + }); + it("should have backoff in [80, 160]ms, attempt = 3", function(done) { + utils.expBackoff({attempt: 3}, function(err, timeout) { + assert.ok(!err); + assert.ok(80 <= timeout && timeout <= 160); + done(); + }); + }); + it("should have backoff in [160, 320]ms, attempt = 4", function(done) { + utils.expBackoff({attempt: 4}, function(err, timeout) { + assert.ok(!err); + assert.ok(160 <= timeout && timeout <= 320); + done(); + }); + }); + it("should have backoff in [320, 640]ms, attempt = 5", function(done) { + utils.expBackoff({attempt: 5}, function(err, timeout) { + assert.ok(!err); + assert.ok(320 <= timeout && timeout <= 640); + done(); + }); + }); + it("should have backoff of 40ms, attempt = 2, rand = 0", function(done) { + utils.expBackoff({attempt: 2, rand: 0}, function(err, timeout) { + assert.ok(!err); + assert.strictEqual(40, timeout); + done(); + }); + }); + it("should have backoff of 80ms, attempt = 2, rand = 1", function(done) { + utils.expBackoff({attempt: 2, rand: 1}, function(err, timeout) { + assert.ok(!err); + assert.strictEqual(80, timeout); + done(); + }); + }); + it("should have backoff of 80ms, attempt = 2, rand = 1 - no done callback", function(done) { + utils.expBackoff({attempt: 2, rand: 1}); + setTimeout(done, 80); + }); + // TODO: this test is takes 2 minutes, rest of the tests take 7s combined... + // it("should have maximum backoff of 2m (slow running test)", function(done) { + // this.timeout(1000 * 60 * 2 + 500); + // utils.expBackoff({attempt: 100, rand: 0}, function(err, timeout) { + // assert.strictEqual(120000, timeout); + // done(); + // }); + // }); + }); }); \ No newline at end of file diff --git a/utils.js b/utils.js index 5360625..f2e9e37 100644 --- a/utils.js +++ b/utils.js @@ -130,6 +130,11 @@ utils.chain = function(tasks, callback) { /** * Asynchronous while loop. + * + * @param {function} [condition] - A function returning a boolean, the loop condition. + * @param {function} [body] - A function, the loop body. + * @param {function} [callback] - Final callback. + * @static */ utils.whilst = function (condition, body, callback) { condition = condition || function() { return false; }; @@ -139,6 +144,8 @@ utils.whilst = function (condition, body, callback) { var wrappedCallback = function(err) { if (err) { callback(err); + // callback.apply(callback, arguments); + // TODO: would this ever work or be useful? callback.apply(callback, arguments); } else { utils.whilst(condition, body, callback); @@ -153,4 +160,41 @@ utils.whilst = function (condition, body, callback) { } }; +/** + * Waits using exponential backoff. + * + * @param {object} [opts] - Settings for this function. Expected keys: attempt, rand. + * @param {function} [callback] - A callback function: function(err, timeout). + */ +utils.expBackoff = function(opts, callback) { + callback = callback || function(){}; + if (!opts || typeof opts !== "object") { + callback(new Error("Must send opts as an object.")); + } + else if (opts && !opts.hasOwnProperty("attempt")) { + callback(new Error("Must set opts.attempt.")); + } + else { + + var min = 10; + var max = 1000 * 60 * 2; // TODO: is 2 minutes a reasonable max timeout? + + var rand = Math.random(); + if (opts.hasOwnProperty("rand")) { + rand = opts.rand; + } + rand++; + + var timeout = Math.round(rand * min * Math.pow(2, opts.attempt)); + + timeout = Math.min(timeout, max); + setTimeout( + function() { + callback(null, timeout); + }, + timeout + ); + } +}; + module.exports = utils; \ No newline at end of file From 76e3a43c9bc1f61bf934c1a72d35941e1c563dcf Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Thu, 8 Oct 2015 20:13:27 -0700 Subject: [PATCH 04/46] Add retry on error logic, via exponential backoff. Specify the max number of retries when configuring a logger. The maximum backoff (time between retries) is capped at 2 minutes. --- splunklogger.js | 84 ++++++-- test/test_config.js | 66 +++++++ test/test_send.js | 467 ++++++++++++++++++++++++++++++++++++++++++++ utils.js | 2 - 4 files changed, 604 insertions(+), 15 deletions(-) diff --git a/splunklogger.js b/splunklogger.js index 3ee674b..3cb67fe 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -57,6 +57,7 @@ function _err(err, context) { * @param {string} config.token - Splunk HTTP Event Collector token, required. * @param {string} [config.name=splunk-javascript-logging/0.8.0] - Name for this logger. * @param {string} [config.host=localhost] - Hostname or IP address of Splunk server. + * @param {string} [config.maxRetries=0] - How many times to retry when HTTP POST to Splunk fails. * @param {string} [config.path=/services/collector/event/1.0] - URL path to send data to on the Splunk server. * @param {string} [config.protocol=https] - Protocol used to communicate with the Splunk server, http or https. * @param {number} [config.port=8088] - HTTP Event Collector port on the Splunk server. @@ -97,7 +98,8 @@ var defaultConfig = { protocol: "https", port: 8088, level: SplunkLogger.prototype.levels.INFO, - autoFlush: true + autoFlush: true, + maxRetries: 0 }; var defaultRequestOptions = { @@ -169,6 +171,15 @@ SplunkLogger.prototype._initializeConfig = function(config) { ret.protocol = config.protocol || ret.protocol || defaultConfig.protocol; ret.level = config.level || ret.level || defaultConfig.level; + ret.maxRetries = config.maxRetries || ret.maxRetries || defaultConfig.maxRetries; + ret.maxRetries = parseInt(ret.maxRetries, 10); + if (isNaN(ret.maxRetries)) { + throw new Error("Max retries must be a number, found: " + ret.maxRetries); + } + else if (ret.maxRetries < 0) { + throw new Error("Max retries must be a positive number, found: " + ret.maxRetries); + } + // Start with the default autoFlush value ret.autoFlush = defaultConfig.autoFlush; // Then check this.config.autoFlush @@ -362,6 +373,17 @@ SplunkLogger.prototype.use = function(middleware) { /** * Makes an HTTP POST to the configured server. * + * @param requestOptions + * @param {function} callback = A callback function: function(err, response, body). + * @private + */ +SplunkLogger.prototype._post = function(requestOptions, callback) { + request.post(requestOptions, callback); +}; + +/** + * Sends events to Splunk, optionally with retries on non-Splunk errors. + * * @param context * @param {function} callback - A callback function: function(err, response, body) * @private @@ -383,20 +405,56 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { // since json is set to true. context.requestOptions.headers["content-type"] = "application/x-www-form-urlencoded"; } + var that = this; - request.post(context.requestOptions, function(err, resp, body) { - // Call error() if error, or body isn't success - var error = err; - // Assume this is a non-success response from Splunk, build the error accordingly - if (!err && body && body.code.toString() !== "0") { - error = new Error(body.text); - error.code = body.code; - } - if (error) { - that.error(error, context); + + var splunkError = null; // Errors returned by Splunk + var requestError = null; // Any non-Splunk errors + + // References so we don't have to deal with callback parameters + var _response = null; + var _body = null; + + var numRetries = 0; + + utils.whilst( + function() { + // Continue if we can (re)try + return numRetries++ <= context.config.maxRetries; + }, + function(done) { + that._post(context.requestOptions, function(err, resp, body) { + // Store the latest error, response & body + splunkError = null; + requestError = err; + _response = resp; + _body = body; + + // Try to parse an error response from Splunk + if (!requestError && body && body.code.toString() !== "0") { + splunkError = new Error(body.text); + splunkError.code = body.code; + } + + // Request error (non-200), retry if numRetries hasn't exceeded the limit + if (requestError && numRetries <= context.config.maxRetries) { + utils.expBackoff({attempt: numRetries}, done); + } + else { + // Stop iterating + done(true); + } + }); + }, + function() { + // Call error() for a request error or Splunk error + if (requestError || splunkError) { + that.error(requestError || splunkError, context); + } + + callback(requestError, _response, _body); } - callback(err, resp, body); - }); + ); }; /** diff --git a/test/test_config.js b/test/test_config.js index 12ac35b..3941f4d 100644 --- a/test/test_config.js +++ b/test/test_config.js @@ -141,6 +141,7 @@ describe("SplunkLogger", function() { assert.strictEqual("info", logger.config.level); assert.strictEqual(logger.levels.INFO, logger.config.level); assert.strictEqual(8088, logger.config.port); + assert.strictEqual(0, logger.config.maxRetries); }); it("should set remaining defaults when setting config with token, autoFlush off, & level", function() { var config = { @@ -159,6 +160,7 @@ describe("SplunkLogger", function() { assert.strictEqual("important", logger.config.level); assert.strictEqual(false, logger.config.autoFlush); assert.strictEqual(8088, logger.config.port); + assert.strictEqual(0, logger.config.maxRetries); }); it("should set non-default boolean config values", function() { var config = { @@ -176,6 +178,7 @@ describe("SplunkLogger", function() { assert.strictEqual("http", logger.config.protocol); assert.strictEqual("info", logger.config.level); assert.strictEqual(8088, logger.config.port); + assert.strictEqual(0, logger.config.maxRetries); }); it("should set non-default path", function() { var config = { @@ -192,6 +195,7 @@ describe("SplunkLogger", function() { assert.strictEqual("https", logger.config.protocol); assert.strictEqual("info", logger.config.level); assert.strictEqual(8088, logger.config.port); + assert.strictEqual(0, logger.config.maxRetries); }); it("should set non-default port", function() { var config = { @@ -208,6 +212,7 @@ describe("SplunkLogger", function() { assert.strictEqual("https", logger.config.protocol); assert.strictEqual("info", logger.config.level); assert.strictEqual(config.port, logger.config.port); + assert.strictEqual(0, logger.config.maxRetries); }); it("should set protocol, host, port & path from url property", function() { var config = { @@ -224,6 +229,7 @@ describe("SplunkLogger", function() { assert.strictEqual("http", logger.config.protocol); assert.strictEqual("info", logger.config.level); assert.strictEqual(9088, logger.config.port); + assert.strictEqual(0, logger.config.maxRetries); }); it("should set protocol from url property", function() { var config = { @@ -240,6 +246,7 @@ describe("SplunkLogger", function() { assert.strictEqual("http", logger.config.protocol); assert.strictEqual("info", logger.config.level); assert.strictEqual(8088, logger.config.port); + assert.strictEqual(0, logger.config.maxRetries); }); it("should set everything but path from url property", function() { var config = { @@ -256,6 +263,7 @@ describe("SplunkLogger", function() { assert.strictEqual("http", logger.config.protocol); assert.strictEqual("info", logger.config.level); assert.strictEqual(9088, logger.config.port); + assert.strictEqual(0, logger.config.maxRetries); }); it("should set everything but path from url property with trailing slash", function() { var config = { @@ -272,6 +280,7 @@ describe("SplunkLogger", function() { assert.strictEqual("http", logger.config.protocol); assert.strictEqual("info", logger.config.level); assert.strictEqual(9088, logger.config.port); + assert.strictEqual(0, logger.config.maxRetries); }); it("should set host from url property with host only", function() { var config = { @@ -289,6 +298,25 @@ describe("SplunkLogger", function() { assert.strictEqual("info", logger.config.level); assert.strictEqual(true, logger.config.autoFlush); assert.strictEqual(8088, logger.config.port); + assert.strictEqual(0, logger.config.maxRetries); + }); + it("should set maxRetries", function() { + var config = { + token: "a-token-goes-here-usually", + maxRetries: 10 + }; + var logger = new SplunkLogger(config); + + assert.ok(logger); + assert.strictEqual(config.token, logger.config.token); + assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("localhost", logger.config.host); + assert.strictEqual("/services/collector/event/1.0", logger.config.path); + assert.strictEqual("https", logger.config.protocol); + assert.strictEqual("info", logger.config.level); + assert.strictEqual(true, logger.config.autoFlush); + assert.strictEqual(8088, logger.config.port); + assert.strictEqual(10, logger.config.maxRetries); }); }); describe("_initializeConfig", function() { @@ -338,6 +366,36 @@ describe("SplunkLogger", function() { assert.strictEqual(err.message, "Config token must be a string."); } }); + it("should error with NaN maxRetries", function() { + var config = { + token: "a-token-goes-here-usually", + maxRetries: "this isn't a number" + }; + + try { + SplunkLogger.prototype._initializeConfig(config); + assert.fail(false, "Expected an error."); + } + catch (err) { + assert.ok(err); + assert.strictEqual(err.message, "Max retries must be a number, found: NaN"); + } + }); + it("should error with negative maxRetries", function() { + var config = { + token: "a-token-goes-here-usually", + maxRetries: -1 + }; + + try { + SplunkLogger.prototype._initializeConfig(config); + assert.fail(false, "Expected an error."); + } + catch (err) { + assert.ok(err); + assert.strictEqual(err.message, "Max retries must be a positive number, found: -1"); + } + }); it("should error with NaN port", function() { var config = { token: "a-token-goes-here-usually", @@ -390,6 +448,7 @@ describe("SplunkLogger", function() { assert.strictEqual("https", loggerConfig.protocol); assert.strictEqual("info", loggerConfig.level); assert.strictEqual(8088, loggerConfig.port); + assert.strictEqual(0, loggerConfig.maxRetries); }); it("should set non-default boolean config values", function() { var config = { @@ -406,6 +465,7 @@ describe("SplunkLogger", function() { assert.strictEqual("http", loggerConfig.protocol); assert.strictEqual("info", loggerConfig.level); assert.strictEqual(8088, loggerConfig.port); + assert.strictEqual(0, loggerConfig.maxRetries); }); it("should set non-default path", function() { var config = { @@ -422,6 +482,7 @@ describe("SplunkLogger", function() { assert.strictEqual("https", loggerConfig.protocol); assert.strictEqual("info", loggerConfig.level); assert.strictEqual(8088, loggerConfig.port); + assert.strictEqual(0, loggerConfig.maxRetries); }); it("should set non-default port", function() { var config = { @@ -438,6 +499,7 @@ describe("SplunkLogger", function() { assert.strictEqual("https", loggerConfig.protocol); assert.strictEqual("info", loggerConfig.level); assert.strictEqual(config.port, loggerConfig.port); + assert.strictEqual(0, loggerConfig.maxRetries); }); it("should set protocol, host, port & path from url property", function() { var config = { @@ -454,6 +516,7 @@ describe("SplunkLogger", function() { assert.strictEqual("http", loggerConfig.protocol); assert.strictEqual("info", loggerConfig.level); assert.strictEqual(9088, loggerConfig.port); + assert.strictEqual(0, loggerConfig.maxRetries); }); it("should set protocol from url property", function() { var config = { @@ -470,6 +533,7 @@ describe("SplunkLogger", function() { assert.strictEqual("http", loggerConfig.protocol); assert.strictEqual("info", loggerConfig.level); assert.strictEqual(8088, loggerConfig.port); + assert.strictEqual(0, loggerConfig.maxRetries); }); it("should set host from url property with host only", function() { var config = { @@ -486,6 +550,7 @@ describe("SplunkLogger", function() { assert.strictEqual("https", loggerConfig.protocol); assert.strictEqual("info", loggerConfig.level); assert.strictEqual(8088, loggerConfig.port); + assert.strictEqual(0, loggerConfig.maxRetries); }); it("should ignore prototype values", function() { Object.prototype.something = "ignore"; @@ -504,6 +569,7 @@ describe("SplunkLogger", function() { assert.strictEqual("https", loggerConfig.protocol); assert.strictEqual("info", loggerConfig.level); assert.strictEqual(8088, loggerConfig.port); + assert.strictEqual(0, loggerConfig.maxRetries); }); }); describe("_initializeRequestOptions", function() { diff --git a/test/test_send.js b/test/test_send.js index dc66f46..369f3e2 100644 --- a/test/test_send.js +++ b/test/test_send.js @@ -485,6 +485,42 @@ describe("SplunkLogger send", function() { done(); }); }); + it("should fail on wrong Splunk server", function(done) { + var config = { + token: configurationFile.token, + url: "https://something-so-invalid-that-it-should-never-exist.xyz:12345/junk" + }; + + var logger = new SplunkLogger(config); + + var data = "something"; + var context = { + config: config, + message: data + }; + + var run = false; + + logger.error = function(err, errContext) { + // TODO: the resp.statusCode is what we want, but can't access here! + run = true; + assert.ok(err); + assert.strictEqual(err.message, "getaddrinfo ENOTFOUND"); + assert.strictEqual(err.code, "ENOTFOUND"); + assert.ok(errContext); + assert.strictEqual(errContext, context); + }; + + logger.send(context, function(err, resp, body) { + assert.ok(err); + assert.ok(run); + assert.strictEqual(err.message, "getaddrinfo ENOTFOUND"); + assert.strictEqual(err.code, "ENOTFOUND"); + assert.ok(!resp); + assert.ok(!body); + done(); + }); + }); it("should succeed with valid token, using non-default url", function(done) { var config = { token: configurationFile.token, @@ -1136,4 +1172,435 @@ describe("SplunkLogger send", function() { assert.strictEqual(middlewareCount, 2); }); }); + describe("using retry", function() { + it("should retry exactly 0 times (send once only)", function(done) { + var config = { + token: configurationFile.token, + maxRetries: 0 + }; + var logger = new SplunkLogger(config); + + var retryCount = 0; + + // Wrap the _post so we can verify retries + var post = logger._post; + logger._post = function(requestOptions, callback) { + retryCount++; + if (retryCount === config.maxRetries + 1) { + post(requestOptions, callback); + } + else { + callback(new Error(), {body: invalidTokenBody}, invalidTokenBody); + } + }; + + var payload = { + message: "something" + }; + logger.send(payload, function(err, resp, body) { + assert.ok(!err); + assert.ok(resp); + assert.ok(body); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + assert.strictEqual(retryCount, config.maxRetries + 1); + done(); + }); + }); + it("should retry exactly once", function(done) { + var config = { + token: configurationFile.token, + maxRetries: 1 + }; + var logger = new SplunkLogger(config); + + var retryCount = 0; + + // Wrap the _post so we can verify retries + var post = logger._post; + logger._post = function(requestOptions, callback) { + retryCount++; + if (retryCount === config.maxRetries + 1) { + post(requestOptions, callback); + } + else { + callback(new Error(), {body: invalidTokenBody}, invalidTokenBody); + } + }; + + var payload = { + message: "something" + }; + logger.send(payload, function(err, resp, body) { + assert.ok(!err); + assert.ok(resp); + assert.ok(body); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + assert.strictEqual(retryCount, config.maxRetries + 1); + done(); + }); + }); + it("should retry exactly twice", function(done) { + var config = { + token: configurationFile.token, + maxRetries: 2 + }; + var logger = new SplunkLogger(config); + + var retryCount = 0; + + // Wrap the _post so we can verify retries + var post = logger._post; + logger._post = function(requestOptions, callback) { + retryCount++; + if (retryCount === config.maxRetries + 1) { + post(requestOptions, callback); + } + else { + callback(new Error(), {body: invalidTokenBody}, invalidTokenBody); + } + }; + + var payload = { + message: "something" + }; + logger.send(payload, function(err, resp, body) { + assert.ok(!err); + assert.ok(resp); + assert.ok(body); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + assert.strictEqual(retryCount, config.maxRetries + 1); + done(); + }); + }); + it("should retry exactly 5 times", function(done) { + var config = { + token: configurationFile.token, + maxRetries: 5 + }; + var logger = new SplunkLogger(config); + + var retryCount = 0; + + // Wrap the _post so we can verify retries + var post = logger._post; + logger._post = function(requestOptions, callback) { + retryCount++; + if (retryCount === config.maxRetries + 1) { + post(requestOptions, callback); + } + else { + callback(new Error(), {body: invalidTokenBody}, invalidTokenBody); + } + }; + + var payload = { + message: "something" + }; + logger.send(payload, function(err, resp, body) { + assert.ok(!err); + assert.ok(resp); + assert.ok(body); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + assert.strictEqual(retryCount, config.maxRetries + 1); + done(); + }); + }); + it("should not retry on initial success when maxRetries=1", function(done) { + var config = { + token: configurationFile.token, + maxRetries: 1 + }; + var logger = new SplunkLogger(config); + + var retryCount = 0; + + // Wrap the _post so we can verify retries + var post = logger._post; + logger._post = function(requestOptions, callback) { + retryCount++; + if (retryCount === 1) { + post(requestOptions, callback); + } + else { + callback(new Error(), {body: invalidTokenBody}, invalidTokenBody); + } + }; + + var payload = { + message: "something" + }; + logger.send(payload, function(err, resp, body) { + assert.ok(!err); + assert.ok(resp); + assert.ok(body); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + assert.strictEqual(retryCount, 1); + done(); + }); + }); + it("should retry once when maxRetries=10", function(done) { + var config = { + token: configurationFile.token, + maxRetries: 10 + }; + var logger = new SplunkLogger(config); + + var retryCount = 0; + + // Wrap the _post so we can verify retries + var post = logger._post; + logger._post = function(requestOptions, callback) { + retryCount++; + if (retryCount === 2) { + post(requestOptions, callback); + } + else { + callback(new Error(), {body: invalidTokenBody}, invalidTokenBody); + } + }; + + var payload = { + message: "something" + }; + logger.send(payload, function(err, resp, body) { + assert.ok(!err); + assert.ok(resp); + assert.ok(body); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + assert.strictEqual(retryCount, 2); + done(); + }); + }); + it("should retry on request error when maxRetries=0", function(done) { + var config = { + token: configurationFile.token, + maxRetries: 0, + host: "bad-hostname.invalid" + }; + var logger = new SplunkLogger(config); + + var retryCount = 0; + + // Wrap the _post so we can verify retries + var post = logger._post; + logger._post = function(requestOptions, callback) { + retryCount++; + post(requestOptions, callback); + }; + + var run = false; + logger.error = function(err, context) { + assert.ok(err); + assert.ok(context); + run = true; + }; + + var payload = { + message: "something" + }; + logger.send(payload, function(err, resp, body) { + assert.ok(err); + assert.ok(!resp); + assert.ok(!body); + + assert.strictEqual(config.maxRetries + 1, retryCount); + assert.ok(run); + done(); + }); + }); + it("should retry on request error when maxRetries=1", function(done) { + var config = { + token: configurationFile.token, + maxRetries: 1, + host: "bad-hostname.invalid" + }; + var logger = new SplunkLogger(config); + + var retryCount = 0; + + // Wrap the _post so we can verify retries + var post = logger._post; + logger._post = function(requestOptions, callback) { + retryCount++; + post(requestOptions, callback); + }; + + var run = false; + logger.error = function(err, context) { + assert.ok(err); + assert.ok(context); + run = true; + }; + + var payload = { + message: "something" + }; + logger.send(payload, function(err, resp, body) { + assert.ok(err); + assert.ok(!resp); + assert.ok(!body); + + assert.strictEqual(config.maxRetries + 1, retryCount); + assert.ok(run); + done(); + }); + }); + it("should retry on request error when maxRetries=5", function(done) { + var config = { + token: configurationFile.token, + maxRetries: 5, + host: "bad-hostname.invalid" + }; + var logger = new SplunkLogger(config); + + var retryCount = 0; + + // Wrap the _post so we can verify retries + var post = logger._post; + logger._post = function(requestOptions, callback) { + retryCount++; + post(requestOptions, callback); + }; + + var run = false; + logger.error = function(err, context) { + assert.ok(err); + assert.ok(context); + run = true; + }; + + var payload = { + message: "something" + }; + logger.send(payload, function(err, resp, body) { + assert.ok(err); + assert.ok(!resp); + assert.ok(!body); + + assert.strictEqual(config.maxRetries + 1, retryCount); + assert.ok(run); + done(); + }); + }); + it("should not retry on Splunk error when maxRetries=0", function(done) { + var config = { + token: "invalid-token", + maxRetries: 0 + }; + var logger = new SplunkLogger(config); + + var retryCount = 0; + + // Wrap the _post so we can verify retries + var post = logger._post; + logger._post = function(requestOptions, callback) { + retryCount++; + post(requestOptions, callback); + }; + + var run = false; + logger.error = function(err, context) { + assert.ok(err); + assert.ok(context); + run = true; + }; + + var payload = { + message: "something" + }; + logger.send(payload, function(err, resp, body) { + assert.ok(!err); + assert.ok(resp); + assert.ok(body); + assert.strictEqual(invalidTokenBody.code, body.code); + assert.strictEqual(invalidTokenBody.text, body.text); + + assert.strictEqual(1, retryCount); + assert.ok(run); + done(); + }); + }); + it("should not retry on Splunk error when maxRetries=1", function(done) { + var config = { + token: "invalid-token", + maxRetries: 1 + }; + var logger = new SplunkLogger(config); + + var retryCount = 0; + + // Wrap the _post so we can verify retries + var post = logger._post; + logger._post = function(requestOptions, callback) { + retryCount++; + post(requestOptions, callback); + }; + + var run = false; + logger.error = function(err, context) { + assert.ok(err); + assert.ok(context); + run = true; + }; + + var payload = { + message: "something" + }; + logger.send(payload, function(err, resp, body) { + assert.ok(!err); + assert.ok(resp); + assert.ok(body); + assert.strictEqual(invalidTokenBody.code, body.code); + assert.strictEqual(invalidTokenBody.text, body.text); + + assert.strictEqual(1, retryCount); + assert.ok(run); + done(); + }); + }); + it("should not retry on Splunk error when maxRetries=5", function(done) { + var config = { + token: "invalid-token", + maxRetries: 5 + }; + var logger = new SplunkLogger(config); + + var retryCount = 0; + + // Wrap the _post so we can verify retries + var post = logger._post; + logger._post = function(requestOptions, callback) { + retryCount++; + post(requestOptions, callback); + }; + + var run = false; + logger.error = function(err, context) { + assert.ok(err); + assert.ok(context); + run = true; + }; + + var payload = { + message: "something" + }; + logger.send(payload, function(err, resp, body) { + assert.ok(!err); + assert.ok(resp); + assert.ok(body); + assert.strictEqual(invalidTokenBody.code, body.code); + assert.strictEqual(invalidTokenBody.text, body.text); + + assert.strictEqual(1, retryCount); + assert.ok(run); + done(); + }); + }); + }); }); diff --git a/utils.js b/utils.js index f2e9e37..8db9eba 100644 --- a/utils.js +++ b/utils.js @@ -144,8 +144,6 @@ utils.whilst = function (condition, body, callback) { var wrappedCallback = function(err) { if (err) { callback(err); - // callback.apply(callback, arguments); - // TODO: would this ever work or be useful? callback.apply(callback, arguments); } else { utils.whilst(condition, body, callback); From 2042e053f7830d8a63eed658da99390f3a919bcc Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Thu, 8 Oct 2015 20:31:07 -0700 Subject: [PATCH 05/46] Add retry example & update readme --- README.md | 1 + examples/retry.js | 73 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) create mode 100644 examples/retry.js diff --git a/README.md b/README.md index c4eba7c..5ddfb64 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ See the `examples` folder for more examples: * `basic.js`: shows how to configure a logger and send a log message to Splunk. * `batching.js`: shows how to queue log messages, and send them in batches. * `middleware.js`: shows how to add an express-like middleware function to be called before sending log messages to Splunk. +* `retry.js`: shows how to configure retries on errors. ### Basic example diff --git a/examples/retry.js b/examples/retry.js new file mode 100644 index 0000000..d678bbf --- /dev/null +++ b/examples/retry.js @@ -0,0 +1,73 @@ +/* + * Copyright 2015 Splunk, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"): you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +/** + * This example shows how to configure retries with SplunkLogger. + */ + +// Change to require("splunk-logging").Logger; +var SplunkLogger = require("../index").Logger; + +/** + * Only the token property is required. + * Defaults are listed explicitly. + * + * Alternatively, specify config.url like so: + * + * "https://localhost:8088/services/collector/event/1.0" + */ +var config = { + token: "your-token-here", + host: "localhost", + path: "/services/collector/event/1.0", + protocol: "https", + port: 8088, + level: "info", + autoFlush: true, + maxRetries: 5 +}; + +// Create a new logger +var Logger = new SplunkLogger(config); + +Logger.error = function(err, context) { + // Handle errors here + console.log("error", err, "context", context); +}; + +// Define the payload to send to Splunk's Event Collector +var payload = { + // Message can be anything, doesn't have to be an object + message: { + temperature: "70F", + chickenCount: 500 + }, + // Metadata is optional + metadata: { + source: "chicken coop", + sourcetype: "httpevent", + index: "main", + host: "farm.local", + }, + // Severity is also optional + severity: "info" +}; + +console.log("Sending payload", payload); +Logger.send(payload, function(err, resp, body) { + // If successful, body will be { text: 'Success', code: 0 } + console.log("Response from Splunk", body); +}); \ No newline at end of file From 3f4f3bd4e86aceaf529ac14cd1b8c87c8d5cb305 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Fri, 9 Oct 2015 11:09:48 -0700 Subject: [PATCH 06/46] Update examples --- examples/basic.js | 3 ++- examples/batching.js | 3 ++- examples/middleware.js | 3 ++- examples/retry.js | 14 +++++--------- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/examples/basic.js b/examples/basic.js index 668c677..c0656b0 100644 --- a/examples/basic.js +++ b/examples/basic.js @@ -36,7 +36,8 @@ var config = { protocol: "https", port: 8088, level: "info", - autoFlush: true + autoFlush: true, + maxRetries: 0 }; // Create a new logger diff --git a/examples/batching.js b/examples/batching.js index b5933ae..2aff407 100644 --- a/examples/batching.js +++ b/examples/batching.js @@ -41,7 +41,8 @@ var config = { protocol: "https", port: 8088, level: "info", - autoFlush: false + autoFlush: false, + maxRetries: 0 }; // Create a new logger diff --git a/examples/middleware.js b/examples/middleware.js index 5d3b8c1..3745493 100644 --- a/examples/middleware.js +++ b/examples/middleware.js @@ -36,7 +36,8 @@ var config = { protocol: "https", port: 8088, level: "info", - autoFlush: true + autoFlush: true, + maxRetries: 0 }; // Create a new logger diff --git a/examples/retry.js b/examples/retry.js index d678bbf..72d053b 100644 --- a/examples/retry.js +++ b/examples/retry.js @@ -23,20 +23,16 @@ var SplunkLogger = require("../index").Logger; /** * Only the token property is required. - * Defaults are listed explicitly. * - * Alternatively, specify config.url like so: - * - * "https://localhost:8088/services/collector/event/1.0" + * Here we've set maxRetries to 5, + * If there are any connection errors the request + * to Splunk will be retried up to 5 times. + * The default is 0. */ var config = { token: "your-token-here", - host: "localhost", - path: "/services/collector/event/1.0", - protocol: "https", - port: 8088, + url: "https://localhost:8088", level: "info", - autoFlush: true, maxRetries: 5 }; From d598fb0e4dc48a6894a3cf43a04232462e4a2bc9 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Mon, 19 Oct 2015 19:12:10 -0700 Subject: [PATCH 07/46] Add support for automatic interval batching. --- splunklogger.js | 132 ++++++++++++-- test/test_config.js | 112 ++++++++++++ test/test_send.js | 411 +++++++++++++++++++++++++++++++++++++++++++- utils.js | 12 +- 4 files changed, 648 insertions(+), 19 deletions(-) diff --git a/splunklogger.js b/splunklogger.js index 3cb67fe..2a4e60b 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -42,7 +42,7 @@ function _err(err, context) { * token: "your-token-here", * name: "my application", * host: "splunk.local", - * autoFlush: false + * autoFlush: true * }; * * var logger = new SplunkLogger(config); @@ -67,14 +67,32 @@ function _err(err, context) { * @param {string} [config.level=info] - Logging level to use, will show up as the severity field of an event, see * [SplunkLogger.levels]{@link SplunkLogger#levels} for common levels. * @param {bool} [config.autoFlush=true] - Send events immediately or not. + * @param {bool} [config.batchInterval=0] - If config.autoFlush === true, automatically flush events after this many milliseconds. + * When set to a non-positive value, events will be sent one by one. * @constructor * @throws Will throw an error if the config parameter is malformed. */ var SplunkLogger = function(config) { + this._timerID = null; + this._timerDuration = 0; this.config = this._initializeConfig(config); this.middlewares = []; this.contextQueue = []; this.error = _err; + + this._enableTimer = utils.bind(this, this._enableTimer); + this._disableTimer = utils.bind(this, this._disableTimer); + this._initializeConfig = utils.bind(this, this._initializeConfig); + this._initializeRequestOptions = utils.bind(this, this._initializeRequestOptions); + this._initializeMessage = utils.bind(this, this._initializeMessage); + this._initializeMetadata = utils.bind(this, this._initializeMetadata); + this._initializeContext = utils.bind(this, this._initializeContext); + this._makeBody = utils.bind(this, this._makeBody); + this.use = utils.bind(this, this.use); + this._post = utils.bind(this, this._post); + this._sendEvents = utils.bind(this, this._sendEvents); + this.send = utils.bind(this, this.send); + this.flush = utils.bind(this, this.flush); }; /** @@ -99,7 +117,8 @@ var defaultConfig = { port: 8088, level: SplunkLogger.prototype.levels.INFO, autoFlush: true, - maxRetries: 0 + maxRetries: 0, + batchInterval: 0 }; var defaultRequestOptions = { @@ -108,6 +127,51 @@ var defaultRequestOptions = { url: defaultConfig.protocol + "://" + defaultConfig.host + ":" + defaultConfig.port + defaultConfig.path }; +/** + * Disables the interval timer set by this._enableTimer(). + * + * param {Number} interval - The batch interval. + * @private + */ +SplunkLogger.prototype._disableTimer = function() { + if (this._timerID) { + clearInterval(this._timerID); + this._timerDuration = 0; + this._timerID = null; + } +}; + +/** + * Configures an interval timer to flush any events in + * this.contextQueue at the specified interval. + * + * param {Number} interval - The batch interval. + * @private + */ +SplunkLogger.prototype._enableTimer = function(interval) { + // Only enable the timer if possible + if (typeof interval !== "number") { + throw new Error("Batch interval must be a number, found: " + interval); + } + else if (typeof interval === "number" && interval > 0) { + if (this._timerID) { + this._disableTimer(); + } + + // If batch interval is changed, update the config property + if (this.config && this.config.hasOwnProperty("batchInterval")) { + this.config.batchInterval = interval; + } + + this._timerDuration = interval; + + var that = this; + this._timerID = setInterval(function() { + that.flush(); + }, interval); + } +}; + /** * Sets up the config with any default properties, and/or * config properties set on this.config. @@ -184,12 +248,42 @@ SplunkLogger.prototype._initializeConfig = function(config) { ret.autoFlush = defaultConfig.autoFlush; // Then check this.config.autoFlush if (this.hasOwnProperty("config") && this.config.hasOwnProperty("autoFlush")) { - ret.autoFlush = ret.autoFlush; + ret.autoFlush = this.config.autoFlush; } // Then check the config.autoFlush, the function argument if (config.hasOwnProperty("autoFlush")) { ret.autoFlush = config.autoFlush; } + // Convert autoFlush value to boolean + ret.autoFlush = !!ret.autoFlush; + + // Start with the default batchInterval value + ret.batchInterval = defaultConfig.batchInterval; + if (this.hasOwnProperty("config") && this.config.hasOwnProperty("batchInterval")) { + ret.batchInterval = this.config.batchInterval; + } + if (config.hasOwnProperty("batchInterval")) { + ret.batchInterval = config.batchInterval; + } + ret.batchInterval = parseInt(ret.batchInterval, 10); + if (isNaN(ret.batchInterval)) { + throw new Error("Batch interval must be a number, found: " + ret.batchInterval); + } + else if (ret.batchInterval < 0) { + throw new Error("Batch interval must be a positive number, found: " + ret.batchInterval); + } + + var startTimer = !this._timerID && ret.autoFlush && ret.batchInterval > 0; + var changeTimer = this._timerID && ret.autoFlush && this._timerDuration !== ret.batchInterval && ret.batchInterval > 0; + + // Upsert the timer + if (startTimer || changeTimer) { + this._enableTimer(ret.batchInterval); + } + // Disable timer + else if (this._timerID && (this._timerDuration < 0 || !ret.autoFlush)) { + this._disableTimer(); + } if (!config.hasOwnProperty("port")) { ret.port = ret.port || defaultConfig.port; @@ -510,17 +604,19 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { * @public */ SplunkLogger.prototype.send = function (context, callback) { + + console.log(this.config.batchInterval); + console.log(this.config.autoFlush); + callback = callback || function(){}; context = this._initializeContext(context); this.contextQueue.push(context); - if (context.config.autoFlush) { + // Only flush if using manual batching + if (context.config.autoFlush && !this._timerID) { this.flush(callback); } - else { - callback(); - } }; /** @@ -537,14 +633,8 @@ SplunkLogger.prototype.flush = function (callback) { var context = {}; - // Use the batching setting from this.config - if (this.config.autoFlush) { - // TODO: handle case of multiple events with autoFlush off, flushing fast - // Just take the oldest event in the queue - context = this.contextQueue.pop(); - } - else { - // Empty the event queue + // Empty the queue if manual or interval batching with something to flush + if (!this.config.autoFlush || (this._timerID && this.contextQueue.length > 0)) { var queue = this.contextQueue; this.contextQueue = []; var data = ""; @@ -553,6 +643,18 @@ SplunkLogger.prototype.flush = function (callback) { } context.message = data; } + // Noop if there's nothing to flush; let non-batching requests get a no-data response from Splunk in the if block above + else if (this.contextQueue.length === 0) { + var body = { + text: "Nothing to flush." + }; + return callback(null, {body: body}, body); + } + // Just take the oldest event in the queue + else { + // TODO: handle case of multiple events with autoFlush off, flushing fast + context = this.contextQueue.pop(); + } // Initialize the context, then manually set the data context = this._initializeContext(context); diff --git a/test/test_config.js b/test/test_config.js index 3941f4d..b142599 100644 --- a/test/test_config.js +++ b/test/test_config.js @@ -162,6 +162,116 @@ describe("SplunkLogger", function() { assert.strictEqual(8088, logger.config.port); assert.strictEqual(0, logger.config.maxRetries); }); + it("should error when _enableInterval(NaN)", function() { + var config = { + token: "a-token-goes-here-usually" + }; + + try { + var logger = new SplunkLogger(config); + logger._enableTimer("not a number"); + assert.fail(!logger, "Expected an error."); + } + catch (err) { + assert.ok(err); + assert.strictEqual("Batch interval must be a number, found: not a number", err.message); + } + }); + it("should error when batchInterval=NaN", function() { + var config = { + token: "a-token-goes-here-usually", + batchInterval: "not a number", + }; + + try { + var logger = new SplunkLogger(config); + assert.fail(!logger, "Expected an error."); + } + catch (err) { + assert.ok(err); + assert.strictEqual("Batch interval must be a number, found: NaN", err.message); + } + }); + it("should error when batchInterval is negative", function() { + var config = { + token: "a-token-goes-here-usually", + batchInterval: -1, + }; + + try { + var logger = new SplunkLogger(config); + assert.fail(!logger, "Expected an error."); + } + catch (err) { + assert.ok(err); + assert.strictEqual("Batch interval must be a positive number, found: -1", err.message); + } + }); + it("should set a batch interval timer with autoFlush on, & batchInterval set", function() { + var config = { + token: "a-token-goes-here-usually", + batchInterval: 100, + autoFlush: true + }; + var logger = new SplunkLogger(config); + + assert.ok(logger); + assert.ok(logger._timerID); + + assert.strictEqual(config.token, logger.config.token); + assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("localhost", logger.config.host); + assert.strictEqual("/services/collector/event/1.0", logger.config.path); + assert.strictEqual("https", logger.config.protocol); + assert.strictEqual("info", logger.config.level); + assert.strictEqual(true, logger.config.autoFlush); + assert.strictEqual(100, logger.config.batchInterval); + assert.strictEqual(8088, logger.config.port); + assert.strictEqual(0, logger.config.maxRetries); + }); + it("should not set a batch interval timer with autoFlush on, & default batchInterval", function() { + var config = { + token: "a-token-goes-here-usually", + autoFlush: true + }; + var logger = new SplunkLogger(config); + + assert.ok(logger); + assert.ok(!logger._timerID); + + assert.strictEqual(config.token, logger.config.token); + assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("localhost", logger.config.host); + assert.strictEqual("/services/collector/event/1.0", logger.config.path); + assert.strictEqual("https", logger.config.protocol); + assert.strictEqual("info", logger.config.level); + assert.strictEqual(true, logger.config.autoFlush); + assert.strictEqual(0, logger.config.batchInterval); + assert.strictEqual(8088, logger.config.port); + assert.strictEqual(0, logger.config.maxRetries); + }); + it("should not set a batch interval timer with autoFlush off, & batchInterval set", function() { + var config = { + token: "a-token-goes-here-usually", + batchInterval: 100, + autoFlush: false + }; + var logger = new SplunkLogger(config); + + assert.ok(logger); + assert.ok(!logger._timerID); + + assert.strictEqual(config.token, logger.config.token); + assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("localhost", logger.config.host); + assert.strictEqual("/services/collector/event/1.0", logger.config.path); + assert.strictEqual("https", logger.config.protocol); + assert.strictEqual("info", logger.config.level); + assert.strictEqual(false, logger.config.autoFlush); + assert.strictEqual(100, logger.config.batchInterval); + assert.strictEqual(8088, logger.config.port); + assert.strictEqual(0, logger.config.maxRetries); + }); it("should set non-default boolean config values", function() { var config = { token: "a-token-goes-here-usually", @@ -171,6 +281,8 @@ describe("SplunkLogger", function() { var logger = new SplunkLogger(config); assert.ok(logger); + assert.ok(!logger._timerID); + assert.strictEqual(config.token, logger.config.token); assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); assert.strictEqual("localhost", logger.config.host); diff --git a/test/test_send.js b/test/test_send.js index 369f3e2..28fa584 100644 --- a/test/test_send.js +++ b/test/test_send.js @@ -164,8 +164,8 @@ describe("SplunkLogger _makedata", function() { assert.strictEqual(body.event.severity, "urgent"); }); }); -describe("SplunkLogger send", function() { - describe("using default middleware (integration tests)", function () { +describe("SplunkLogger send (integration tests)", function() { + describe("using default middleware ", function () { it("should error with bad token", function(done) { var config = { token: "token-goes-here" @@ -610,7 +610,7 @@ describe("SplunkLogger send", function() { }, 1000); }); }); - describe("without autoFlush (integration tests)", function () { + describe("without autoFlush", function () { it("should get no data response when flushing empty batch with valid token", function(done) { var config = { token: configurationFile.token, @@ -1603,4 +1603,409 @@ describe("SplunkLogger send", function() { }); }); }); + describe("using batch interval", function() { + it("should only not make a POST request if contextQueue is always empty", function(done) { + var config = { + token: configurationFile.token, + autoFlush: true, + batchInterval: 100 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context, callback) { + _post(context, function(err, resp, body) { + posts++; + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + callback(err, resp, body); + }); + }; + + setTimeout(function() { + assert.strictEqual(logger._timerDuration, 100); + assert.strictEqual(0, posts); + assert.strictEqual(0, logger.contextQueue.length); + + // Clean up the timer + logger._disableTimer(); + done(); + }, 500); + }); + it("should only make 1 POST request for 1 event", function(done) { + var config = { + token: configurationFile.token, + autoFlush: true, + batchInterval: 100 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context, callback) { + _post(context, function(err, resp, body) { + posts++; + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + callback(err, resp, body); + }); + }; + + var payload = { + message: "something" + }; + logger.send(payload); + + setTimeout(function() { + assert.strictEqual(logger._timerDuration, 100); + assert.strictEqual(1, posts); + assert.strictEqual(0, logger.contextQueue.length); + + // Clean up the timer + logger._disableTimer(); + done(); + }, 500); + }); + it("should only make 1 POST request for 2 events", function(done) { + var config = { + token: configurationFile.token, + autoFlush: true, + batchInterval: 100 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context, callback) { + _post(context, function(err, resp, body) { + posts++; + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + callback(err, resp, body); + }); + }; + + var payload = { + message: "something" + }; + var payload2 = { + message: "something 2" + }; + logger.send(payload); + logger.send(payload2); + + setTimeout(function() { + assert.strictEqual(logger._timerDuration, 100); + assert.strictEqual(1, posts); + assert.strictEqual(0, logger.contextQueue.length); + + // Clean up the timer + logger._disableTimer(); + done(); + }, 500); + }); + it("should only make 1 POST request for 5 events", function(done) { + var config = { + token: configurationFile.token, + autoFlush: true, + batchInterval: 100 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context, callback) { + _post(context, function(err, resp, body) { + posts++; + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + callback(err, resp, body); + }); + }; + + var payload = { + message: "something" + }; + logger.send(payload); + logger.send(payload); + logger.send(payload); + logger.send(payload); + logger.send(payload); + + setTimeout(function() { + assert.strictEqual(logger._timerDuration, 100); + assert.strictEqual(1, posts); + assert.strictEqual(0, logger.contextQueue.length); + + // Clean up the timer + logger._disableTimer(); + done(); + }, 500); + }); + it("should flush a stale event after enabling autoFlush and batchInterval", function(done) { + var config = { + token: configurationFile.token, + autoFlush: false + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context, callback) { + _post(context, function(err, resp, body) { + posts++; + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + callback(err, resp, body); + }); + }; + + var payload = { + message: "something" + }; + logger.send(payload); + assert.strictEqual(logger._timerDuration, 0); + + logger.config.autoFlush = true; + logger.config.batchInterval = 100; + + var payload2 = { + message: "something else" + }; + logger.send(payload2); + + setTimeout(function() { + assert.strictEqual(1, posts); + assert.strictEqual(0, logger.contextQueue.length); + + // Clean up the timer + logger._disableTimer(); + done(); + }, 500); + }); + it("should flush an event with batchInterval, then disable autoFlush for manual batching", function(done) { + var config = { + token: configurationFile.token, + autoFlush: true, + batchInterval: 100 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context, callback) { + _post(context, function(err, resp, body) { + posts++; + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + callback(err, resp, body); + }); + }; + + var payload = { + message: "something" + }; + logger.send(payload); + + var run = false; + setTimeout(function() { + logger.config.autoFlush = false; + + var payload2 = { + message: "something else" + }; + logger.send(payload2); + + assert.strictEqual(logger.contextQueue.length, 1); + logger.flush(); + run = true; + }, 150); + + setTimeout(function() { + assert.ok(run); + assert.strictEqual(2, posts); + assert.strictEqual(0, logger.contextQueue.length); + + // Clean up the timer + logger._disableTimer(); + done(); + }, 500); + }); + it("should flush an event with batchInterval, then set batchInterval=0 for manual batching", function(done) { + var config = { + token: configurationFile.token, + autoFlush: true, + batchInterval: 100 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context, callback) { + _post(context, function(err, resp, body) { + posts++; + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + callback(err, resp, body); + }); + }; + + var payload = { + message: "something" + }; + logger.send(payload); + + var run = false; + setTimeout(function() { + logger.config.batchInterval = 0; + + var payload2 = { + message: "something else" + }; + logger.send(payload2); + + assert.strictEqual(logger.contextQueue.length, 1); + logger.flush(); + run = true; + }, 150); + + setTimeout(function() { + assert.ok(run); + assert.strictEqual(2, posts); + assert.strictEqual(0, logger.contextQueue.length); + + // Clean up the timer + logger._disableTimer(); + done(); + }, 500); + }); + it("should autoFlush an event at batchInterval, then again when batchInterval has changed", function(done) { + var config = { + token: configurationFile.token, + autoFlush: true, + batchInterval: 500 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context, callback) { + _post(context, function(err, resp, body) { + posts++; + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + callback(err, resp, body); + }); + }; + + var payload = { + message: "something" + }; + logger.send(payload); + + setTimeout(function() { + assert.strictEqual(0, logger.contextQueue.length); + + if (posts === 1) { + assert.strictEqual(logger._timerDuration, 500); + logger.config.autoFlush = true; + logger.config.batchInterval = 100; + var payload2 = { + message: "something else" + }; + logger.send(payload2); + assert.strictEqual(logger._timerDuration, 100); + } + else { + assert.fail(); // TODO: + } + }, 550); + + + setTimeout(function() { + assert.strictEqual(2, posts); + assert.strictEqual(0, logger.contextQueue.length); + + // Clean up the timer + logger._disableTimer(); + done(); + }, 700); + }); + it("should flush an event with batchInterval, then set batchInterval=0 for manual batching", function(done) { + var config = { + token: configurationFile.token, + autoFlush: true, + batchInterval: 100 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context, callback) { + _post(context, function(err, resp, body) { + posts++; + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + callback(err, resp, body); + }); + }; + + var payload = { + message: "something" + }; + logger.send(payload); + + var run = false; + setTimeout(function() { + logger.config.batchInterval = 100; + + var payload2 = { + message: "something else" + }; + logger.send(payload2); + + assert.strictEqual(logger.contextQueue.length, 1); + run = true; + }, 150); + + + setTimeout(function() { + assert.ok(run); + assert.strictEqual(2, posts); + assert.strictEqual(0, logger.contextQueue.length); + + // Clean up the timer + logger._disableTimer(); + done(); + }, 300); + }); + }); }); diff --git a/utils.js b/utils.js index 8db9eba..19c392e 100644 --- a/utils.js +++ b/utils.js @@ -175,7 +175,7 @@ utils.expBackoff = function(opts, callback) { else { var min = 10; - var max = 1000 * 60 * 2; // TODO: is 2 minutes a reasonable max timeout? + var max = 1000 * 60 * 2; // 2 minutes is a reasonable max delay var rand = Math.random(); if (opts.hasOwnProperty("rand")) { @@ -195,4 +195,14 @@ utils.expBackoff = function(opts, callback) { } }; +/** + * TODO: docs + tests + * Binds a function to an instance of an object. + */ +utils.bind = function(self, fn) { + return function () { + return fn.apply(self, arguments); + }; +}; + module.exports = utils; \ No newline at end of file From 38709375f3c0907a9bfb5db40af4bf4c29d187fd Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Wed, 28 Oct 2015 14:55:21 -0700 Subject: [PATCH 08/46] Clean up batch interval implementation --- splunklogger.js | 41 +++++++++++++++++++++-------------------- test/test_send.js | 17 ++++++++++++++++- test/test_utils.js | 17 ++++++++++++++++- utils.js | 4 +++- 4 files changed, 56 insertions(+), 23 deletions(-) diff --git a/splunklogger.js b/splunklogger.js index 2a4e60b..f6ca70e 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -150,25 +150,30 @@ SplunkLogger.prototype._disableTimer = function() { */ SplunkLogger.prototype._enableTimer = function(interval) { // Only enable the timer if possible - if (typeof interval !== "number") { - throw new Error("Batch interval must be a number, found: " + interval); - } - else if (typeof interval === "number" && interval > 0) { - if (this._timerID) { - this._disableTimer(); - } - - // If batch interval is changed, update the config property - if (this.config && this.config.hasOwnProperty("batchInterval")) { - this.config.batchInterval = interval; + if (typeof interval === "number") { + if (interval <= 0) { + throw new Error("Batch interval must be a positive number, found: " + interval); } + else { + if (this._timerID) { + this._disableTimer(); + } + + // If batch interval is changed, update the config property + if (this.config && this.config.hasOwnProperty("batchInterval")) { + this.config.batchInterval = interval; + } - this._timerDuration = interval; + this._timerDuration = interval; - var that = this; - this._timerID = setInterval(function() { - that.flush(); - }, interval); + var that = this; + this._timerID = setInterval(function() { + that.flush(); + }, interval); + } + } + else { + throw new Error("Batch interval must be a number, found: " + interval); } }; @@ -604,10 +609,6 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { * @public */ SplunkLogger.prototype.send = function (context, callback) { - - console.log(this.config.batchInterval); - console.log(this.config.autoFlush); - callback = callback || function(){}; context = this._initializeContext(context); diff --git a/test/test_send.js b/test/test_send.js index 28fa584..587a5bd 100644 --- a/test/test_send.js +++ b/test/test_send.js @@ -502,7 +502,6 @@ describe("SplunkLogger send (integration tests)", function() { var run = false; logger.error = function(err, errContext) { - // TODO: the resp.statusCode is what we want, but can't access here! run = true; assert.ok(err); assert.strictEqual(err.message, "getaddrinfo ENOTFOUND"); @@ -1755,6 +1754,22 @@ describe("SplunkLogger send (integration tests)", function() { done(); }, 500); }); + it("should error when trying to set batchInterval to a negative value after logger creation", function() { + var config = { + token: configurationFile.token, + autoFlush: false + }; + var logger = new SplunkLogger(config); + + try { + logger._enableTimer(-1); + assert.ok(false, "Expected an error."); + } + catch(err) { + assert.ok(err); + assert.strictEqual(err.message, "Batch interval must be a positive number, found: -1"); + } + }); it("should flush a stale event after enabling autoFlush and batchInterval", function(done) { var config = { token: configurationFile.token, diff --git a/test/test_utils.js b/test/test_utils.js index 1fa2ef2..8a6f0ff 100644 --- a/test/test_utils.js +++ b/test/test_utils.js @@ -443,7 +443,7 @@ describe("Utils", function() { utils.expBackoff({attempt: 2, rand: 1}); setTimeout(done, 80); }); - // TODO: this test is takes 2 minutes, rest of the tests take 7s combined... + // TODO: this test is takes 2 minutes, rest of the tests take 12s combined... // it("should have maximum backoff of 2m (slow running test)", function(done) { // this.timeout(1000 * 60 * 2 + 500); // utils.expBackoff({attempt: 100, rand: 0}, function(err, timeout) { @@ -452,4 +452,19 @@ describe("Utils", function() { // }); // }); }); + describe("bind", function() { + it("should successfully bind a function", function(done) { + var f; + (function() { + f = function(a) { + this.a = a; + }; + })(); + var q = {}; + var g = utils.bind(q, f); + g(12); + assert.strictEqual(q.a, 12); + done(); + }); + }); }); \ No newline at end of file diff --git a/utils.js b/utils.js index 19c392e..e8502b7 100644 --- a/utils.js +++ b/utils.js @@ -196,8 +196,10 @@ utils.expBackoff = function(opts, callback) { }; /** - * TODO: docs + tests * Binds a function to an instance of an object. + * + * @param {object} [self] - An object to bind the fn function parameter to. + * @param {object} [fn] - A function to bind to the self argument. */ utils.bind = function(self, fn) { return function () { From fc98ed9883b91c7f065068191bc9dafe060cb294 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Fri, 30 Oct 2015 13:51:41 -0700 Subject: [PATCH 09/46] Add another config test for autoflush --- test/test_config.js | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/test/test_config.js b/test/test_config.js index b142599..f3ff3fb 100644 --- a/test/test_config.js +++ b/test/test_config.js @@ -683,6 +683,42 @@ describe("SplunkLogger", function() { assert.strictEqual(8088, loggerConfig.port); assert.strictEqual(0, loggerConfig.maxRetries); }); + it("should set autoFlush property to true after initially false", function() { + Object.prototype.something = "ignore"; + var config = { + token: "a-token-goes-here-usually", + url: "splunk.local", + autoFlush: false + }; + + var logger = new SplunkLogger(config); + var loggerConfig = logger.config; + + assert.ok(loggerConfig); + assert.ok(!loggerConfig.hasOwnProperty("something")); + assert.strictEqual(config.token, loggerConfig.token); + assert.strictEqual("splunk-javascript-logging/0.8.0", loggerConfig.name); + assert.strictEqual("splunk.local", loggerConfig.host); + assert.strictEqual("/services/collector/event/1.0", loggerConfig.path); + assert.strictEqual("https", loggerConfig.protocol); + assert.strictEqual("info", loggerConfig.level); + assert.strictEqual(8088, loggerConfig.port); + assert.strictEqual(0, loggerConfig.maxRetries); + assert.strictEqual(false, loggerConfig.autoFlush); + + config.autoFlush = true; + loggerConfig = logger._initializeConfig(config); + + assert.strictEqual(config.token, loggerConfig.token); + assert.strictEqual("splunk-javascript-logging/0.8.0", loggerConfig.name); + assert.strictEqual("splunk.local", loggerConfig.host); + assert.strictEqual("/services/collector/event/1.0", loggerConfig.path); + assert.strictEqual("https", loggerConfig.protocol); + assert.strictEqual("info", loggerConfig.level); + assert.strictEqual(8088, loggerConfig.port); + assert.strictEqual(0, loggerConfig.maxRetries); + assert.strictEqual(true, loggerConfig.autoFlush); + }); }); describe("_initializeRequestOptions", function() { it("should get defaults with no args", function() { From 566c3cb0f5c14d31966e2cd0ba657855d1ab432a Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Fri, 30 Oct 2015 13:55:49 -0700 Subject: [PATCH 10/46] Update doc datatype for batch interval --- splunklogger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/splunklogger.js b/splunklogger.js index f6ca70e..f0be6a6 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -67,7 +67,7 @@ function _err(err, context) { * @param {string} [config.level=info] - Logging level to use, will show up as the severity field of an event, see * [SplunkLogger.levels]{@link SplunkLogger#levels} for common levels. * @param {bool} [config.autoFlush=true] - Send events immediately or not. - * @param {bool} [config.batchInterval=0] - If config.autoFlush === true, automatically flush events after this many milliseconds. + * @param {number} [config.batchInterval=0] - If config.autoFlush === true, automatically flush events after this many milliseconds. * When set to a non-positive value, events will be sent one by one. * @constructor * @throws Will throw an error if the config parameter is malformed. From fa7f3b60c7e76d80f5c786e3d537c76c53d4c44f Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Tue, 3 Nov 2015 11:04:19 -0800 Subject: [PATCH 11/46] Add configuration setting for batch size --- splunklogger.js | 17 +++++--- test/test_config.js | 102 ++++++++++++++++++++++++++++---------------- 2 files changed, 76 insertions(+), 43 deletions(-) diff --git a/splunklogger.js b/splunklogger.js index f0be6a6..257808f 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -118,7 +118,8 @@ var defaultConfig = { level: SplunkLogger.prototype.levels.INFO, autoFlush: true, maxRetries: 0, - batchInterval: 0 + batchInterval: 0, + batchSize: 0 }; var defaultRequestOptions = { @@ -262,14 +263,16 @@ SplunkLogger.prototype._initializeConfig = function(config) { // Convert autoFlush value to boolean ret.autoFlush = !!ret.autoFlush; - // Start with the default batchInterval value - ret.batchInterval = defaultConfig.batchInterval; - if (this.hasOwnProperty("config") && this.config.hasOwnProperty("batchInterval")) { - ret.batchInterval = this.config.batchInterval; + ret.batchSize = config.batchSize || ret.batchSize || defaultConfig.batchSize; + ret.batchSize = parseInt(ret.batchSize, 10); + if (isNaN(ret.batchSize)) { + throw new Error("Batch size must be a number, found: " + ret.batchSize); } - if (config.hasOwnProperty("batchInterval")) { - ret.batchInterval = config.batchInterval; + else if (ret.batchSize < 0) { + throw new Error("Batch size must be a positive number, found: " + ret.batchSize); } + + ret.batchInterval = config.batchInterval || ret.batchInterval || defaultConfig.batchInterval; ret.batchInterval = parseInt(ret.batchInterval, 10); if (isNaN(ret.batchInterval)) { throw new Error("Batch interval must be a number, found: " + ret.batchInterval); diff --git a/test/test_config.js b/test/test_config.js index f3ff3fb..a16ddec 100644 --- a/test/test_config.js +++ b/test/test_config.js @@ -272,6 +272,36 @@ describe("SplunkLogger", function() { assert.strictEqual(8088, logger.config.port); assert.strictEqual(0, logger.config.maxRetries); }); + it("should error when batchSize=NaN", function() { + var config = { + token: "a-token-goes-here-usually", + batchSize: "not a number", + }; + + try { + var logger = new SplunkLogger(config); + assert.fail(!logger, "Expected an error."); + } + catch (err) { + assert.ok(err); + assert.strictEqual("Batch size must be a number, found: NaN", err.message); + } + }); + it("should error when batchSize is negative", function() { + var config = { + token: "a-token-goes-here-usually", + batchSize: -1, + }; + + try { + var logger = new SplunkLogger(config); + assert.fail(!logger, "Expected an error."); + } + catch (err) { + assert.ok(err); + assert.strictEqual("Batch size must be a positive number, found: -1", err.message); + } + }); it("should set non-default boolean config values", function() { var config = { token: "a-token-goes-here-usually", @@ -683,42 +713,6 @@ describe("SplunkLogger", function() { assert.strictEqual(8088, loggerConfig.port); assert.strictEqual(0, loggerConfig.maxRetries); }); - it("should set autoFlush property to true after initially false", function() { - Object.prototype.something = "ignore"; - var config = { - token: "a-token-goes-here-usually", - url: "splunk.local", - autoFlush: false - }; - - var logger = new SplunkLogger(config); - var loggerConfig = logger.config; - - assert.ok(loggerConfig); - assert.ok(!loggerConfig.hasOwnProperty("something")); - assert.strictEqual(config.token, loggerConfig.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", loggerConfig.name); - assert.strictEqual("splunk.local", loggerConfig.host); - assert.strictEqual("/services/collector/event/1.0", loggerConfig.path); - assert.strictEqual("https", loggerConfig.protocol); - assert.strictEqual("info", loggerConfig.level); - assert.strictEqual(8088, loggerConfig.port); - assert.strictEqual(0, loggerConfig.maxRetries); - assert.strictEqual(false, loggerConfig.autoFlush); - - config.autoFlush = true; - loggerConfig = logger._initializeConfig(config); - - assert.strictEqual(config.token, loggerConfig.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", loggerConfig.name); - assert.strictEqual("splunk.local", loggerConfig.host); - assert.strictEqual("/services/collector/event/1.0", loggerConfig.path); - assert.strictEqual("https", loggerConfig.protocol); - assert.strictEqual("info", loggerConfig.level); - assert.strictEqual(8088, loggerConfig.port); - assert.strictEqual(0, loggerConfig.maxRetries); - assert.strictEqual(true, loggerConfig.autoFlush); - }); }); describe("_initializeRequestOptions", function() { it("should get defaults with no args", function() { @@ -1007,5 +1001,41 @@ describe("SplunkLogger", function() { assert.strictEqual(Logger.config.port, expected.port); assert.strictEqual(Logger.middlewares.length, 0); }); + it("should set autoFlush property to true after initially false", function() { + Object.prototype.something = "ignore"; + var config = { + token: "a-token-goes-here-usually", + url: "splunk.local", + autoFlush: false + }; + + var logger = new SplunkLogger(config); + var loggerConfig = logger.config; + + assert.ok(loggerConfig); + assert.ok(!loggerConfig.hasOwnProperty("something")); + assert.strictEqual(config.token, loggerConfig.token); + assert.strictEqual("splunk-javascript-logging/0.8.0", loggerConfig.name); + assert.strictEqual("splunk.local", loggerConfig.host); + assert.strictEqual("/services/collector/event/1.0", loggerConfig.path); + assert.strictEqual("https", loggerConfig.protocol); + assert.strictEqual("info", loggerConfig.level); + assert.strictEqual(8088, loggerConfig.port); + assert.strictEqual(0, loggerConfig.maxRetries); + assert.strictEqual(false, loggerConfig.autoFlush); + + config.autoFlush = true; + loggerConfig = logger._initializeConfig(config); + + assert.strictEqual(config.token, loggerConfig.token); + assert.strictEqual("splunk-javascript-logging/0.8.0", loggerConfig.name); + assert.strictEqual("splunk.local", loggerConfig.host); + assert.strictEqual("/services/collector/event/1.0", loggerConfig.path); + assert.strictEqual("https", loggerConfig.protocol); + assert.strictEqual("info", loggerConfig.level); + assert.strictEqual(8088, loggerConfig.port); + assert.strictEqual(0, loggerConfig.maxRetries); + assert.strictEqual(true, loggerConfig.autoFlush); + }); }); }); From e5ded8d766e2d3d8a7df4cb03f19bb8dc5f9e711 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Wed, 4 Nov 2015 18:36:23 -0800 Subject: [PATCH 12/46] Moved complicated logic from _sendEvents() to flush() Also updated tests. --- splunklogger.js | 92 ++++++++++------ test/test_config.js | 33 ++++-- test/test_send.js | 251 +++++++++++++++++++++++++++++++++++++------- 3 files changed, 300 insertions(+), 76 deletions(-) diff --git a/splunklogger.js b/splunklogger.js index 257808f..513ca96 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -69,6 +69,8 @@ function _err(err, context) { * @param {bool} [config.autoFlush=true] - Send events immediately or not. * @param {number} [config.batchInterval=0] - If config.autoFlush === true, automatically flush events after this many milliseconds. * When set to a non-positive value, events will be sent one by one. + * @param {number} [config.maxBatchSize=0] - If config.autoFlush === true, automatically flush events after the size of queued + * events exceeds this many bytes. * @constructor * @throws Will throw an error if the config parameter is malformed. */ @@ -119,7 +121,7 @@ var defaultConfig = { autoFlush: true, maxRetries: 0, batchInterval: 0, - batchSize: 0 + maxBatchSize: 0 }; var defaultRequestOptions = { @@ -263,13 +265,13 @@ SplunkLogger.prototype._initializeConfig = function(config) { // Convert autoFlush value to boolean ret.autoFlush = !!ret.autoFlush; - ret.batchSize = config.batchSize || ret.batchSize || defaultConfig.batchSize; - ret.batchSize = parseInt(ret.batchSize, 10); - if (isNaN(ret.batchSize)) { - throw new Error("Batch size must be a number, found: " + ret.batchSize); + ret.maxBatchSize = config.maxBatchSize || ret.maxBatchSize || defaultConfig.maxBatchSize; + ret.maxBatchSize = parseInt(ret.maxBatchSize, 10); + if (isNaN(ret.maxBatchSize)) { + throw new Error("Max batch size must be a number, found: " + ret.maxBatchSize); } - else if (ret.batchSize < 0) { - throw new Error("Batch size must be a positive number, found: " + ret.batchSize); + else if (ret.maxBatchSize < 0) { + throw new Error("Max batch size must be a positive number, found: " + ret.maxBatchSize); } ret.batchInterval = config.batchInterval || ret.batchInterval || defaultConfig.batchInterval; @@ -472,6 +474,26 @@ SplunkLogger.prototype.use = function(middleware) { } }; +/** + * Estimates the size in bytes of the events in the + * queue parameter as if they were sent + * in a single HTTP request. + * + * @param {Array} queue - a queue of events, typically this.contextQueue. + * @returns {number} the estimated size. + */ +SplunkLogger.prototype.calculateBatchSize = function(queue) { + var size = 0; + var that = this; + for (var i = 0; i < queue.length; i++) { + var con = queue[i]; + // TODO: REALLY INEFFICIENT + // TODO: Using Buffer probably breaks all browser support, if any + size += Buffer.byteLength(JSON.stringify(that._makeBody(con)), "utf8"); + } + return size; +}; + /** * Makes an HTTP POST to the configured server. * @@ -493,21 +515,6 @@ SplunkLogger.prototype._post = function(requestOptions, callback) { SplunkLogger.prototype._sendEvents = function(context, callback) { callback = callback || /* istanbul ignore next*/ function(){}; - // Validate the context again, right before using it - context = this._initializeContext(context); - context.requestOptions.headers["Authorization"] = "Splunk " + context.config.token; - - if (context.config.autoFlush) { - context.requestOptions.body = this._makeBody(context); - } - else { - // Don't run _makeBody since we've already done that - context.requestOptions.body = context.message; - // Manually set the content-type header for batched requests, default is application/json - // since json is set to true. - context.requestOptions.headers["content-type"] = "application/x-www-form-urlencoded"; - } - var that = this; var splunkError = null; // Errors returned by Splunk @@ -611,14 +618,16 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { * @throws Will throw an error if the context parameter is malformed. * @public */ -SplunkLogger.prototype.send = function (context, callback) { +SplunkLogger.prototype.send = function(context, callback) { callback = callback || function(){}; context = this._initializeContext(context); this.contextQueue.push(context); - // Only flush if using manual batching - if (context.config.autoFlush && !this._timerID) { + var batchOverSize = this.calculateBatchSize(this.contextQueue) > context.config.maxBatchSize; + + // Only flush if using manual batching, no timer, and if batch is too large + if (context.config.autoFlush && !this._timerID && batchOverSize) { this.flush(callback); } }; @@ -632,13 +641,16 @@ SplunkLogger.prototype.send = function (context, callback) { * @param {function} [callback] - A callback function: function(err, response, body). * @public */ -SplunkLogger.prototype.flush = function (callback) { +SplunkLogger.prototype.flush = function(callback) { callback = callback || function(){}; var context = {}; - // Empty the queue if manual or interval batching with something to flush - if (!this.config.autoFlush || (this._timerID && this.contextQueue.length > 0)) { + var batchOverSize = this.config.maxBatchSize > 0 && this.calculateBatchSize(this.contextQueue) > this.config.maxBatchSize; + var isBatched = !this.config.autoFlush || (this.config.autoFlush && this.contextQueue.length > 0 && (batchOverSize || this._timerID)); + + // Empty the queue if something to flush and manual/interval/size batching + if (isBatched) { var queue = this.contextQueue; this.contextQueue = []; var data = ""; @@ -677,17 +689,35 @@ SplunkLogger.prototype.flush = function (callback) { // After running all, if any, middlewares send the events var that = this; utils.chain(callbacks, function(err, passedContext) { + // The passedContext parameter could be named context to + // do this automatically, but the || notation adds a bit of clarity. + context = passedContext || context; + // Errors from any of the middleware callbacks will fall through to here // If the context is modified at any point the error callback will get it also // event if next("error"); is called w/o the context parameter! // This works because context inside & outside the scope of this function // point to the same memory block. - // The passedContext parameter could be named context to - // do this automatically, but the || notation adds a bit of clarity. if (err) { - that.error(err, passedContext || context); + that.error(err, context); } else { + // Validate the context again, right before using it + context = that._initializeContext(context); + context.requestOptions.headers["Authorization"] = "Splunk " + context.config.token; + + if (isBatched) { + // Don't run _makeBody since we've already done that for batching + // Manually set the content-type header for batched requests, default is application/json + // since json is set to true. + context.requestOptions.headers["content-type"] = "application/x-www-form-urlencoded"; + } + else { + context.message = that._makeBody(context); + } + + context.requestOptions.body = context.message; + that._sendEvents(context, callback); } }); diff --git a/test/test_config.js b/test/test_config.js index a16ddec..2a6f9fd 100644 --- a/test/test_config.js +++ b/test/test_config.js @@ -141,7 +141,10 @@ describe("SplunkLogger", function() { assert.strictEqual("info", logger.config.level); assert.strictEqual(logger.levels.INFO, logger.config.level); assert.strictEqual(8088, logger.config.port); + assert.strictEqual(true, logger.config.autoFlush); assert.strictEqual(0, logger.config.maxRetries); + assert.strictEqual(0, logger.config.batchInterval); + assert.strictEqual(0, logger.config.maxBatchSize); }); it("should set remaining defaults when setting config with token, autoFlush off, & level", function() { var config = { @@ -272,10 +275,10 @@ describe("SplunkLogger", function() { assert.strictEqual(8088, logger.config.port); assert.strictEqual(0, logger.config.maxRetries); }); - it("should error when batchSize=NaN", function() { + it("should error when maxBatchSize=NaN", function() { var config = { token: "a-token-goes-here-usually", - batchSize: "not a number", + maxBatchSize: "not a number", }; try { @@ -284,13 +287,13 @@ describe("SplunkLogger", function() { } catch (err) { assert.ok(err); - assert.strictEqual("Batch size must be a number, found: NaN", err.message); + assert.strictEqual("Max batch size must be a number, found: NaN", err.message); } }); - it("should error when batchSize is negative", function() { + it("should error when maxBatchSize is negative", function() { var config = { token: "a-token-goes-here-usually", - batchSize: -1, + maxBatchSize: -1, }; try { @@ -299,7 +302,7 @@ describe("SplunkLogger", function() { } catch (err) { assert.ok(err); - assert.strictEqual("Batch size must be a positive number, found: -1", err.message); + assert.strictEqual("Max batch size must be a positive number, found: -1", err.message); } }); it("should set non-default boolean config values", function() { @@ -356,6 +359,24 @@ describe("SplunkLogger", function() { assert.strictEqual(config.port, logger.config.port); assert.strictEqual(0, logger.config.maxRetries); }); + it("should set non-default maxBatchSize", function() { + var config = { + token: "a-token-goes-here-usually", + maxBatchSize: 1234 + }; + var logger = new SplunkLogger(config); + + assert.ok(logger); + assert.strictEqual(config.token, logger.config.token); + assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("localhost", logger.config.host); + assert.strictEqual("/services/collector/event/1.0", logger.config.path); + assert.strictEqual("https", logger.config.protocol); + assert.strictEqual("info", logger.config.level); + assert.strictEqual(8088, logger.config.port); + assert.strictEqual(0, logger.config.maxRetries); + assert.strictEqual(1234, logger.config.maxBatchSize); + }); it("should set protocol, host, port & path from url property", function() { var config = { token: "a-token-goes-here-usually", diff --git a/test/test_send.js b/test/test_send.js index 587a5bd..ca11b41 100644 --- a/test/test_send.js +++ b/test/test_send.js @@ -774,6 +774,64 @@ describe("SplunkLogger send (integration tests)", function() { done(); }); }); + it("should succeed using non-default middleware with manual batching", function(done) { + var config = { + token: "token-goes-here", + autoFlush: false + }; + + var initialData = "something"; + var sentContext = { + config: config, + message: initialData + }; + + var middlewareCount = 0; + + function middleware(context, next) { + middlewareCount++; + var expectedString = JSON.stringify(logger._makeBody(sentContext)); + + assert.strictEqual(expectedString.length * 3, context.message.length); + + // Search the payload, we should have 3 identical events + for (var i = 1; i <= 3; i++) { + var start = (i - 1) * expectedString.length; + var end = start + expectedString.length; + + assert.strictEqual(expectedString, context.message.substring(start, end)); + } + + next(null, context); + } + + var logger = new SplunkLogger(config); + logger.use(middleware); + + logger._sendEvents = function(context, next) { + var response = { + headers: { + "content-type": "application/json; charset=UTF-8", + isCustom: true + }, + body: successBody + }; + next(null, response, successBody); + }; + + logger.send(sentContext); + logger.send(sentContext); + logger.send(sentContext); + + logger.flush(function(err, resp, body) { + assert.ok(!err); + assert.strictEqual(resp.body, body); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + assert.strictEqual(middlewareCount, 1); + done(); + }); + }); it("should succeed using non-default middleware, without passing the context through", function(done) { var config = { token: "token-goes-here" @@ -842,7 +900,14 @@ describe("SplunkLogger send (integration tests)", function() { logger.use(middleware2); logger._sendEvents = function(context, next) { - assert.strictEqual(context.message, "somet%3F%3Fhing"); + assert.ok(context.message.hasOwnProperty("event")); + assert.ok(context.message.hasOwnProperty("time")); + + var e = context.message.event; + assert.ok(e.hasOwnProperty("message")); + assert.ok(e.hasOwnProperty("severity")); + assert.strictEqual(e.message, "somet%3F%3Fhing"); + var response = { headers: { "content-type": "application/json; charset=UTF-8", @@ -870,7 +935,7 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed using 3 middlewares", function(done) { var config = { - token: "token-goes-here" + token: configurationFile.token }; var middlewareCount = 0; @@ -901,16 +966,17 @@ describe("SplunkLogger send (integration tests)", function() { logger.use(middleware2); logger.use(middleware3); + var _sendEvents = logger._sendEvents; logger._sendEvents = function(context, next) { - assert.strictEqual(context.message, "somet??hing changed"); - var response = { - headers: { - "content-type": "application/json; charset=UTF-8", - isCustom: true - }, - body: successBody - }; - next(null, response, successBody); + assert.ok(context.message.hasOwnProperty("event")); + assert.ok(context.message.hasOwnProperty("time")); + + var e = context.message.event; + assert.ok(e.hasOwnProperty("message")); + assert.ok(e.hasOwnProperty("severity")); + assert.strictEqual(e.message, "somet??hing changed"); + + _sendEvents(context, next); }; var initialData = "somet??hing"; @@ -930,7 +996,7 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed using 3 middlewares with data object", function(done) { var config = { - token: "token-goes-here" + token: configurationFile.token }; var middlewareCount = 0; @@ -977,20 +1043,20 @@ describe("SplunkLogger send (integration tests)", function() { logger.use(middleware2); logger.use(middleware3); + var _sendEvents = logger._sendEvents; logger._sendEvents = function(context, next) { - assert.strictEqual(context.message.property, "new"); - assert.strictEqual(context.message.nested.object, initialData.nested.object); - assert.strictEqual(context.message.number, 789); - assert.strictEqual(context.message.bool, true); + assert.ok(context.message.hasOwnProperty("event")); + assert.ok(context.message.hasOwnProperty("time")); - var response = { - headers: { - "content-type": "application/json; charset=UTF-8", - isCustom: true - }, - body: successBody - }; - next(null, response, successBody); + var e = context.message.event; + assert.ok(e.hasOwnProperty("message")); + assert.ok(e.hasOwnProperty("severity")); + assert.strictEqual(e.message.property, "new"); + assert.strictEqual(e.message.nested.object, initialData.nested.object); + assert.strictEqual(e.message.number, 789); + assert.strictEqual(e.message.bool, true); + + _sendEvents(context, next); }; var initialData = { @@ -1603,7 +1669,7 @@ describe("SplunkLogger send (integration tests)", function() { }); }); describe("using batch interval", function() { - it("should only not make a POST request if contextQueue is always empty", function(done) { + it("should not make a POST request if contextQueue is always empty", function(done) { var config = { token: configurationFile.token, autoFlush: true, @@ -1945,20 +2011,15 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.strictEqual(0, logger.contextQueue.length); - - if (posts === 1) { - assert.strictEqual(logger._timerDuration, 500); - logger.config.autoFlush = true; - logger.config.batchInterval = 100; - var payload2 = { - message: "something else" - }; - logger.send(payload2); - assert.strictEqual(logger._timerDuration, 100); - } - else { - assert.fail(); // TODO: - } + assert.strictEqual(1, posts); + assert.strictEqual(logger._timerDuration, 500); + logger.config.autoFlush = true; + logger.config.batchInterval = 100; + var payload2 = { + message: "something else" + }; + logger.send(payload2); + assert.strictEqual(logger._timerDuration, 100); }, 550); @@ -2023,4 +2084,116 @@ describe("SplunkLogger send (integration tests)", function() { }, 300); }); }); + // TODO: add combo tests of batchInterval + batchSize/count + describe("using max batch size", function(){ + it("should flush first event immediately with maxBatchSize=1", function(done) { + var config = { + token: configurationFile.token, + autoFlush: true, + maxBatchSize: 1 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context) { + _post(context, function(err, resp, body) { + posts++; + + assert.ok(!logger._timerID); + assert.strictEqual(1, posts); + assert.strictEqual(0, logger.contextQueue.length); + + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + + done(); + }); + }; + + var payload = { + message: "more than 1 byte" + }; + logger.send(payload); + }); + it("should not flush first event with maxBatchSize=1 && autoFlush=false", function(done) { + var config = { + token: configurationFile.token, + autoFlush: false, + maxBatchSize: 1 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context) { + _post(context, function(err, resp, body) { + posts++; + + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + }); + }; + + var payload = { + message: "more than 1 byte" + }; + logger.send(payload); + + setTimeout(function() { + assert.ok(!logger._timerID); + assert.strictEqual(0, posts); + assert.strictEqual(1, logger.contextQueue.length); + + done(); + }, 1000); + }); + it("should flush first 2 events after maxBatchSize>100", function(done) { + var config = { + token: configurationFile.token, + autoFlush: true, + maxBatchSize: 100 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context) { + _post(context, function(err, resp, body) { + posts++; + + assert.ok(!logger._timerID); + assert.strictEqual(1, posts); + assert.strictEqual(0, logger.contextQueue.length); + + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + + done(); + }); + }; + + var payload = { + message: "more than 1 byte" + }; + logger.send(payload); + + setTimeout(function() { + assert.ok(!logger._timerID); + assert.strictEqual(0, posts); + assert.strictEqual(1, logger.contextQueue.length); + + logger.send(payload); + }, 300); + }); + }); }); From 2f4bb81bd9afdac787736ad71a6570846e92d1e3 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Fri, 6 Nov 2015 11:40:58 -0800 Subject: [PATCH 13/46] Optimize calculateBatchSize, add additional test coverage --- splunklogger.js | 37 +++++------ test/test_send.js | 153 ++++++++++++++++++++++++++++++++++++---------- 2 files changed, 136 insertions(+), 54 deletions(-) diff --git a/splunklogger.js b/splunklogger.js index 513ca96..b0e6d6f 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -80,6 +80,7 @@ var SplunkLogger = function(config) { this.config = this._initializeConfig(config); this.middlewares = []; this.contextQueue = []; + this.eventSizes = []; this.error = _err; this._enableTimer = utils.bind(this, this._enableTimer); @@ -171,7 +172,9 @@ SplunkLogger.prototype._enableTimer = function(interval) { var that = this; this._timerID = setInterval(function() { - that.flush(); + if (that.contextQueue.length > 0) { + that.flush(); + } }, interval); } } @@ -479,17 +482,12 @@ SplunkLogger.prototype.use = function(middleware) { * queue parameter as if they were sent * in a single HTTP request. * - * @param {Array} queue - a queue of events, typically this.contextQueue. - * @returns {number} the estimated size. + * @returns {number} the estimated size in bytes. */ -SplunkLogger.prototype.calculateBatchSize = function(queue) { +SplunkLogger.prototype.calculateBatchSize = function() { var size = 0; - var that = this; - for (var i = 0; i < queue.length; i++) { - var con = queue[i]; - // TODO: REALLY INEFFICIENT - // TODO: Using Buffer probably breaks all browser support, if any - size += Buffer.byteLength(JSON.stringify(that._makeBody(con)), "utf8"); + for (var i = 0; i < this.eventSizes.length; i++) { + size += this.eventSizes[i]; } return size; }; @@ -622,9 +620,11 @@ SplunkLogger.prototype.send = function(context, callback) { callback = callback || function(){}; context = this._initializeContext(context); + // Store the context, and its estimated length this.contextQueue.push(context); + this.eventSizes.push(Buffer.byteLength(JSON.stringify(this._makeBody(context)), "utf8")); - var batchOverSize = this.calculateBatchSize(this.contextQueue) > context.config.maxBatchSize; + var batchOverSize = this.calculateBatchSize() > context.config.maxBatchSize; // Only flush if using manual batching, no timer, and if batch is too large if (context.config.autoFlush && !this._timerID && batchOverSize) { @@ -646,30 +646,25 @@ SplunkLogger.prototype.flush = function(callback) { var context = {}; - var batchOverSize = this.config.maxBatchSize > 0 && this.calculateBatchSize(this.contextQueue) > this.config.maxBatchSize; - var isBatched = !this.config.autoFlush || (this.config.autoFlush && this.contextQueue.length > 0 && (batchOverSize || this._timerID)); + var batchOverSize = this.config.maxBatchSize > 0 && this.calculateBatchSize() > this.config.maxBatchSize; + var isBatched = !this.config.autoFlush || (this.config.autoFlush && (batchOverSize || this._timerID)); - // Empty the queue if something to flush and manual/interval/size batching + // Empty the queue if manual/interval/size batching if (isBatched) { var queue = this.contextQueue; this.contextQueue = []; + this.eventSizes = []; var data = ""; for (var i = 0; i < queue.length; i++) { data += JSON.stringify(this._makeBody(queue[i])); } context.message = data; } - // Noop if there's nothing to flush; let non-batching requests get a no-data response from Splunk in the if block above - else if (this.contextQueue.length === 0) { - var body = { - text: "Nothing to flush." - }; - return callback(null, {body: body}, body); - } // Just take the oldest event in the queue else { // TODO: handle case of multiple events with autoFlush off, flushing fast context = this.contextQueue.pop(); + this.eventSizes.pop(); } // Initialize the context, then manually set the data diff --git a/test/test_send.js b/test/test_send.js index ca11b41..de7c834 100644 --- a/test/test_send.js +++ b/test/test_send.js @@ -165,7 +165,7 @@ describe("SplunkLogger _makedata", function() { }); }); describe("SplunkLogger send (integration tests)", function() { - describe("using default middleware ", function () { + describe("using default middleware", function () { it("should error with bad token", function(done) { var config = { token: "token-goes-here" @@ -187,7 +187,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.strictEqual(err.message, invalidTokenBody.text); assert.strictEqual(err.code, invalidTokenBody.code); assert.ok(errContext); - assert.strictEqual(errContext, context); + assert.strictEqual(errContext.message, context.message); }; logger.send(context, function(err, resp, body) { @@ -215,9 +215,11 @@ describe("SplunkLogger send (integration tests)", function() { }; assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); logger.send(context); setTimeout(function() { assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); done(); }, 500); }); @@ -600,10 +602,14 @@ describe("SplunkLogger send (integration tests)", function() { }; logger.send(context); - logger.send(context); + var context2 = { + message: "second batched event" + }; + logger.send(context2); setTimeout(function() { assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); assert.strictEqual(sent, 2); done(); }, 1000); @@ -629,6 +635,7 @@ describe("SplunkLogger send (integration tests)", function() { }; assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); logger.flush(function(err, resp, body) { assert.ok(!err); assert.ok(run); @@ -637,6 +644,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.strictEqual(body.text, noDataBody.text); assert.strictEqual(body.code, noDataBody.code); assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); done(); }); }); @@ -658,8 +666,12 @@ describe("SplunkLogger send (integration tests)", function() { // Nothing should be sent if queue is empty assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.calculateBatchSize(), 0); logger.flush(); assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.calculateBatchSize(), 0); }); it("should flush a batch of 1 event with valid token", function(done) { var config = { @@ -669,7 +681,7 @@ describe("SplunkLogger send (integration tests)", function() { var logger = new SplunkLogger(config); - var data = "batched event 1"; + var data = this.test.fullTitle(); var context = { config: config, message: data @@ -678,6 +690,9 @@ describe("SplunkLogger send (integration tests)", function() { logger.send(context); assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.eventSizes.length, 1); + assert.ok(logger.calculateBatchSize() > 0); + assert.strictEqual(logger.calculateBatchSize(), logger.eventSizes[0]); logger.flush(function(err, resp, body) { assert.ok(!err); assert.strictEqual(resp.headers["content-type"], "application/json; charset=UTF-8"); @@ -685,6 +700,8 @@ describe("SplunkLogger send (integration tests)", function() { assert.strictEqual(body.text, successBody.text); assert.strictEqual(body.code, successBody.code); assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.calculateBatchSize(), 0); done(); }); }); @@ -696,7 +713,7 @@ describe("SplunkLogger send (integration tests)", function() { var logger = new SplunkLogger(config); - var data = "batched event"; + var data = this.test.fullTitle(); var context = { config: config, message: data @@ -706,6 +723,8 @@ describe("SplunkLogger send (integration tests)", function() { logger.send(context); assert.strictEqual(logger.contextQueue.length, 2); + assert.strictEqual(logger.eventSizes.length, 2); + assert.strictEqual(logger.calculateBatchSize(), logger.eventSizes[0] * 2); logger.flush(function(err, resp, body) { assert.ok(!err); assert.strictEqual(resp.headers["content-type"], "application/json; charset=UTF-8"); @@ -713,6 +732,8 @@ describe("SplunkLogger send (integration tests)", function() { assert.strictEqual(body.text, successBody.text); assert.strictEqual(body.code, successBody.code); assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.calculateBatchSize(), 0); done(); }); }); @@ -799,7 +820,14 @@ describe("SplunkLogger send (integration tests)", function() { var start = (i - 1) * expectedString.length; var end = start + expectedString.length; - assert.strictEqual(expectedString, context.message.substring(start, end)); + // Don't compare timestamps + var expected = JSON.parse(expectedString); + var actual = JSON.parse(context.message.substring(start, end)); + assert.ok(expected.hasOwnProperty("event")); + assert.ok(expected.hasOwnProperty("time")); + assert.ok(actual.hasOwnProperty("event")); + assert.ok(actual.hasOwnProperty("time")); + assert.strictEqual(JSON.stringify(expected.event), JSON.stringify(actual.event)); } next(null, context); @@ -1659,8 +1687,8 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(!err); assert.ok(resp); assert.ok(body); - assert.strictEqual(invalidTokenBody.code, body.code); - assert.strictEqual(invalidTokenBody.text, body.text); + assert.strictEqual(body.code, invalidTokenBody.code); + assert.strictEqual(body.text, invalidTokenBody.text); assert.strictEqual(1, retryCount); assert.ok(run); @@ -1693,8 +1721,9 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.strictEqual(logger._timerDuration, 100); - assert.strictEqual(0, posts); - assert.strictEqual(0, logger.contextQueue.length); + assert.strictEqual(posts, 0); + assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); // Clean up the timer logger._disableTimer(); @@ -1730,8 +1759,9 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.strictEqual(logger._timerDuration, 100); - assert.strictEqual(1, posts); - assert.strictEqual(0, logger.contextQueue.length); + assert.strictEqual(posts, 1); + assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); // Clean up the timer logger._disableTimer(); @@ -1771,8 +1801,8 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.strictEqual(logger._timerDuration, 100); - assert.strictEqual(1, posts); - assert.strictEqual(0, logger.contextQueue.length); + assert.strictEqual(posts, 1); + assert.strictEqual(logger.contextQueue.length, 0); // Clean up the timer logger._disableTimer(); @@ -1812,8 +1842,9 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.strictEqual(logger._timerDuration, 100); - assert.strictEqual(1, posts); - assert.strictEqual(0, logger.contextQueue.length); + assert.strictEqual(posts, 1); + assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); // Clean up the timer logger._disableTimer(); @@ -1872,8 +1903,9 @@ describe("SplunkLogger send (integration tests)", function() { logger.send(payload2); setTimeout(function() { - assert.strictEqual(1, posts); - assert.strictEqual(0, logger.contextQueue.length); + assert.strictEqual(posts, 1); + assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); // Clean up the timer logger._disableTimer(); @@ -1917,6 +1949,7 @@ describe("SplunkLogger send (integration tests)", function() { logger.send(payload2); assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.eventSizes.length, 1); logger.flush(); run = true; }, 150); @@ -1925,6 +1958,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(run); assert.strictEqual(2, posts); assert.strictEqual(0, logger.contextQueue.length); + assert.strictEqual(0, logger.eventSizes.length); // Clean up the timer logger._disableTimer(); @@ -1968,6 +2002,7 @@ describe("SplunkLogger send (integration tests)", function() { logger.send(payload2); assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.eventSizes.length, 1); logger.flush(); run = true; }, 150); @@ -1976,6 +2011,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(run); assert.strictEqual(2, posts); assert.strictEqual(0, logger.contextQueue.length); + assert.strictEqual(0, logger.eventSizes.length); // Clean up the timer logger._disableTimer(); @@ -2010,8 +2046,9 @@ describe("SplunkLogger send (integration tests)", function() { logger.send(payload); setTimeout(function() { - assert.strictEqual(0, logger.contextQueue.length); - assert.strictEqual(1, posts); + assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(posts, 1); assert.strictEqual(logger._timerDuration, 500); logger.config.autoFlush = true; logger.config.batchInterval = 100; @@ -2024,8 +2061,8 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { - assert.strictEqual(2, posts); - assert.strictEqual(0, logger.contextQueue.length); + assert.strictEqual(posts, 2); + assert.strictEqual(logger.contextQueue.length, 0); // Clean up the timer logger._disableTimer(); @@ -2069,14 +2106,16 @@ describe("SplunkLogger send (integration tests)", function() { logger.send(payload2); assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.eventSizes.length, 1); run = true; }, 150); setTimeout(function() { assert.ok(run); - assert.strictEqual(2, posts); - assert.strictEqual(0, logger.contextQueue.length); + assert.strictEqual(posts, 2); + assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); // Clean up the timer logger._disableTimer(); @@ -2084,7 +2123,6 @@ describe("SplunkLogger send (integration tests)", function() { }, 300); }); }); - // TODO: add combo tests of batchInterval + batchSize/count describe("using max batch size", function(){ it("should flush first event immediately with maxBatchSize=1", function(done) { var config = { @@ -2103,8 +2141,9 @@ describe("SplunkLogger send (integration tests)", function() { posts++; assert.ok(!logger._timerID); - assert.strictEqual(1, posts); - assert.strictEqual(0, logger.contextQueue.length); + assert.strictEqual(posts, 1); + assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); assert.ok(!err); assert.strictEqual(body.code, successBody.code); @@ -2148,8 +2187,9 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.ok(!logger._timerID); - assert.strictEqual(0, posts); - assert.strictEqual(1, logger.contextQueue.length); + assert.strictEqual(posts, 0); + assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.eventSizes.length, 1); done(); }, 1000); @@ -2171,8 +2211,9 @@ describe("SplunkLogger send (integration tests)", function() { posts++; assert.ok(!logger._timerID); - assert.strictEqual(1, posts); - assert.strictEqual(0, logger.contextQueue.length); + assert.strictEqual(posts, 1); + assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); assert.ok(!err); assert.strictEqual(body.code, successBody.code); @@ -2189,11 +2230,57 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.ok(!logger._timerID); - assert.strictEqual(0, posts); - assert.strictEqual(1, logger.contextQueue.length); + assert.strictEqual(posts, 0); + assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.eventSizes.length, 1); logger.send(payload); }, 300); }); + it("should flush first event after 200ms, with maxBatchSize=200", function(done) { + var config = { + token: configurationFile.token, + autoFlush: true, + maxBatchSize: 200, + batchInterval: 200 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context) { + _post(context, function(err, resp, body) { + posts++; + + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + }); + }; + + var payload = { + message: "more than 1 byte" + }; + logger.send(payload); + + // Make sure the event wasn't flushed yet + setTimeout(function() { + assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.eventSizes.length, 1); + }, 150); + + setTimeout(function() { + assert.ok(logger._timerID); + assert.strictEqual(logger._timerDuration, 200); + logger._disableTimer(); + + assert.strictEqual(posts, 1); + assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); + done(); + }, 250); + }); }); }); From 806b2206942472c5487f33d9f9509503339e18bd Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Mon, 9 Nov 2015 15:36:21 -0800 Subject: [PATCH 14/46] Add configuration setting for batch count When this many events are in the contextQueue, flush() will be called. --- splunklogger.js | 29 ++++-- test/test_config.js | 30 +++++++ test/test_send.js | 212 +++++++++++++++++++++++++++++++++++++++++++- 3 files changed, 263 insertions(+), 8 deletions(-) diff --git a/splunklogger.js b/splunklogger.js index b0e6d6f..99a8320 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -50,6 +50,7 @@ function _err(err, context) { * @property {object} config - Configuration settings for this SplunkLogger instance. * @property {function[]} middlewares - Middleware functions to run before sending data to Splunk. * @property {object[]} contextQueue - Queue of context objects to be sent to Splunk. + * @property {number[]} eventSizes - Parallel queue of the sizes of events stored in contextQueue. * @property {function} error - A callback function for errors: function(err, context). * Defaults to console.log both values; * @@ -71,6 +72,8 @@ function _err(err, context) { * When set to a non-positive value, events will be sent one by one. * @param {number} [config.maxBatchSize=0] - If config.autoFlush === true, automatically flush events after the size of queued * events exceeds this many bytes. + * @param {number} [config.maxBatchCount=0] - If config.autoFlush === true, automatically flush events after this many + * events have been queued. * @constructor * @throws Will throw an error if the config parameter is malformed. */ @@ -122,7 +125,8 @@ var defaultConfig = { autoFlush: true, maxRetries: 0, batchInterval: 0, - maxBatchSize: 0 + maxBatchSize: 0, + maxBatchCount: 0 }; var defaultRequestOptions = { @@ -164,7 +168,7 @@ SplunkLogger.prototype._enableTimer = function(interval) { } // If batch interval is changed, update the config property - if (this.config && this.config.hasOwnProperty("batchInterval")) { + if (this.config) { this.config.batchInterval = interval; } @@ -268,6 +272,15 @@ SplunkLogger.prototype._initializeConfig = function(config) { // Convert autoFlush value to boolean ret.autoFlush = !!ret.autoFlush; + ret.maxBatchCount = config.maxBatchCount || ret.maxBatchCount || defaultConfig.maxBatchCount; + ret.maxBatchCount = parseInt(ret.maxBatchCount, 10); + if (isNaN(ret.maxBatchCount)) { + throw new Error("Max batch count must be a number, found: " + ret.maxBatchCount); + } + else if (ret.maxBatchCount < 0) { + throw new Error("Max batch count must be a positive number, found: " + ret.maxBatchCount); + } + ret.maxBatchSize = config.maxBatchSize || ret.maxBatchSize || defaultConfig.maxBatchSize; ret.maxBatchSize = parseInt(ret.maxBatchSize, 10); if (isNaN(ret.maxBatchSize)) { @@ -625,10 +638,11 @@ SplunkLogger.prototype.send = function(context, callback) { this.eventSizes.push(Buffer.byteLength(JSON.stringify(this._makeBody(context)), "utf8")); var batchOverSize = this.calculateBatchSize() > context.config.maxBatchSize; + var batchOverCount = this.contextQueue.length >= context.config.maxBatchCount && context.config.maxBatchCount > 0; - // Only flush if using manual batching, no timer, and if batch is too large - if (context.config.autoFlush && !this._timerID && batchOverSize) { - this.flush(callback); + // Only flush if not using manual batching, and if the contextQueue is too large or has many events + if (context.config.autoFlush && !this._timerID && (batchOverSize || batchOverCount)) { + this.flush(callback || function(){}); } }; @@ -647,9 +661,10 @@ SplunkLogger.prototype.flush = function(callback) { var context = {}; var batchOverSize = this.config.maxBatchSize > 0 && this.calculateBatchSize() > this.config.maxBatchSize; - var isBatched = !this.config.autoFlush || (this.config.autoFlush && (batchOverSize || this._timerID)); + var batchOverCount = this.config.maxBatchCount > 0 && this.contextQueue.length >= this.config.maxBatchCount; + var isBatched = !this.config.autoFlush || (this.config.autoFlush && (batchOverSize || batchOverCount || this._timerID)); - // Empty the queue if manual/interval/size batching + // Empty the queue if manual/interval/size/count batching if (isBatched) { var queue = this.contextQueue; this.contextQueue = []; diff --git a/test/test_config.js b/test/test_config.js index 2a6f9fd..9e3f6d4 100644 --- a/test/test_config.js +++ b/test/test_config.js @@ -275,6 +275,36 @@ describe("SplunkLogger", function() { assert.strictEqual(8088, logger.config.port); assert.strictEqual(0, logger.config.maxRetries); }); + it("should error when maxBatchCount=NaN", function() { + var config = { + token: "a-token-goes-here-usually", + maxBatchCount: "not a number", + }; + + try { + var logger = new SplunkLogger(config); + assert.fail(!logger, "Expected an error."); + } + catch (err) { + assert.ok(err); + assert.strictEqual("Max batch count must be a number, found: NaN", err.message); + } + }); + it("should error when maxBatchCount is negative", function() { + var config = { + token: "a-token-goes-here-usually", + maxBatchCount: -1, + }; + + try { + var logger = new SplunkLogger(config); + assert.fail(!logger, "Expected an error."); + } + catch (err) { + assert.ok(err); + assert.strictEqual("Max batch count must be a positive number, found: -1", err.message); + } + }); it("should error when maxBatchSize=NaN", function() { var config = { token: "a-token-goes-here-usually", diff --git a/test/test_send.js b/test/test_send.js index de7c834..4d8b2ea 100644 --- a/test/test_send.js +++ b/test/test_send.js @@ -2123,7 +2123,7 @@ describe("SplunkLogger send (integration tests)", function() { }, 300); }); }); - describe("using max batch size", function(){ + describe("using max batch size", function() { it("should flush first event immediately with maxBatchSize=1", function(done) { var config = { token: configurationFile.token, @@ -2236,6 +2236,13 @@ describe("SplunkLogger send (integration tests)", function() { logger.send(payload); }, 300); + + setTimeout(function() { + assert.ok(!logger._timerID); + assert.strictEqual(posts, 1); + assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); + }, 400); }); it("should flush first event after 200ms, with maxBatchSize=200", function(done) { var config = { @@ -2283,4 +2290,207 @@ describe("SplunkLogger send (integration tests)", function() { }, 250); }); }); + describe("using max batch count", function() { + it("should flush first event immediately with maxBatchCount=1 with large maxBatchSize", function(done) { + var config = { + token: configurationFile.token, + maxBatchCount: 1, + maxBatchSize: 123456 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context) { + _post(context, function(err, resp, body) { + posts++; + + assert.ok(!logger._timerID); + assert.strictEqual(posts, 1); + assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); + + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + + done(); + }); + }; + + var payload = { + message: "one event" + }; + logger.send(payload); + }); + it("should not flush first event with maxBatchCount=1 && autoFlush=false", function(done) { + var config = { + token: configurationFile.token, + autoFlush: false, + maxBatchCount: 1 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context) { + _post(context, function(err, resp, body) { + posts++; + + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + }); + }; + + var payload = { + message: "one event" + }; + logger.send(payload); + + setTimeout(function() { + assert.ok(!logger._timerID); + assert.strictEqual(posts, 0); + assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.eventSizes.length, 1); + + done(); + }, 1000); + }); + it("should not flush events with maxBatchCount=0 (meaning ignore) and large maxBatchSize", function(done) { + var config = { + token: configurationFile.token, + maxBatchCount: 0, + maxBatchSize: 123456 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context) { + _post(context, function(err, resp, body) { + posts++; + + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + }); + }; + + var payload = { + message: "one event" + }; + logger.send(payload); + + setTimeout(function() { + assert.ok(!logger._timerID); + assert.strictEqual(posts, 0); + assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.eventSizes.length, 1); + + done(); + }, 1000); + }); + it("should flush first 2 events after maxBatchCount=2, ignoring large maxBatchSize", function(done) { + var config = { + token: configurationFile.token, + maxBatchCount: 2, + maxBatchSize: 123456 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context) { + _post(context, function(err, resp, body) { + posts++; + + assert.ok(!logger._timerID); + assert.strictEqual(posts, 1); + assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); + + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + + done(); + }); + }; + + var payload = { + message: "one event" + }; + logger.send(payload); + + setTimeout(function() { + assert.ok(!logger._timerID); + assert.strictEqual(posts, 0); + assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.eventSizes.length, 1); + + logger.send(payload); + }, 300); + + setTimeout(function() { + assert.ok(!logger._timerID); + assert.strictEqual(posts, 1); + assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); + }, 400); + }); + it("should flush first event after 200ms, with maxBatchCount=10", function(done) { + var config = { + token: configurationFile.token, + autoFlush: true, + maxBatchCount: 10, + batchInterval: 200 + }; + var logger = new SplunkLogger(config); + + var posts = 0; + + // Wrap _post so we can verify how many times we called it + var _post = logger._post; + logger._post = function(context) { + _post(context, function(err, resp, body) { + posts++; + + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + }); + }; + + var payload = { + message: "one event" + }; + logger.send(payload); + + // Make sure the event wasn't flushed yet + setTimeout(function() { + assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.eventSizes.length, 1); + }, 150); + + setTimeout(function() { + assert.ok(logger._timerID); + assert.strictEqual(logger._timerDuration, 200); + logger._disableTimer(); + + assert.strictEqual(posts, 1); + assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventSizes.length, 0); + done(); + }, 300); + }); + }); }); From bdc30acd230f9e461e307405bf2e2a0e20951f81 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Mon, 9 Nov 2015 17:02:37 -0800 Subject: [PATCH 15/46] Add comments to timer initialization logic --- splunklogger.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/splunklogger.js b/splunklogger.js index 99a8320..adcde48 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -299,14 +299,16 @@ SplunkLogger.prototype._initializeConfig = function(config) { throw new Error("Batch interval must be a positive number, found: " + ret.batchInterval); } + // Has the interval timer not started, and needs to be started? var startTimer = !this._timerID && ret.autoFlush && ret.batchInterval > 0; + // Has the interval timer already started, and the interval changed to a different duration? var changeTimer = this._timerID && ret.autoFlush && this._timerDuration !== ret.batchInterval && ret.batchInterval > 0; // Upsert the timer if (startTimer || changeTimer) { this._enableTimer(ret.batchInterval); } - // Disable timer + // Disable timer - there is currently a timer, but config says we no longer need a timer else if (this._timerID && (this._timerDuration < 0 || !ret.autoFlush)) { this._disableTimer(); } From 9949cdc992e37098a871b4a066576cf2ecab5e05 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Fri, 13 Nov 2015 14:08:36 -0800 Subject: [PATCH 16/46] API changes based on feedback from #1 TODO: - Make some decisions about middleware - Update remaining examples - Add a few timer tests for 100% coverage - Added ability to have custom event formatter - Context no longer has config or requestOptions - If !autoflush and any batching settings, throw an error - Simplified logic in many places - Added several util functions - Updated tests - Clarified the basic.js example --- examples/basic.js | 44 ++- splunklogger.js | 335 +++++++----------- test/test_config.js | 202 +++-------- test/test_send.js | 845 +++++++++++++++++++++++--------------------- test/test_utils.js | 71 ++++ utils.js | 95 +++++ 6 files changed, 811 insertions(+), 781 deletions(-) diff --git a/examples/basic.js b/examples/basic.js index c0656b0..a457b4d 100644 --- a/examples/basic.js +++ b/examples/basic.js @@ -23,21 +23,11 @@ var SplunkLogger = require("../index").Logger; /** * Only the token property is required. - * Defaults are listed explicitly. - * - * Alternatively, specify config.url like so: - * - * "https://localhost:8088/services/collector/event/1.0" */ var config = { token: "your-token-here", - host: "localhost", - path: "/services/collector/event/1.0", - protocol: "https", - port: 8088, - level: "info", - autoFlush: true, - maxRetries: 0 + url: "https://localhost:8088", + maxBatchCount: 1 // Send events 1 at a time }; // Create a new logger @@ -50,7 +40,7 @@ Logger.error = function(err, context) { // Define the payload to send to Splunk's Event Collector var payload = { - // Message can be anything, doesn't have to be an object + // Message can be anything, it doesn't have to be an object message: { temperature: "70F", chickenCount: 500 @@ -67,6 +57,34 @@ var payload = { }; console.log("Sending payload", payload); + +/** + * Since maxBatchCount is set to 1, calling send + * will immediately send the payload. + * + * The underlying HTTP POST request is made to + * + * https://localhost:8088/services/collector/event/1.0 + * + * with the following data + * + * "{ + * metadata: { + * source: "chicken coop", + * sourcetype: "httpevent", + * index: "main", + * host: "farm.local" + * }, + * event: { + * message: { + * temperature: "70F", + * chickenCount: 500 + * }, + * severity: "info" + * }, + * }" + * + */ Logger.send(payload, function(err, resp, body) { // If successful, body will be { text: 'Success', code: 0 } console.log("Response from Splunk", body); diff --git a/splunklogger.js b/splunklogger.js index adcde48..35e5bba 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -31,6 +31,23 @@ function _err(err, context) { console.log("ERROR:", err, " CONTEXT", context); } +/** + * The default format for Splunk events. + * + * This function can be overwritten, and can return any type (string, object, array, etc.) + * + * @param {anything} [message] - The event message. + * @param {string} [severity] - The event severity. + * @return {any} The event format to send to Splunk, + */ +function _defaultEventFormatter(message, severity) { + var event = { + message: message, + severity: severity + }; + return event; +} + /** * Constructs a SplunkLogger, to send events to Splunk via the HTTP Event Collector. * See defaultConfig for default configuration settings. @@ -48,9 +65,12 @@ function _err(err, context) { * var logger = new SplunkLogger(config); * * @property {object} config - Configuration settings for this SplunkLogger instance. + * @param {object} requestOptions - Options to pass to {@link https://github.com/request/request#requestpost|request.post()}. + * See the {@link http://github.com/request/request|request documentation} for all available options. * @property {function[]} middlewares - Middleware functions to run before sending data to Splunk. * @property {object[]} contextQueue - Queue of context objects to be sent to Splunk. - * @property {number[]} eventSizes - Parallel queue of the sizes of events stored in contextQueue. + * @property {function} eventFormatter - Formats events, returning an event as a string, function(message, severity). + * Can be overwritten, the default event formatter will display event and severity as properties in a JSON object. * @property {function} error - A callback function for errors: function(err, context). * Defaults to console.log both values; * @@ -70,9 +90,9 @@ function _err(err, context) { * @param {bool} [config.autoFlush=true] - Send events immediately or not. * @param {number} [config.batchInterval=0] - If config.autoFlush === true, automatically flush events after this many milliseconds. * When set to a non-positive value, events will be sent one by one. - * @param {number} [config.maxBatchSize=0] - If config.autoFlush === true, automatically flush events after the size of queued + * @param {number} [config.maxBatchSize=10240] - If config.autoFlush === true, automatically flush events after the size of queued * events exceeds this many bytes. - * @param {number} [config.maxBatchCount=0] - If config.autoFlush === true, automatically flush events after this many + * @param {number} [config.maxBatchCount=10] - If config.autoFlush === true, automatically flush events after this many * events have been queued. * @constructor * @throws Will throw an error if the config parameter is malformed. @@ -81,16 +101,18 @@ var SplunkLogger = function(config) { this._timerID = null; this._timerDuration = 0; this.config = this._initializeConfig(config); + this.requestOptions = this._initializeRequestOptions(); this.middlewares = []; this.contextQueue = []; - this.eventSizes = []; + this.eventsBatchSize = 0; + this.eventFormatter = _defaultEventFormatter; this.error = _err; this._enableTimer = utils.bind(this, this._enableTimer); this._disableTimer = utils.bind(this, this._disableTimer); this._initializeConfig = utils.bind(this, this._initializeConfig); this._initializeRequestOptions = utils.bind(this, this._initializeRequestOptions); - this._initializeMessage = utils.bind(this, this._initializeMessage); + this._validateMessage = utils.bind(this, this._validateMessage); this._initializeMetadata = utils.bind(this, this._initializeMetadata); this._initializeContext = utils.bind(this, this._initializeContext); this._makeBody = utils.bind(this, this._makeBody); @@ -125,14 +147,13 @@ var defaultConfig = { autoFlush: true, maxRetries: 0, batchInterval: 0, - maxBatchSize: 0, - maxBatchCount: 0 + maxBatchSize: 10 * 1024, + maxBatchCount: 10 }; var defaultRequestOptions = { json: true, // Sets the content-type header to application/json - strictSSL: false, - url: defaultConfig.protocol + "://" + defaultConfig.host + ":" + defaultConfig.port + defaultConfig.path + strictSSL: false }; /** @@ -158,33 +179,25 @@ SplunkLogger.prototype._disableTimer = function() { */ SplunkLogger.prototype._enableTimer = function(interval) { // Only enable the timer if possible - if (typeof interval === "number") { - if (interval <= 0) { - throw new Error("Batch interval must be a positive number, found: " + interval); - } - else { - if (this._timerID) { - this._disableTimer(); - } - - // If batch interval is changed, update the config property - if (this.config) { - this.config.batchInterval = interval; - } - - this._timerDuration = interval; + interval = utils.validatePositiveInt(interval, "Batch interval"); - var that = this; - this._timerID = setInterval(function() { - if (that.contextQueue.length > 0) { - that.flush(); - } - }, interval); - } + if (this._timerID) { + this._disableTimer(); } - else { - throw new Error("Batch interval must be a number, found: " + interval); + + // If batch interval is changed, update the config property + if (this.config) { + this.config.batchInterval = interval; } + + this._timerDuration = interval; + + var that = this; + this._timerID = setInterval(function() { + if (that.contextQueue.length > 0) { + that.flush(); + } + }, interval); }; /** @@ -197,12 +210,7 @@ SplunkLogger.prototype._enableTimer = function(interval) { */ SplunkLogger.prototype._initializeConfig = function(config) { // Copy over the instance config - var ret = {}; - for (var key in this.config) { - if (this.config.hasOwnProperty(key)) { - ret[key] = this.config[key]; - } - } + var ret = utils.copyObject(this.config); if (!config) { throw new Error("Config is required."); @@ -243,60 +251,43 @@ SplunkLogger.prototype._initializeConfig = function(config) { } // Take the argument's value, then instance value, then the default value - ret.token = config.token || ret.token; - ret.name = config.name || ret.name || defaultConfig.name; - ret.host = config.host || ret.host || defaultConfig.host; - ret.path = config.path || ret.path || defaultConfig.path; - ret.protocol = config.protocol || ret.protocol || defaultConfig.protocol; - ret.level = config.level || ret.level || defaultConfig.level; - - ret.maxRetries = config.maxRetries || ret.maxRetries || defaultConfig.maxRetries; - ret.maxRetries = parseInt(ret.maxRetries, 10); - if (isNaN(ret.maxRetries)) { - throw new Error("Max retries must be a number, found: " + ret.maxRetries); - } - else if (ret.maxRetries < 0) { - throw new Error("Max retries must be a positive number, found: " + ret.maxRetries); + ret.token = utils.orByProp("token", config, ret); + ret.name = utils.orByProp("name", config, ret, defaultConfig); + ret.level = utils.orByProp("level", config, ret, defaultConfig); + + ret.host = utils.orByProp("host", config, ret, defaultConfig); + ret.path = utils.orByProp("path", config, ret, defaultConfig); + ret.protocol = utils.orByProp("protocol", config, ret, defaultConfig); + ret.port = utils.orByProp("port", config, ret, defaultConfig); + ret.port = utils.validatePositiveInt(ret.port, "Port"); + if (ret.port < 1000 || ret.port > 65535) { + throw new Error("Port must be an integer between 1000 and 65535, found: " + ret.port); } - // Start with the default autoFlush value - ret.autoFlush = defaultConfig.autoFlush; - // Then check this.config.autoFlush - if (this.hasOwnProperty("config") && this.config.hasOwnProperty("autoFlush")) { - ret.autoFlush = this.config.autoFlush; - } - // Then check the config.autoFlush, the function argument - if (config.hasOwnProperty("autoFlush")) { - ret.autoFlush = config.autoFlush; - } - // Convert autoFlush value to boolean - ret.autoFlush = !!ret.autoFlush; + ret.maxRetries = utils.orByProp("maxRetries", config, ret, defaultConfig); + ret.maxRetries = utils.validatePositiveInt(ret.maxRetries, "Max retries"); - ret.maxBatchCount = config.maxBatchCount || ret.maxBatchCount || defaultConfig.maxBatchCount; - ret.maxBatchCount = parseInt(ret.maxBatchCount, 10); - if (isNaN(ret.maxBatchCount)) { - throw new Error("Max batch count must be a number, found: " + ret.maxBatchCount); - } - else if (ret.maxBatchCount < 0) { - throw new Error("Max batch count must be a positive number, found: " + ret.maxBatchCount); - } + ret.autoFlush = utils.orByBooleanProp("autoFlush", config, this.config, defaultConfig); - ret.maxBatchSize = config.maxBatchSize || ret.maxBatchSize || defaultConfig.maxBatchSize; - ret.maxBatchSize = parseInt(ret.maxBatchSize, 10); - if (isNaN(ret.maxBatchSize)) { - throw new Error("Max batch size must be a number, found: " + ret.maxBatchSize); + // If manual batching, don't complain about batch settings from defaultConfig + if (!ret.autoFlush) { + ret.maxBatchCount = utils.orByProp("maxBatchCount", config, ret) || 0; + ret.maxBatchSize = utils.orByProp("maxBatchSize", config, ret) || 0; + ret.batchInterval = utils.orByProp("batchInterval", config, ret) || 0; } - else if (ret.maxBatchSize < 0) { - throw new Error("Max batch size must be a positive number, found: " + ret.maxBatchSize); + else { + ret.maxBatchCount = utils.orByProp("maxBatchCount", config, ret, defaultConfig); + ret.maxBatchSize = utils.orByProp("maxBatchSize", config, ret, defaultConfig); + ret.batchInterval = utils.orByProp("batchInterval", config, ret, defaultConfig); } - ret.batchInterval = config.batchInterval || ret.batchInterval || defaultConfig.batchInterval; - ret.batchInterval = parseInt(ret.batchInterval, 10); - if (isNaN(ret.batchInterval)) { - throw new Error("Batch interval must be a number, found: " + ret.batchInterval); - } - else if (ret.batchInterval < 0) { - throw new Error("Batch interval must be a positive number, found: " + ret.batchInterval); + ret.maxBatchCount = utils.validatePositiveInt(ret.maxBatchCount, "Max batch count"); + ret.maxBatchSize = utils.validatePositiveInt(ret.maxBatchSize, "Max batch size"); + ret.batchInterval = utils.validatePositiveInt(ret.batchInterval, "Batch interval"); + + // Error if autoFlush is off and any batching settings are set + if (!ret.autoFlush && (ret.batchInterval || ret.maxBatchSize || ret.maxBatchCount)) { + throw new Error("Autoflush is disabled, cannot configure batching settings."); } // Has the interval timer not started, and needs to be started? @@ -312,19 +303,6 @@ SplunkLogger.prototype._initializeConfig = function(config) { else if (this._timerID && (this._timerDuration < 0 || !ret.autoFlush)) { this._disableTimer(); } - - if (!config.hasOwnProperty("port")) { - ret.port = ret.port || defaultConfig.port; - } - else { - ret.port = parseInt(config.port, 10); - if (isNaN(ret.port)) { - throw new Error("Port must be an integer, found: " + ret.port); - } - } - if (ret.port < 1000 || ret.port > 65535) { - throw new Error("Port must be an integer between 1000 and 65535, found: " + ret.port); - } } return ret; }; @@ -338,25 +316,16 @@ SplunkLogger.prototype._initializeConfig = function(config) { * @returns {object} requestOptions * @private */ -SplunkLogger.prototype._initializeRequestOptions = function(config, options) { - var ret = {}; - for (var key in defaultRequestOptions) { - if (defaultRequestOptions.hasOwnProperty(key)) { - ret[key] = defaultRequestOptions[key]; - } - } +SplunkLogger.prototype._initializeRequestOptions = function(options) { + var ret = utils.copyObject(options || defaultRequestOptions); - config = config || this.config || defaultConfig; - options = options || ret; - - ret.url = config.protocol + "://" + config.host + ":" + config.port + config.path; - ret.json = options.hasOwnProperty("json") ? options.json : ret.json; - ret.strictSSL = options.strictSSL || ret.strictSSL; - ret.headers = options.headers || {}; - if (config.token) { - ret.headers.Authorization = "Splunk " + config.token; + if (options) { + ret.json = options.hasOwnProperty("json") ? options.json : defaultRequestOptions.json; + ret.strictSSL = options.strictSSL || defaultRequestOptions.strictSSL; } + ret.headers = ret.headers || {}; + return ret; }; @@ -366,7 +335,7 @@ SplunkLogger.prototype._initializeRequestOptions = function(config, options) { * @private * @throws Will throw an error if the message parameter is malformed. */ -SplunkLogger.prototype._initializeMessage = function(message) { +SplunkLogger.prototype._validateMessage = function(message) { if (typeof message === "undefined" || message === null) { throw new Error("Message argument is required."); } @@ -375,7 +344,7 @@ SplunkLogger.prototype._initializeMessage = function(message) { /** * Initialized metadata, if context.metadata is falsey or empty, - * return an empty object; + * return an empty object. * * @param {object} context * @returns {object} metadata @@ -383,7 +352,7 @@ SplunkLogger.prototype._initializeMessage = function(message) { */ SplunkLogger.prototype._initializeMetadata = function(context) { var metadata = {}; - if (context.hasOwnProperty("metadata")) { + if (context && context.hasOwnProperty("metadata")) { if (context.metadata.hasOwnProperty("time")) { metadata.time = context.metadata.time; } @@ -404,7 +373,7 @@ SplunkLogger.prototype._initializeMetadata = function(context) { }; /** - * Initializes a context. + * Initializes a context object. * * @param context * @returns {object} context @@ -422,15 +391,9 @@ SplunkLogger.prototype._initializeContext = function(context) { throw new Error("Context argument must have the message property set."); } - // _initializeConfig will throw an error config or this.config is - // undefined, or doesn't have at least the token property set - context.config = this._initializeConfig(context.config || this.config); + context.message = this._validateMessage(context.message); - context.requestOptions = this._initializeRequestOptions(context.config, context.requestOptions); - - context.message = this._initializeMessage(context.message); - - context.severity = context.severity || SplunkLogger.prototype.levels.INFO; + context.severity = context.severity || defaultConfig.level; context.metadata = context.metadata || this._initializeMetadata(context); @@ -453,11 +416,8 @@ SplunkLogger.prototype._makeBody = function(context) { var body = this._initializeMetadata(context); var time = utils.formatTime(body.time || Date.now()); body.time = time.toString(); - body.event = { - message: context.message, - severity: context.severity || SplunkLogger.prototype.levels.INFO - }; - + + body.event = this.eventFormatter(context.message, context.severity || defaultConfig.level); return body; }; @@ -492,21 +452,6 @@ SplunkLogger.prototype.use = function(middleware) { } }; -/** - * Estimates the size in bytes of the events in the - * queue parameter as if they were sent - * in a single HTTP request. - * - * @returns {number} the estimated size in bytes. - */ -SplunkLogger.prototype.calculateBatchSize = function() { - var size = 0; - for (var i = 0; i < this.eventSizes.length; i++) { - size += this.eventSizes[i]; - } - return size; -}; - /** * Makes an HTTP POST to the configured server. * @@ -528,6 +473,22 @@ SplunkLogger.prototype._post = function(requestOptions, callback) { SplunkLogger.prototype._sendEvents = function(context, callback) { callback = callback || /* istanbul ignore next*/ function(){}; + // Initialize the config once more to avoid undefined vals below + this.config = this._initializeConfig(this.config); + + // Makes a copy of the request options so we can set the body + var requestOptions = this._initializeRequestOptions(this.requestOptions); + requestOptions.body = this._validateMessage(context.message); + requestOptions.headers["Authorization"] = "Splunk " + this.config.token; + // Manually set the content-type header for batched requests, default is application/json + // since json is set to true. + requestOptions.headers["Content-Type"] = "application/x-www-form-urlencoded"; + requestOptions.url = this.config.protocol + "://" + this.config.host + ":" + this.config.port + this.config.path; + + + // Initialize the context again, right before using it + context = this._initializeContext(context); + var that = this; var splunkError = null; // Errors returned by Splunk @@ -542,10 +503,10 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { utils.whilst( function() { // Continue if we can (re)try - return numRetries++ <= context.config.maxRetries; + return numRetries++ <= that.config.maxRetries; }, function(done) { - that._post(context.requestOptions, function(err, resp, body) { + that._post(requestOptions, function(err, resp, body) { // Store the latest error, response & body splunkError = null; requestError = err; @@ -558,8 +519,8 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { splunkError.code = body.code; } - // Request error (non-200), retry if numRetries hasn't exceeded the limit - if (requestError && numRetries <= context.config.maxRetries) { + // Retry if no Splunk error, a non-200 request response, and numRetries hasn't exceeded the limit + if (!splunkError && requestError && numRetries <= that.config.maxRetries) { utils.expBackoff({attempt: numRetries}, done); } else { @@ -580,7 +541,7 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { }; /** - * Sends or queues data to be sent based on context.config.autoFlush. + * Sends or queues data to be sent based on this.config.autoFlush. * Default behavior is to send immediately. * * @example @@ -616,10 +577,6 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { * * @param {object} context - An object with at least the data property. * @param {(object|string|Array|number|bool)} context.message - Data to send to Splunk. - * @param {object} [context.requestOptions] - Defaults are {json:true, strictSSL:false}. Additional - * options to pass to {@link https://github.com/request/request#requestpost|request.post()}. - * See the {@link http://github.com/request/request|request documentation} for all available options. - * @param {object} [context.config] - See {@link SplunkLogger} for default values. * @param {string} [context.severity=info] - Severity level of this event. * @param {object} [context.metadata] - Metadata for this event. * @param {string} [context.metadata.host] - If not specified, Splunk will decide the value. @@ -632,27 +589,24 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { * @public */ SplunkLogger.prototype.send = function(context, callback) { - callback = callback || function(){}; context = this._initializeContext(context); // Store the context, and its estimated length this.contextQueue.push(context); - this.eventSizes.push(Buffer.byteLength(JSON.stringify(this._makeBody(context)), "utf8")); + this.eventsBatchSize += Buffer.byteLength(JSON.stringify(this._makeBody(context)), "utf8"); - var batchOverSize = this.calculateBatchSize() > context.config.maxBatchSize; - var batchOverCount = this.contextQueue.length >= context.config.maxBatchCount && context.config.maxBatchCount > 0; + var batchOverSize = this.eventsBatchSize > this.config.maxBatchSize; + var batchOverCount = this.contextQueue.length >= this.config.maxBatchCount; // Only flush if not using manual batching, and if the contextQueue is too large or has many events - if (context.config.autoFlush && !this._timerID && (batchOverSize || batchOverCount)) { + if (this.config.autoFlush && (batchOverSize || batchOverCount)) { this.flush(callback || function(){}); } }; /** - * Manually send events in this.contextQueue to Splunk, after + * Manually send all events in this.contextQueue to Splunk, after * chaining any functions in this.middlewares. - * Auto flush settings will be used from this.config.autoFlush, - * ignoring auto flush settings every on every context in this.contextQueue. * * @param {function} [callback] - A callback function: function(err, response, body). * @public @@ -662,36 +616,21 @@ SplunkLogger.prototype.flush = function(callback) { var context = {}; - var batchOverSize = this.config.maxBatchSize > 0 && this.calculateBatchSize() > this.config.maxBatchSize; - var batchOverCount = this.config.maxBatchCount > 0 && this.contextQueue.length >= this.config.maxBatchCount; - var isBatched = !this.config.autoFlush || (this.config.autoFlush && (batchOverSize || batchOverCount || this._timerID)); - - // Empty the queue if manual/interval/size/count batching - if (isBatched) { - var queue = this.contextQueue; - this.contextQueue = []; - this.eventSizes = []; - var data = ""; - for (var i = 0; i < queue.length; i++) { - data += JSON.stringify(this._makeBody(queue[i])); - } - context.message = data; - } - // Just take the oldest event in the queue - else { - // TODO: handle case of multiple events with autoFlush off, flushing fast - context = this.contextQueue.pop(); - this.eventSizes.pop(); + // Empty the queue, reset the eventsBatchSize + var queue = this.contextQueue; + this.contextQueue = []; + this.eventsBatchSize = 0; + var data = ""; + for (var i = 0; i < queue.length; i++) { + data += JSON.stringify(this._makeBody(queue[i])); } + context.message = data; - // Initialize the context, then manually set the data + // Initialize the context before invoking middleware functions context = this._initializeContext(context); // Copy over the middlewares - var callbacks = []; - for (var j = 0; j < this.middlewares.length; j++) { - callbacks[j] = this.middlewares[j]; - } + var callbacks = utils.copyArray(this.middlewares); // Send the data to the first middleware callbacks.unshift(function(cb) { @@ -705,7 +644,7 @@ SplunkLogger.prototype.flush = function(callback) { // do this automatically, but the || notation adds a bit of clarity. context = passedContext || context; - // Errors from any of the middleware callbacks will fall through to here + // Errors from any of the middleware callbacks will fall through to here. // If the context is modified at any point the error callback will get it also // event if next("error"); is called w/o the context parameter! // This works because context inside & outside the scope of this function @@ -714,22 +653,6 @@ SplunkLogger.prototype.flush = function(callback) { that.error(err, context); } else { - // Validate the context again, right before using it - context = that._initializeContext(context); - context.requestOptions.headers["Authorization"] = "Splunk " + context.config.token; - - if (isBatched) { - // Don't run _makeBody since we've already done that for batching - // Manually set the content-type header for batched requests, default is application/json - // since json is set to true. - context.requestOptions.headers["content-type"] = "application/x-www-form-urlencoded"; - } - else { - context.message = that._makeBody(context); - } - - context.requestOptions.body = context.message; - that._sendEvents(context, callback); } }); diff --git a/test/test_config.js b/test/test_config.js index 9e3f6d4..686a78d 100644 --- a/test/test_config.js +++ b/test/test_config.js @@ -100,7 +100,7 @@ describe("SplunkLogger", function() { } catch (err) { assert.ok(err); - assert.strictEqual(err.message, "Port must be an integer, found: NaN"); + assert.strictEqual(err.message, "Port must be a number, found: NaN"); } }); it("should correctly parse port with leading zero", function() { @@ -144,7 +144,18 @@ describe("SplunkLogger", function() { assert.strictEqual(true, logger.config.autoFlush); assert.strictEqual(0, logger.config.maxRetries); assert.strictEqual(0, logger.config.batchInterval); - assert.strictEqual(0, logger.config.maxBatchSize); + assert.strictEqual(10240, logger.config.maxBatchSize); + + var expectedRO = { + json: true, + strictSSL: false, + headers: {} + }; + assert.ok(logger.hasOwnProperty("requestOptions")); + assert.strictEqual(Object.keys(logger.requestOptions).length, 3); + assert.strictEqual(expectedRO.json, logger.requestOptions.json); + assert.strictEqual(expectedRO.strictSSL, logger.requestOptions.strictSSL); + assert.strictEqual(Object.keys(expectedRO.headers).length, Object.keys(logger.requestOptions.headers).length); }); it("should set remaining defaults when setting config with token, autoFlush off, & level", function() { var config = { @@ -165,7 +176,7 @@ describe("SplunkLogger", function() { assert.strictEqual(8088, logger.config.port); assert.strictEqual(0, logger.config.maxRetries); }); - it("should error when _enableInterval(NaN)", function() { + it("should error when _enableTimer(NaN)", function() { var config = { token: "a-token-goes-here-usually" }; @@ -177,7 +188,7 @@ describe("SplunkLogger", function() { } catch (err) { assert.ok(err); - assert.strictEqual("Batch interval must be a number, found: not a number", err.message); + assert.strictEqual("Batch interval must be a number, found: NaN", err.message); } }); it("should error when batchInterval=NaN", function() { @@ -253,27 +264,21 @@ describe("SplunkLogger", function() { assert.strictEqual(8088, logger.config.port); assert.strictEqual(0, logger.config.maxRetries); }); - it("should not set a batch interval timer with autoFlush off, & batchInterval set", function() { + it("should error trying set a batch interval timer with autoFlush off, & batchInterval set", function() { var config = { token: "a-token-goes-here-usually", batchInterval: 100, autoFlush: false }; - var logger = new SplunkLogger(config); - assert.ok(logger); - assert.ok(!logger._timerID); - - assert.strictEqual(config.token, logger.config.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); - assert.strictEqual("localhost", logger.config.host); - assert.strictEqual("/services/collector/event/1.0", logger.config.path); - assert.strictEqual("https", logger.config.protocol); - assert.strictEqual("info", logger.config.level); - assert.strictEqual(false, logger.config.autoFlush); - assert.strictEqual(100, logger.config.batchInterval); - assert.strictEqual(8088, logger.config.port); - assert.strictEqual(0, logger.config.maxRetries); + try { + var logger = new SplunkLogger(config); + assert.ok(!logger, "Expected an error."); + } + catch (err) { + assert.ok(err); + assert.strictEqual(err.message, "Autoflush is disabled, cannot configure batching settings."); + } }); it("should error when maxBatchCount=NaN", function() { var config = { @@ -601,7 +606,7 @@ describe("SplunkLogger", function() { } catch (err) { assert.ok(err); - assert.strictEqual(err.message, "Port must be an integer, found: NaN"); + assert.strictEqual(err.message, "Port must be a number, found: NaN"); } }); it("should correctly parse port with leading zero", function() { @@ -769,122 +774,52 @@ describe("SplunkLogger", function() { it("should get defaults with no args", function() { var options = SplunkLogger.prototype._initializeRequestOptions(); assert.ok(options); + assert.ok(Object.keys(options).length, 3); assert.strictEqual(options.json, true); assert.strictEqual(options.strictSSL, false); - assert.strictEqual(options.url, "https://localhost:8088/services/collector/event/1.0"); - assert.ok(options.hasOwnProperty("headers")); + assert.ok(options.headers); assert.strictEqual(Object.keys(options.headers).length, 0); - assert.ok(!options.headers.hasOwnProperty("Authorization")); }); - it("should create default options with token in config", function() { - var config = { - token: "some-value" + it("should get defaults with none of the default props configured", function() { + var optionsOriginal = { + something: "here", + value: 1234 }; - // Get the defaults because we're passing in a config - config = SplunkLogger.prototype._initializeConfig(config); - - var options = SplunkLogger.prototype._initializeRequestOptions(config); + var options = SplunkLogger.prototype._initializeRequestOptions(optionsOriginal); assert.ok(options); - assert.strictEqual(options.url, "https://localhost:8088/services/collector/event/1.0"); - assert.ok(options.headers); - assert.ok(options.headers.hasOwnProperty("Authorization")); - assert.ok(options.headers.Authorization, "Splunk " + config.token); + assert.ok(Object.keys(options).length, 5); assert.strictEqual(options.json, true); assert.strictEqual(options.strictSSL, false); - }); - it("should create options with full config", function() { - var config = { - token: "some-value", - protocol: "http", - host: "splunk.local", - port: 1234, - path: "/services/collector/custom/1.0" - }; - config = SplunkLogger.prototype._initializeConfig(config); - - var options = SplunkLogger.prototype._initializeRequestOptions(config); - assert.ok(options); - assert.strictEqual(options.url, "http://splunk.local:1234/services/collector/custom/1.0"); + assert.strictEqual(options.something, optionsOriginal.something); + assert.strictEqual(options.value, optionsOriginal.value); assert.ok(options.headers); - assert.ok(options.headers.hasOwnProperty("Authorization")); - assert.ok(options.headers.Authorization, "Splunk " + config.token); - assert.strictEqual(options.json, true); - assert.strictEqual(options.strictSSL, false); - }); - it("should create options with full config, empty options", function() { - var config = { - token: "some-value", - protocol: "http", - host: "splunk.local", - port: 1234, - path: "/services/collector/custom/1.0" - }; - config = SplunkLogger.prototype._initializeConfig(config); - - var options = SplunkLogger.prototype._initializeRequestOptions(config, {}); - assert.ok(options); - assert.strictEqual(options.url, "http://splunk.local:1234/services/collector/custom/1.0"); - assert.ok(options.headers); - assert.ok(options.headers.hasOwnProperty("Authorization")); - assert.ok(options.headers.Authorization, "Splunk " + config.token); - assert.strictEqual(options.json, true); - assert.strictEqual(options.strictSSL, false); + assert.strictEqual(Object.keys(options.headers).length, 0); }); - - it("should create options with full config, & full options", function() { - var config = { - token: "some-value", - protocol: "http", - host: "splunk.local", - port: 1234, - path: "/services/collector/custom/1.0" - }; - config = SplunkLogger.prototype._initializeConfig(config); - - var initialOptions = { + it("should get defaults with non-default values", function() { + var optionsOriginal = { json: false, strictSSL: true, - url: "should be overwritten", headers: { - Custom: "header-value", - Authorization: "Should be overwritten" - } + Authorization: "nothing" + }, + dummy: "value" }; - var options = SplunkLogger.prototype._initializeRequestOptions(config, initialOptions); + var options = SplunkLogger.prototype._initializeRequestOptions(optionsOriginal); assert.ok(options); - assert.strictEqual(options.url, "http://splunk.local:1234/services/collector/custom/1.0"); - assert.ok(options.headers); - assert.ok(options.headers.hasOwnProperty("Custom")); - assert.strictEqual(options.headers.Custom, initialOptions.headers.Custom); - assert.ok(options.headers.hasOwnProperty("Authorization")); - assert.ok(options.headers.Authorization, "Splunk " + config.token); + assert.ok(Object.keys(options).length, 4); assert.strictEqual(options.json, false); assert.strictEqual(options.strictSSL, true); - }); - it("should create default options with token in config", function() { - Object.prototype.someproperty = "ignore"; - var config = { - token: "some-value" - }; - // Get the defaults because we're passing in a config - config = SplunkLogger.prototype._initializeConfig(config); - - var options = SplunkLogger.prototype._initializeRequestOptions(config); - assert.ok(options); - assert.ok(!options.hasOwnProperty("someproperty")); - assert.strictEqual(options.url, "https://localhost:8088/services/collector/event/1.0"); + assert.strictEqual(options.dummy, "value"); assert.ok(options.headers); - assert.ok(options.headers.hasOwnProperty("Authorization")); - assert.ok(options.headers.Authorization, "Splunk " + config.token); - assert.strictEqual(options.json, true); - assert.strictEqual(options.strictSSL, false); + assert.strictEqual(Object.keys(options.headers).length, 1); + assert.strictEqual(options.headers["Authorization"], "nothing"); }); }); - describe("_initializeMessage", function() { + describe("_validateMessage", function() { it("should error with no args", function() { try { - SplunkLogger.prototype._initializeMessage(); + SplunkLogger.prototype._validateMessage(); assert.ok(false, "Expected an error."); } catch (err) { @@ -894,7 +829,7 @@ describe("SplunkLogger", function() { }); it("should leave string intact", function() { var beforeMessage = "something"; - var afterMessage = SplunkLogger.prototype._initializeMessage(beforeMessage); + var afterMessage = SplunkLogger.prototype._validateMessage(beforeMessage); assert.ok(afterMessage); assert.strictEqual(afterMessage, beforeMessage); }); @@ -930,52 +865,17 @@ describe("SplunkLogger", function() { assert.strictEqual(err.message, "Context argument must have the message property set."); } }); - it("should error with data only", function() { - try { - var context = { - message: "something" - }; - SplunkLogger.prototype._initializeContext(context); - assert.ok(false, "Expected an error."); - } - catch(err) { - assert.ok(err); - assert.strictEqual(err.message, "Config is required."); - } - }); - it("should succeed with default context, specifying data & config token", function() { + it("should succeed with default context, specifying a string message", function() { var context = { - message: "some data", - config: { - token: "a-token-goes-here-usually" - } + message: "some data" }; var initialized = SplunkLogger.prototype._initializeContext(context); var data = initialized.message; - var config = initialized.config; - var requestOptions = initialized.requestOptions; assert.ok(initialized); assert.ok(data); assert.strictEqual(data, context.message); - - assert.ok(config); - assert.strictEqual(config.token, context.config.token); - assert.strictEqual(config.name, "splunk-javascript-logging/0.8.0"); - assert.strictEqual(config.host, "localhost"); - assert.strictEqual(config.path, "/services/collector/event/1.0"); - assert.strictEqual(config.protocol, "https"); - assert.strictEqual(config.level, "info"); - assert.strictEqual(config.port, 8088); - - assert.ok(requestOptions); - assert.strictEqual(requestOptions.json, true); - assert.strictEqual(requestOptions.strictSSL, false); - assert.strictEqual(requestOptions.url, "https://localhost:8088/services/collector/event/1.0"); - assert.ok(requestOptions.hasOwnProperty("headers")); - assert.strictEqual(Object.keys(requestOptions.headers).length, 1); - assert.strictEqual(requestOptions.headers.Authorization, "Splunk " + context.config.token); }); }); describe("constructor + _initializeConfig", function() { diff --git a/test/test_send.js b/test/test_send.js index 4d8b2ea..9ae5513 100644 --- a/test/test_send.js +++ b/test/test_send.js @@ -65,10 +65,11 @@ function unmute() { console.log = ___log; } -describe("SplunkLogger _makedata", function() { +describe("SplunkLogger _makeBody", function() { it("should error with no args", function() { try { - SplunkLogger.prototype._makeBody(); + var logger = new SplunkLogger({token: "token-goes-here"}); + logger._makeBody(); assert.ok(false, "Expected an error."); } catch(err) { @@ -80,27 +81,32 @@ describe("SplunkLogger _makedata", function() { var context = { message: "something" }; - var body = SplunkLogger.prototype._makeBody(context); + var logger = new SplunkLogger({token: "token-goes-here"}); + var body = logger._makeBody(context); assert.ok(body); assert.ok(body.hasOwnProperty("event")); assert.strictEqual(Object.keys(body).length, 2); - assert.ok(body.event.hasOwnProperty("message")); - assert.strictEqual(body.event.message, context.message); - assert.strictEqual(body.event.severity, "info"); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.strictEqual(event.message, context.message); + assert.strictEqual(event.severity, "info"); }); it("should objectify data as array, without severity param", function() { var context = { message: ["something"] }; - var body = SplunkLogger.prototype._makeBody(context); + var logger = new SplunkLogger({token: "token-goes-here"}); + var body = logger._makeBody(context); assert.ok(body); assert.ok(body.hasOwnProperty("event")); assert.strictEqual(Object.keys(body).length, 2); - assert.ok(body.event.hasOwnProperty("message")); - assert.strictEqual(body.event.message, context.message); - assert.strictEqual(body.event.severity, "info"); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.strictEqual(event.message.length, context.message.length); + assert.strictEqual(event.message[0], context.message[0]); + assert.strictEqual(event.severity, "info"); }); it("should objectify data as object, without severity param", function() { var context = { @@ -108,43 +114,50 @@ describe("SplunkLogger _makedata", function() { prop: "something" } }; - var body = SplunkLogger.prototype._makeBody(context); + var logger = new SplunkLogger({token: "token-goes-here"}); + var body = logger._makeBody(context); assert.ok(body); assert.ok(body.hasOwnProperty("event")); assert.strictEqual(Object.keys(body).length, 2); - assert.ok(body.event.hasOwnProperty("message")); - assert.strictEqual(body.event.message, context.message); - assert.strictEqual(body.event.message.prop, "something"); - assert.strictEqual(body.event.severity, "info"); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.strictEqual(Object.keys(event.message).length, Object.keys(context.message).length); + assert.strictEqual(event.message.prop, "something"); + assert.strictEqual(event.severity, "info"); }); it("should objectify data as string, with severity param", function() { var context = { message: "something", severity: "urgent" }; - var body = SplunkLogger.prototype._makeBody(context); + var logger = new SplunkLogger({token: "token-goes-here"}); + var body = logger._makeBody(context); assert.ok(body); assert.ok(body.hasOwnProperty("event")); assert.strictEqual(Object.keys(body).length, 2); - assert.ok(body.event.hasOwnProperty("message")); - assert.strictEqual(body.event.message, context.message); - assert.strictEqual(body.event.severity, "urgent"); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.strictEqual(event.message, context.message); + assert.strictEqual(event.severity, "urgent"); }); it("should objectify data as array, with severity param", function() { var context = { message: ["something"], severity: "urgent" }; - var body = SplunkLogger.prototype._makeBody(context); + var logger = new SplunkLogger({token: "token-goes-here"}); + var body = logger._makeBody(context); assert.ok(body); assert.ok(body.hasOwnProperty("event")); assert.strictEqual(Object.keys(body).length, 2); - assert.ok(body.event.hasOwnProperty("message")); - assert.strictEqual(body.event.message, context.message); - assert.strictEqual(body.event.severity, "urgent"); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.strictEqual(event.message.length, context.message.length); + assert.strictEqual(event.message[0], context.message[0]); + assert.strictEqual(event.severity, "urgent"); }); it("should objectify data as object, with severity param", function() { var context = { @@ -153,29 +166,31 @@ describe("SplunkLogger _makedata", function() { }, severity: "urgent" }; - var body = SplunkLogger.prototype._makeBody(context); + var logger = new SplunkLogger({token: "token-goes-here"}); + var body = logger._makeBody(context); assert.ok(body); assert.ok(body.hasOwnProperty("event")); assert.strictEqual(Object.keys(body).length, 2); - assert.ok(body.event.hasOwnProperty("message")); - assert.strictEqual(body.event.message, context.message); - assert.strictEqual(body.event.message.prop, "something"); - assert.strictEqual(body.event.severity, "urgent"); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.strictEqual(Object.keys(event.message).length, Object.keys(context.message).length); + assert.strictEqual(event.message.prop, "something"); + assert.strictEqual(event.severity, "urgent"); }); }); describe("SplunkLogger send (integration tests)", function() { - describe("using default middleware", function () { + describe("using no middleware", function () { it("should error with bad token", function(done) { var config = { - token: "token-goes-here" + token: "token-goes-here", + maxBatchCount: 1 }; var logger = new SplunkLogger(config); var data = "something"; var context = { - config: config, message: data }; @@ -186,8 +201,15 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(err); assert.strictEqual(err.message, invalidTokenBody.text); assert.strictEqual(err.code, invalidTokenBody.code); + assert.ok(errContext); - assert.strictEqual(errContext.message, context.message); + var body = JSON.parse(errContext.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + assert.strictEqual(Object.keys(body).length, 2); + var event = body.event; + assert.strictEqual(event.message, context.message); + assert.strictEqual(event.severity, "info"); }; logger.send(context, function(err, resp, body) { @@ -202,7 +224,8 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should send without callback", function(done) { var config = { - token: configurationFile.token + token: configurationFile.token, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -210,22 +233,22 @@ describe("SplunkLogger send (integration tests)", function() { var data = "something"; var context = { - config: config, message: data }; assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); logger.send(context); setTimeout(function() { assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); done(); }, 500); }); it("should succeed with valid token", function(done) { var config = { - token: configurationFile.token + token: configurationFile.token, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -233,7 +256,6 @@ describe("SplunkLogger send (integration tests)", function() { var data = "something"; var context = { - config: config, message: data }; @@ -248,7 +270,8 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed with valid token, using custom time", function(done) { var config = { - token: configurationFile.token + token: configurationFile.token, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -256,7 +279,6 @@ describe("SplunkLogger send (integration tests)", function() { var data = "something else"; var context = { - config: config, message: data, metadata: { time: new Date("January 1, 2015") @@ -275,7 +297,8 @@ describe("SplunkLogger send (integration tests)", function() { // TODO: test unsuccessfully sending to another index with specific index token settings it("should succeed with valid token, sending to a different index", function(done) { var config = { - token: configurationFile.token + token: configurationFile.token, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -283,7 +306,6 @@ describe("SplunkLogger send (integration tests)", function() { var data = "something else"; var context = { - config: config, message: data, metadata: { index: "default" @@ -305,7 +327,8 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed with valid token, changing source", function(done) { var config = { - token: configurationFile.token + token: configurationFile.token, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -313,7 +336,6 @@ describe("SplunkLogger send (integration tests)", function() { var data = "something else"; var context = { - config: config, message: data, metadata: { source: "_____new____source" @@ -331,7 +353,8 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed with valid token, changing sourcetype", function(done) { var config = { - token: configurationFile.token + token: configurationFile.token, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -339,7 +362,6 @@ describe("SplunkLogger send (integration tests)", function() { var data = "something else"; var context = { - config: config, message: data, metadata: { sourcetype: "_____new____sourcetype" @@ -357,7 +379,8 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed with valid token, changing host", function(done) { var config = { - token: configurationFile.token + token: configurationFile.token, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -365,7 +388,6 @@ describe("SplunkLogger send (integration tests)", function() { var data = "something else"; var context = { - config: logger.config, message: data, metadata: { host: "some.other.host" @@ -381,33 +403,10 @@ describe("SplunkLogger send (integration tests)", function() { done(); }); }); - it("should succeed with different valid token passed through context", function(done) { - var config = { - token: "invalid-token" - }; - - var logger = new SplunkLogger(config); - - var data = "something"; - - config.token = configurationFile.token; - var context = { - config: config, - message: data - }; - - logger.send(context, function(err, resp, body) { - assert.ok(!err); - assert.strictEqual(resp.headers["content-type"], "application/json; charset=UTF-8"); - assert.strictEqual(resp.body, body); - assert.strictEqual(body.text, successBody.text); - assert.strictEqual(body.code, successBody.code); - done(); - }); - }); it("should succeed with valid token", function(done) { var config = { - token: configurationFile.token + token: configurationFile.token, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -415,7 +414,6 @@ describe("SplunkLogger send (integration tests)", function() { var data = "something"; var context = { - config: config, message: data }; @@ -430,7 +428,8 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed without token passed through context", function(done) { var config = { - token: configurationFile.token + token: configurationFile.token, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -455,14 +454,14 @@ describe("SplunkLogger send (integration tests)", function() { it("should fail on wrong protocol (assumes HTTP is invalid)", function(done) { var config = { token: configurationFile.token, - protocol: "http" + protocol: "http", + maxBatchCount: 1 }; var logger = new SplunkLogger(config); var data = "something"; var context = { - config: config, message: data }; @@ -473,8 +472,15 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(err); assert.strictEqual(err.message, "socket hang up"); assert.strictEqual(err.code, "ECONNRESET"); + assert.ok(errContext); - assert.strictEqual(errContext, context); + var body = JSON.parse(errContext.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + assert.strictEqual(Object.keys(body).length, 2); + var event = body.event; + assert.strictEqual(event.message, context.message); + assert.strictEqual(event.severity, "info"); }; logger.send(context, function(err, resp, body) { @@ -490,14 +496,14 @@ describe("SplunkLogger send (integration tests)", function() { it("should fail on wrong Splunk server", function(done) { var config = { token: configurationFile.token, - url: "https://something-so-invalid-that-it-should-never-exist.xyz:12345/junk" + url: "https://something-so-invalid-that-it-should-never-exist.xyz:12345/junk", + maxBatchCount: 1 }; var logger = new SplunkLogger(config); var data = "something"; var context = { - config: config, message: data }; @@ -508,8 +514,15 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(err); assert.strictEqual(err.message, "getaddrinfo ENOTFOUND"); assert.strictEqual(err.code, "ENOTFOUND"); + assert.ok(errContext); - assert.strictEqual(errContext, context); + var body = JSON.parse(errContext.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + assert.strictEqual(Object.keys(body).length, 2); + var event = body.event; + assert.strictEqual(event.message, context.message); + assert.strictEqual(event.severity, "info"); }; logger.send(context, function(err, resp, body) { @@ -525,14 +538,14 @@ describe("SplunkLogger send (integration tests)", function() { it("should succeed with valid token, using non-default url", function(done) { var config = { token: configurationFile.token, - url: "https://localhost:8088/services/collector/event/1.0" + url: "https://localhost:8088/services/collector/event/1.0", + maxBatchCount: 1 }; var logger = new SplunkLogger(config); var data = "something"; var context = { - config: config, message: data }; @@ -548,17 +561,16 @@ describe("SplunkLogger send (integration tests)", function() { it("should error with valid token, using strict SSL", function(done) { var config = { token: configurationFile.token, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); + logger.requestOptions.strictSSL = true; + var data = "something"; var context = { - config: config, - message: data, - requestOptions: { - strictSSL: true - } + message: data }; var run = false; @@ -568,7 +580,16 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(err); assert.strictEqual(err.message, "SELF_SIGNED_CERT_IN_CHAIN"); assert.ok(errContext); - assert.strictEqual(errContext, context); + + var body = JSON.parse(errContext.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.ok(event.hasOwnProperty("severity")); + + assert.strictEqual(event.message, context.message); + assert.strictEqual(event.severity, "info"); }; logger.send(context, function(err, resp, body) { @@ -582,23 +603,24 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should send 2 events with valid token, w/o callbacks", function(done) { var config = { - token: configurationFile.token + token: configurationFile.token, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); var data = "batched event"; var context = { - config: config, message: data }; var sent = 0; // Wrap sendevents to ensure it gets called + var sendEvents = logger._sendEvents; logger._sendEvents = function(cont, cb) { sent++; - SplunkLogger.prototype._sendEvents(cont, cb); + sendEvents(cont, cb); }; logger.send(context); @@ -609,7 +631,7 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); assert.strictEqual(sent, 2); done(); }, 1000); @@ -635,7 +657,7 @@ describe("SplunkLogger send (integration tests)", function() { }; assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); logger.flush(function(err, resp, body) { assert.ok(!err); assert.ok(run); @@ -644,7 +666,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.strictEqual(body.text, noDataBody.text); assert.strictEqual(body.code, noDataBody.code); assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); done(); }); }); @@ -666,12 +688,10 @@ describe("SplunkLogger send (integration tests)", function() { // Nothing should be sent if queue is empty assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); - assert.strictEqual(logger.calculateBatchSize(), 0); + assert.strictEqual(logger.eventsBatchSize, 0); logger.flush(); assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); - assert.strictEqual(logger.calculateBatchSize(), 0); + assert.strictEqual(logger.eventsBatchSize, 0); }); it("should flush a batch of 1 event with valid token", function(done) { var config = { @@ -683,16 +703,13 @@ describe("SplunkLogger send (integration tests)", function() { var data = this.test.fullTitle(); var context = { - config: config, message: data }; logger.send(context); assert.strictEqual(logger.contextQueue.length, 1); - assert.strictEqual(logger.eventSizes.length, 1); - assert.ok(logger.calculateBatchSize() > 0); - assert.strictEqual(logger.calculateBatchSize(), logger.eventSizes[0]); + assert.ok(logger.eventsBatchSize > 50); logger.flush(function(err, resp, body) { assert.ok(!err); assert.strictEqual(resp.headers["content-type"], "application/json; charset=UTF-8"); @@ -700,8 +717,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.strictEqual(body.text, successBody.text); assert.strictEqual(body.code, successBody.code); assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); - assert.strictEqual(logger.calculateBatchSize(), 0); + assert.strictEqual(logger.eventsBatchSize, 0); done(); }); }); @@ -715,7 +731,6 @@ describe("SplunkLogger send (integration tests)", function() { var data = this.test.fullTitle(); var context = { - config: config, message: data }; @@ -723,8 +738,7 @@ describe("SplunkLogger send (integration tests)", function() { logger.send(context); assert.strictEqual(logger.contextQueue.length, 2); - assert.strictEqual(logger.eventSizes.length, 2); - assert.strictEqual(logger.calculateBatchSize(), logger.eventSizes[0] * 2); + assert.ok(logger.eventsBatchSize > 100); logger.flush(function(err, resp, body) { assert.ok(!err); assert.strictEqual(resp.headers["content-type"], "application/json; charset=UTF-8"); @@ -732,8 +746,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.strictEqual(body.text, successBody.text); assert.strictEqual(body.code, successBody.code); assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); - assert.strictEqual(logger.calculateBatchSize(), 0); + assert.strictEqual(logger.eventsBatchSize, 0); done(); }); }); @@ -755,14 +768,20 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed using non-default middleware", function(done) { var config = { - token: "token-goes-here" + token: "token-goes-here", + maxBatchCount: 1 }; var middlewareCount = 0; function middleware(context, next) { middlewareCount++; - assert.strictEqual(context.message, "something"); + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + assert.strictEqual(Object.keys(body).length, 2); + var event = body.event; + assert.strictEqual(event.message, "something"); next(null, context); } @@ -782,7 +801,6 @@ describe("SplunkLogger send (integration tests)", function() { var initialData = "something"; var context = { - config: config, message: initialData }; @@ -803,7 +821,6 @@ describe("SplunkLogger send (integration tests)", function() { var initialData = "something"; var sentContext = { - config: config, message: initialData }; @@ -862,14 +879,20 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed using non-default middleware, without passing the context through", function(done) { var config = { - token: "token-goes-here" + token: "token-goes-here", + maxBatchCount: 1 }; var middlewareCount = 0; function middleware(context, next) { middlewareCount++; - assert.strictEqual(context.message, "something"); + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + assert.strictEqual(Object.keys(body).length, 2); + var event = body.event; + assert.strictEqual(event.message, "something"); next(null); } @@ -877,7 +900,14 @@ describe("SplunkLogger send (integration tests)", function() { logger.use(middleware); logger._sendEvents = function(context, next) { - assert.strictEqual(context, initialContext); + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + assert.strictEqual(Object.keys(body).length, 2); + var event = body.event; + assert.strictEqual(event.message, initialContext.message); + assert.strictEqual(event.severity, "info"); + var response = { headers: { "content-type": "application/json; charset=UTF-8", @@ -890,7 +920,6 @@ describe("SplunkLogger send (integration tests)", function() { var initialData = "something"; var initialContext = { - config: config, message: initialData }; @@ -905,21 +934,38 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed using 2 middlewares", function(done) { var config = { - token: "token-goes-here" + token: "token-goes-here", + maxBatchCount: 1 }; var middlewareCount = 0; function middleware(context, callback) { middlewareCount++; - assert.strictEqual(context.message, "somet??hing"); - context.message = encodeURIComponent(context.message); + + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + assert.strictEqual(Object.keys(body).length, 2); + var event = body.event; + assert.strictEqual(event.message, "some??thing"); + + body.event.message = encodeURIComponent(event.message); + context.message = JSON.stringify(body); callback(null, context); } function middleware2(context, callback) { middlewareCount++; - assert.strictEqual(context.message, "somet%3F%3Fhing"); + + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + assert.strictEqual(Object.keys(body).length, 2); + var event = body.event; + assert.strictEqual(event.message, "some%3F%3Fthing"); + assert.strictEqual(event.severity, "info"); + callback(null, context); } @@ -928,13 +974,14 @@ describe("SplunkLogger send (integration tests)", function() { logger.use(middleware2); logger._sendEvents = function(context, next) { - assert.ok(context.message.hasOwnProperty("event")); - assert.ok(context.message.hasOwnProperty("time")); - - var e = context.message.event; - assert.ok(e.hasOwnProperty("message")); - assert.ok(e.hasOwnProperty("severity")); - assert.strictEqual(e.message, "somet%3F%3Fhing"); + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + assert.strictEqual(Object.keys(body).length, 2); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.ok(event.hasOwnProperty("severity")); + assert.strictEqual(event.message, "some%3F%3Fthing"); var response = { headers: { @@ -946,9 +993,8 @@ describe("SplunkLogger send (integration tests)", function() { next(null, response, successBody); }; - var initialData = "somet??hing"; + var initialData = "some??thing"; var context = { - config: config, message: initialData }; @@ -963,29 +1009,55 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed using 3 middlewares", function(done) { var config = { - token: configurationFile.token + token: configurationFile.token, + maxBatchCount: 1 }; var middlewareCount = 0; function middleware(context, next) { middlewareCount++; - assert.strictEqual(context.message, "somet??hing"); - context.message = encodeURIComponent(context.message); + + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.ok(event.hasOwnProperty("severity")); + assert.strictEqual(event.message, "some??thing"); + assert.strictEqual(event.severity, "info"); + + body.event.message = encodeURIComponent(event.message); + context.message = JSON.stringify(body); next(null, context); } function middleware2(context, next) { middlewareCount++; - assert.strictEqual(context.message, "somet%3F%3Fhing"); - context.message = decodeURIComponent(context.message) + " changed"; - assert.strictEqual(context.message, "somet??hing changed"); + + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.ok(event.hasOwnProperty("severity")); + assert.strictEqual(event.message, "some%3F%3Fthing"); + assert.strictEqual(event.severity, "info"); + + body.event.message = decodeURIComponent(event.message) + " changed"; + context.message = JSON.stringify(body); next(null, context); } function middleware3(context, next) { middlewareCount++; - assert.strictEqual(context.message, "somet??hing changed"); + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.ok(event.hasOwnProperty("severity")); + assert.strictEqual(event.message, "some??thing changed"); next(null, context); } @@ -996,20 +1068,19 @@ describe("SplunkLogger send (integration tests)", function() { var _sendEvents = logger._sendEvents; logger._sendEvents = function(context, next) { - assert.ok(context.message.hasOwnProperty("event")); - assert.ok(context.message.hasOwnProperty("time")); - - var e = context.message.event; - assert.ok(e.hasOwnProperty("message")); - assert.ok(e.hasOwnProperty("severity")); - assert.strictEqual(e.message, "somet??hing changed"); + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.ok(event.hasOwnProperty("severity")); + assert.strictEqual(event.message, "some??thing changed"); _sendEvents(context, next); }; - var initialData = "somet??hing"; + var initialData = "some??thing"; var context = { - config: config, message: initialData }; @@ -1024,44 +1095,69 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed using 3 middlewares with data object", function(done) { var config = { - token: configurationFile.token + token: configurationFile.token, + maxBatchCount: 1 }; var middlewareCount = 0; function middleware(context, next) { middlewareCount++; - assert.strictEqual(context.message, initialData); - - assert.strictEqual(context.message.property, initialData.property); - assert.strictEqual(context.message.nested.object, initialData.nested.object); - assert.strictEqual(context.message.number, initialData.number); - assert.strictEqual(context.message.bool, initialData.bool); - - context.message.property = "new"; - context.message.bool = true; + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.ok(event.hasOwnProperty("severity")); + + assert.strictEqual(Object.keys(event.message).length, Object.keys(initialData).length); + assert.strictEqual(event.message.property, initialData.property); + assert.strictEqual(event.message.nested.object, initialData.nested.object); + assert.strictEqual(event.message.number, initialData.number); + assert.strictEqual(event.message.bool, initialData.bool); + + body.event.message.property = "new"; + body.event.message.bool = true; + context.message = JSON.stringify(body); next(null, context); } function middleware2(context, next) { middlewareCount++; - - assert.strictEqual(context.message.property, "new"); - assert.strictEqual(context.message.nested.object, initialData.nested.object); - assert.strictEqual(context.message.number, initialData.number); - assert.strictEqual(context.message.bool, true); - context.message.number = 789; + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.ok(event.hasOwnProperty("severity")); + + assert.strictEqual(Object.keys(event.message).length, Object.keys(initialData).length); + assert.strictEqual(event.message.property, "new"); + assert.strictEqual(event.message.nested.object, initialData.nested.object); + assert.strictEqual(event.message.number, initialData.number); + assert.strictEqual(event.message.bool, true); + + body.event.message.number = 789; + context.message = JSON.stringify(body); next(null, context); } function middleware3(context, next) { middlewareCount++; - - assert.strictEqual(context.message.property, "new"); - assert.strictEqual(context.message.nested.object, initialData.nested.object); - assert.strictEqual(context.message.number, 789); - assert.strictEqual(context.message.bool, true); + + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.ok(event.hasOwnProperty("severity")); + + assert.strictEqual(Object.keys(event.message).length, Object.keys(initialData).length); + assert.strictEqual(event.message.property, "new"); + assert.strictEqual(event.message.nested.object, initialData.nested.object); + assert.strictEqual(event.message.number, 789); + assert.strictEqual(event.message.bool, true); next(null, context); } @@ -1073,16 +1169,17 @@ describe("SplunkLogger send (integration tests)", function() { var _sendEvents = logger._sendEvents; logger._sendEvents = function(context, next) { - assert.ok(context.message.hasOwnProperty("event")); - assert.ok(context.message.hasOwnProperty("time")); - - var e = context.message.event; - assert.ok(e.hasOwnProperty("message")); - assert.ok(e.hasOwnProperty("severity")); - assert.strictEqual(e.message.property, "new"); - assert.strictEqual(e.message.nested.object, initialData.nested.object); - assert.strictEqual(e.message.number, 789); - assert.strictEqual(e.message.bool, true); + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.ok(event.hasOwnProperty("severity")); + + assert.strictEqual(event.message.property, "new"); + assert.strictEqual(event.message.nested.object, initialData.nested.object); + assert.strictEqual(event.message.number, 789); + assert.strictEqual(event.message.bool, true); _sendEvents(context, next); }; @@ -1096,7 +1193,6 @@ describe("SplunkLogger send (integration tests)", function() { bool: false }; var context = { - config: config, message: initialData }; @@ -1113,15 +1209,25 @@ describe("SplunkLogger send (integration tests)", function() { describe("error handlers", function() { it("should get error and context using default error handler, without passing context to next()", function(done) { var config = { - token: "token-goes-here" + token: "token-goes-here", + maxBatchCount: 1 }; var middlewareCount = 0; function middleware(context, next) { middlewareCount++; - assert.strictEqual(context.message, "something"); - context.message = "something else"; + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.ok(event.hasOwnProperty("severity")); + assert.strictEqual(event.message, "something"); + + + body.event.message = "something else"; + context.message = JSON.stringify(body); next(new Error("error!")); } @@ -1130,7 +1236,6 @@ describe("SplunkLogger send (integration tests)", function() { var initialData = "something"; var initialContext = { - config: config, message: initialData }; @@ -1143,8 +1248,15 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(err); assert.ok(context); assert.strictEqual(err.message, "error!"); - initialContext.message = "something else"; - assert.strictEqual(context, initialContext); + + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.ok(event.hasOwnProperty("severity")); + + assert.strictEqual(event.message, "something else"); mute(); errCallback(err, context); @@ -1161,15 +1273,26 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should get error and context using default error handler", function(done) { var config = { - token: "token-goes-here" + token: "token-goes-here", + maxBatchCount: 1 }; var middlewareCount = 0; function middleware(context, next) { middlewareCount++; - assert.strictEqual(context.message, "something"); - context.message = "something else"; + + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.ok(event.hasOwnProperty("severity")); + + assert.strictEqual(event.message, "something"); + + body.event.message = "something else"; + context.message = JSON.stringify(body); next(new Error("error!"), context); } @@ -1178,7 +1301,6 @@ describe("SplunkLogger send (integration tests)", function() { var initialData = "something"; var initialContext = { - config: config, message: initialData }; @@ -1191,8 +1313,16 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(err); assert.ok(context); assert.strictEqual(err.message, "error!"); - initialContext.message = "something else"; - assert.strictEqual(context, initialContext); + + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.ok(event.hasOwnProperty("severity")); + + body.event.message = "something else"; + context.message = JSON.stringify(body); mute(); errCallback(err, context); @@ -1209,15 +1339,26 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should get error and context sending twice using default error handler", function(done) { var config = { - token: "token-goes-here" + token: "token-goes-here", + maxBatchCount: 1 }; var middlewareCount = 0; function middleware(context, next) { middlewareCount++; - assert.strictEqual(context.message, "something"); - context.message = "something else"; + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.ok(event.hasOwnProperty("severity")); + + assert.strictEqual(event.message, "something"); + assert.strictEqual(event.severity, "info"); + + body.event.message = "something else"; + context.message = JSON.stringify(body); next(new Error("error!"), context); } @@ -1226,7 +1367,6 @@ describe("SplunkLogger send (integration tests)", function() { var initialData = "something"; var context1 = { - config: config, message: initialData }; @@ -1236,16 +1376,16 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(err); assert.strictEqual(err.message, "error!"); assert.ok(context); - assert.strictEqual(context.message, "something else"); - var comparing = context1; - if (middlewareCount === 2) { - comparing = context2; - } - assert.strictEqual(context.severity, comparing.severity); - assert.strictEqual(context.config, comparing.config); - assert.strictEqual(context.requestOptions, comparing.requestOptions); - + var body = JSON.parse(context.message); + assert.ok(body.hasOwnProperty("time")); + assert.ok(body.hasOwnProperty("event")); + var event = body.event; + assert.ok(event.hasOwnProperty("message")); + assert.ok(event.hasOwnProperty("severity")); + + assert.strictEqual(event.message, "something else"); + mute(); errCallback(err, context); unmute(); @@ -1269,7 +1409,8 @@ describe("SplunkLogger send (integration tests)", function() { it("should retry exactly 0 times (send once only)", function(done) { var config = { token: configurationFile.token, - maxRetries: 0 + maxRetries: 0, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -1303,7 +1444,8 @@ describe("SplunkLogger send (integration tests)", function() { it("should retry exactly once", function(done) { var config = { token: configurationFile.token, - maxRetries: 1 + maxRetries: 1, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -1337,7 +1479,8 @@ describe("SplunkLogger send (integration tests)", function() { it("should retry exactly twice", function(done) { var config = { token: configurationFile.token, - maxRetries: 2 + maxRetries: 2, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -1371,7 +1514,8 @@ describe("SplunkLogger send (integration tests)", function() { it("should retry exactly 5 times", function(done) { var config = { token: configurationFile.token, - maxRetries: 5 + maxRetries: 5, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -1405,7 +1549,8 @@ describe("SplunkLogger send (integration tests)", function() { it("should not retry on initial success when maxRetries=1", function(done) { var config = { token: configurationFile.token, - maxRetries: 1 + maxRetries: 1, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -1439,7 +1584,8 @@ describe("SplunkLogger send (integration tests)", function() { it("should retry once when maxRetries=10", function(done) { var config = { token: configurationFile.token, - maxRetries: 10 + maxRetries: 10, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -1474,7 +1620,8 @@ describe("SplunkLogger send (integration tests)", function() { var config = { token: configurationFile.token, maxRetries: 0, - host: "bad-hostname.invalid" + host: "bad-hostname.invalid", + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -1511,7 +1658,8 @@ describe("SplunkLogger send (integration tests)", function() { var config = { token: configurationFile.token, maxRetries: 1, - host: "bad-hostname.invalid" + host: "bad-hostname.invalid", + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -1548,7 +1696,8 @@ describe("SplunkLogger send (integration tests)", function() { var config = { token: configurationFile.token, maxRetries: 5, - host: "bad-hostname.invalid" + host: "bad-hostname.invalid", + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -1584,7 +1733,8 @@ describe("SplunkLogger send (integration tests)", function() { it("should not retry on Splunk error when maxRetries=0", function(done) { var config = { token: "invalid-token", - maxRetries: 0 + maxRetries: 0, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -1622,7 +1772,8 @@ describe("SplunkLogger send (integration tests)", function() { it("should not retry on Splunk error when maxRetries=1", function(done) { var config = { token: "invalid-token", - maxRetries: 1 + maxRetries: 1, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -1660,7 +1811,8 @@ describe("SplunkLogger send (integration tests)", function() { it("should not retry on Splunk error when maxRetries=5", function(done) { var config = { token: "invalid-token", - maxRetries: 5 + maxRetries: 5, + maxBatchCount: 1 }; var logger = new SplunkLogger(config); @@ -1723,7 +1875,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.strictEqual(logger._timerDuration, 100); assert.strictEqual(posts, 0); assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); // Clean up the timer logger._disableTimer(); @@ -1761,7 +1913,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.strictEqual(logger._timerDuration, 100); assert.strictEqual(posts, 1); assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); // Clean up the timer logger._disableTimer(); @@ -1813,7 +1965,9 @@ describe("SplunkLogger send (integration tests)", function() { var config = { token: configurationFile.token, autoFlush: true, - batchInterval: 100 + batchInterval: 200, + maxBatchSize: 5000, + maxBatchCount: 10 }; var logger = new SplunkLogger(config); @@ -1841,10 +1995,10 @@ describe("SplunkLogger send (integration tests)", function() { logger.send(payload); setTimeout(function() { - assert.strictEqual(logger._timerDuration, 100); + assert.strictEqual(logger._timerDuration, 200); assert.strictEqual(posts, 1); assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); // Clean up the timer logger._disableTimer(); @@ -1905,71 +2059,20 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.strictEqual(posts, 1); assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); // Clean up the timer logger._disableTimer(); done(); }, 500); }); - it("should flush an event with batchInterval, then disable autoFlush for manual batching", function(done) { + it("should flush an event with batchInterval, then set batchInterval=0 and maxBatchCount=3 for manual batching", function(done) { var config = { token: configurationFile.token, autoFlush: true, - batchInterval: 100 - }; - var logger = new SplunkLogger(config); - - var posts = 0; - - // Wrap _post so we can verify how many times we called it - var _post = logger._post; - logger._post = function(context, callback) { - _post(context, function(err, resp, body) { - posts++; - assert.ok(!err); - assert.strictEqual(body.code, successBody.code); - assert.strictEqual(body.text, successBody.text); - callback(err, resp, body); - }); - }; - - var payload = { - message: "something" - }; - logger.send(payload); - - var run = false; - setTimeout(function() { - logger.config.autoFlush = false; - - var payload2 = { - message: "something else" - }; - logger.send(payload2); - - assert.strictEqual(logger.contextQueue.length, 1); - assert.strictEqual(logger.eventSizes.length, 1); - logger.flush(); - run = true; - }, 150); - - setTimeout(function() { - assert.ok(run); - assert.strictEqual(2, posts); - assert.strictEqual(0, logger.contextQueue.length); - assert.strictEqual(0, logger.eventSizes.length); - - // Clean up the timer - logger._disableTimer(); - done(); - }, 500); - }); - it("should flush an event with batchInterval, then set batchInterval=0 for manual batching", function(done) { - var config = { - token: configurationFile.token, - autoFlush: true, - batchInterval: 100 + batchInterval: 100, + maxBatchSize: 100000, + maxBatchCount: 3 }; var logger = new SplunkLogger(config); @@ -2000,76 +2103,26 @@ describe("SplunkLogger send (integration tests)", function() { message: "something else" }; logger.send(payload2); + logger.send(payload2); - assert.strictEqual(logger.contextQueue.length, 1); - assert.strictEqual(logger.eventSizes.length, 1); - logger.flush(); + assert.strictEqual(logger.contextQueue.length, 2); + assert.ok(logger.eventsBatchSize > 150); + logger.send(payload2); // This should trigger a flush run = true; }, 150); setTimeout(function() { assert.ok(run); - assert.strictEqual(2, posts); - assert.strictEqual(0, logger.contextQueue.length); - assert.strictEqual(0, logger.eventSizes.length); - - // Clean up the timer - logger._disableTimer(); - done(); - }, 500); - }); - it("should autoFlush an event at batchInterval, then again when batchInterval has changed", function(done) { - var config = { - token: configurationFile.token, - autoFlush: true, - batchInterval: 500 - }; - var logger = new SplunkLogger(config); - - var posts = 0; - - // Wrap _post so we can verify how many times we called it - var _post = logger._post; - logger._post = function(context, callback) { - _post(context, function(err, resp, body) { - posts++; - assert.ok(!err); - assert.strictEqual(body.code, successBody.code); - assert.strictEqual(body.text, successBody.text); - callback(err, resp, body); - }); - }; - - var payload = { - message: "something" - }; - logger.send(payload); - - setTimeout(function() { - assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); - assert.strictEqual(posts, 1); - assert.strictEqual(logger._timerDuration, 500); - logger.config.autoFlush = true; - logger.config.batchInterval = 100; - var payload2 = { - message: "something else" - }; - logger.send(payload2); - assert.strictEqual(logger._timerDuration, 100); - }, 550); - - - setTimeout(function() { assert.strictEqual(posts, 2); assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); // Clean up the timer logger._disableTimer(); done(); - }, 700); + }, 500); }); - it("should flush an event with batchInterval, then set batchInterval=0 for manual batching", function(done) { + it("should flush an event with batchInterval=100", function(done) { var config = { token: configurationFile.token, autoFlush: true, @@ -2098,15 +2151,13 @@ describe("SplunkLogger send (integration tests)", function() { var run = false; setTimeout(function() { - logger.config.batchInterval = 100; - var payload2 = { message: "something else" }; logger.send(payload2); assert.strictEqual(logger.contextQueue.length, 1); - assert.strictEqual(logger.eventSizes.length, 1); + assert.ok(logger.eventsBatchSize > 50); run = true; }, 150); @@ -2115,7 +2166,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(run); assert.strictEqual(posts, 2); assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); // Clean up the timer logger._disableTimer(); @@ -2143,7 +2194,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 1); assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); assert.ok(!err); assert.strictEqual(body.code, successBody.code); @@ -2158,42 +2209,6 @@ describe("SplunkLogger send (integration tests)", function() { }; logger.send(payload); }); - it("should not flush first event with maxBatchSize=1 && autoFlush=false", function(done) { - var config = { - token: configurationFile.token, - autoFlush: false, - maxBatchSize: 1 - }; - var logger = new SplunkLogger(config); - - var posts = 0; - - // Wrap _post so we can verify how many times we called it - var _post = logger._post; - logger._post = function(context) { - _post(context, function(err, resp, body) { - posts++; - - assert.ok(!err); - assert.strictEqual(body.code, successBody.code); - assert.strictEqual(body.text, successBody.text); - }); - }; - - var payload = { - message: "more than 1 byte" - }; - logger.send(payload); - - setTimeout(function() { - assert.ok(!logger._timerID); - assert.strictEqual(posts, 0); - assert.strictEqual(logger.contextQueue.length, 1); - assert.strictEqual(logger.eventSizes.length, 1); - - done(); - }, 1000); - }); it("should flush first 2 events after maxBatchSize>100", function(done) { var config = { token: configurationFile.token, @@ -2213,7 +2228,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 1); assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); assert.ok(!err); assert.strictEqual(body.code, successBody.code); @@ -2232,7 +2247,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 0); assert.strictEqual(logger.contextQueue.length, 1); - assert.strictEqual(logger.eventSizes.length, 1); + assert.ok(logger.eventsBatchSize > 50); logger.send(payload); }, 300); @@ -2241,7 +2256,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 1); assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); }, 400); }); it("should flush first event after 200ms, with maxBatchSize=200", function(done) { @@ -2275,7 +2290,7 @@ describe("SplunkLogger send (integration tests)", function() { // Make sure the event wasn't flushed yet setTimeout(function() { assert.strictEqual(logger.contextQueue.length, 1); - assert.strictEqual(logger.eventSizes.length, 1); + assert.ok(logger.eventsBatchSize > 50); }, 150); setTimeout(function() { @@ -2285,17 +2300,16 @@ describe("SplunkLogger send (integration tests)", function() { assert.strictEqual(posts, 1); assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); done(); }, 250); }); - }); - describe("using max batch count", function() { - it("should flush first event immediately with maxBatchCount=1 with large maxBatchSize", function(done) { + it("should flush first event before 200ms, with maxBatchSize=1", function(done) { var config = { token: configurationFile.token, - maxBatchCount: 1, - maxBatchSize: 123456 + autoFlush: true, + maxBatchSize: 1, + batchInterval: 200 }; var logger = new SplunkLogger(config); @@ -2307,29 +2321,40 @@ describe("SplunkLogger send (integration tests)", function() { _post(context, function(err, resp, body) { posts++; - assert.ok(!logger._timerID); - assert.strictEqual(posts, 1); - assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); - assert.ok(!err); assert.strictEqual(body.code, successBody.code); assert.strictEqual(body.text, successBody.text); - - done(); }); }; var payload = { - message: "one event" + message: "more than 1 byte" }; logger.send(payload); + + // Event should be sent before the interval timer runs the first time + setTimeout(function() { + assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(posts, 1); + }, 150); + + setTimeout(function() { + assert.ok(logger._timerID); + assert.strictEqual(logger._timerDuration, 200); + logger._disableTimer(); + + assert.strictEqual(posts, 1); + assert.strictEqual(logger.contextQueue.length, 0); + done(); + }, 250); }); - it("should not flush first event with maxBatchCount=1 && autoFlush=false", function(done) { + }); + describe("using max batch count", function() { + it("should flush first event immediately with maxBatchCount=1 with large maxBatchSize", function(done) { var config = { token: configurationFile.token, - autoFlush: false, - maxBatchCount: 1 + maxBatchCount: 1, + maxBatchSize: 123456 }; var logger = new SplunkLogger(config); @@ -2341,9 +2366,16 @@ describe("SplunkLogger send (integration tests)", function() { _post(context, function(err, resp, body) { posts++; + assert.ok(!logger._timerID); + assert.strictEqual(posts, 1); + assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); + assert.ok(!err); assert.strictEqual(body.code, successBody.code); assert.strictEqual(body.text, successBody.text); + + done(); }); }; @@ -2351,15 +2383,6 @@ describe("SplunkLogger send (integration tests)", function() { message: "one event" }; logger.send(payload); - - setTimeout(function() { - assert.ok(!logger._timerID); - assert.strictEqual(posts, 0); - assert.strictEqual(logger.contextQueue.length, 1); - assert.strictEqual(logger.eventSizes.length, 1); - - done(); - }, 1000); }); it("should not flush events with maxBatchCount=0 (meaning ignore) and large maxBatchSize", function(done) { var config = { @@ -2392,7 +2415,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 0); assert.strictEqual(logger.contextQueue.length, 1); - assert.strictEqual(logger.eventSizes.length, 1); + assert.ok(logger.eventsBatchSize > 50); done(); }, 1000); @@ -2416,7 +2439,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 1); assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); assert.ok(!err); assert.strictEqual(body.code, successBody.code); @@ -2435,7 +2458,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 0); assert.strictEqual(logger.contextQueue.length, 1); - assert.strictEqual(logger.eventSizes.length, 1); + assert.ok(logger.eventsBatchSize > 50); logger.send(payload); }, 300); @@ -2444,7 +2467,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 1); assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); }, 400); }); it("should flush first event after 200ms, with maxBatchCount=10", function(done) { @@ -2478,7 +2501,7 @@ describe("SplunkLogger send (integration tests)", function() { // Make sure the event wasn't flushed yet setTimeout(function() { assert.strictEqual(logger.contextQueue.length, 1); - assert.strictEqual(logger.eventSizes.length, 1); + assert.ok(logger.eventsBatchSize > 50); }, 150); setTimeout(function() { @@ -2488,7 +2511,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.strictEqual(posts, 1); assert.strictEqual(logger.contextQueue.length, 0); - assert.strictEqual(logger.eventSizes.length, 0); + assert.strictEqual(logger.eventsBatchSize, 0); done(); }, 300); }); diff --git a/test/test_utils.js b/test/test_utils.js index 8a6f0ff..51a1d13 100644 --- a/test/test_utils.js +++ b/test/test_utils.js @@ -467,4 +467,75 @@ describe("Utils", function() { done(); }); }); + describe("copyObject", function() { + it("should copy a 5 property object", function() { + var o = { + a: 1, + b: 2, + c: 3, + d: 4, + e: 5, + }; + + var copy = o; + + // Pointing to the same memory block + assert.strictEqual(copy, o); + + // Verify it was a real copy + copy = utils.copyObject(o); + assert.notStrictEqual(copy, o); + assert.strictEqual(Object.keys(copy).length, 5); + assert.strictEqual(copy.a, o.a); + assert.strictEqual(copy.b, o.b); + assert.strictEqual(copy.c, o.c); + assert.strictEqual(copy.d, o.d); + assert.strictEqual(copy.e, o.e); + + // Verify changing original object vals doesn't change copy's vals + for (var k in o) { + if (o.hasOwnProperty(k)){ + o[k]++; + } + } + assert.notStrictEqual(copy.a, o.a); + assert.notStrictEqual(copy.b, o.b); + assert.notStrictEqual(copy.c, o.c); + assert.notStrictEqual(copy.d, o.d); + assert.notStrictEqual(copy.e, o.e); + + }); + }); + describe("copyArray", function() { + it("should copy a 5 element array", function() { + var a = [0, 1, 2, 3, 4]; + + var copy = a; + + // Pointing to the same memory block + assert.strictEqual(copy, a); + + // Verify it was a real copy + copy = utils.copyArray(a); + assert.notStrictEqual(copy, a); + assert.strictEqual(Object.keys(copy).length, 5); + assert.strictEqual(copy[0], a[0]); + assert.strictEqual(copy[1], a[1]); + assert.strictEqual(copy[2], a[2]); + assert.strictEqual(copy[3], a[3]); + assert.strictEqual(copy[4], a[4]); + + // Verify changing original array vals doesn't change copy's vals + for (var k in a) { + a[k]++; + } + assert.notStrictEqual(copy[0], a[0]); + assert.notStrictEqual(copy[1], a[1]); + assert.notStrictEqual(copy[2], a[2]); + assert.notStrictEqual(copy[3], a[3]); + assert.notStrictEqual(copy[4], a[4]); + + }); + }); + // TODO: test for orByProp, orByBooleanProp, validatePositiveInt }); \ No newline at end of file diff --git a/utils.js b/utils.js index e8502b7..3ec80ef 100644 --- a/utils.js +++ b/utils.js @@ -200,6 +200,8 @@ utils.expBackoff = function(opts, callback) { * * @param {object} [self] - An object to bind the fn function parameter to. * @param {object} [fn] - A function to bind to the self argument. + * @returns {function} + * @static */ utils.bind = function(self, fn) { return function () { @@ -207,4 +209,97 @@ utils.bind = function(self, fn) { }; }; +/** + * Copies all properties into a new object which is returned. + * + * @param {object} [obj] - Object to copy properties from. + */ +utils.copyObject = function(obj) { + var ret = {}; + for (var key in obj) { + if (obj.hasOwnProperty(key)) { + ret[key] = obj[key]; + } + } + return ret; +}; + +/** + * Copies all elements into a new array which is returned. + * + * @param {array} [arr] - Array to copy elements from. + * @returns {array} + * @static + */ +utils.copyArray = function(arr) { + var ret = []; + for (var i = 0; arr && i < arr.length; i++) { + ret[i] = arr[i]; + } + return ret; +}; + +/** + * Takes a property name, then any number of objects as arguments + * and performs logical OR operations on them one at a time + * Returns true as soon as a truthy + * value is found, else returning false. + * + * @param {string} [prop] - property name for other arguments. + * @returns {boolean} + * @static + */ +utils.orByProp = function(prop) { + var ret = false; + for (var i = 1; !ret && i < arguments.length; i++) { + if (arguments[i]) { + ret = ret || arguments[i][prop]; + } + } + return ret; +}; + +/** + * Like utisl.orByProp() but for a boolean property. + * The first argument after prop with that property + * defined will be returned. + * + * @param {string} [prop] - property name for other arguments. + * @returns {boolean} + * @static + */ +utils.orByBooleanProp = function(prop) { + var ret = null; + // Logic is reversed here, first value wins + for (var i = arguments.length - 1; i > 0; i--) { + if (arguments[i] && arguments[i].hasOwnProperty(prop)) { + ret = arguments[i][prop]; + } + } + // Convert whatever we have to a boolean + return !!ret; +}; + + /** + * Tries to validate the value parameter as a positive + * integer. + * + * @param {number} [value] - Some value, expected to be a positive integer. + * @param {number} [label] - Human readable name for value + * for error messages. + * @returns {number} + * @throws Will throw an error if the value parameter cannot by parsed as an integer. + * @static + */ +utils.validatePositiveInt = function(value, label) { + value = parseInt(value, 10); + if (isNaN(value)) { + throw new Error(label + " must be a number, found: " + value); + } + else if (value < 0) { + throw new Error(label + " must be a positive number, found: " + value); + } + return value; +}; + module.exports = utils; \ No newline at end of file From d50f1bd9f1f430a68ade368e2a082dda14fbc722 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Fri, 13 Nov 2015 14:24:47 -0800 Subject: [PATCH 17/46] Add units for _enableTimer() --- splunklogger.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/splunklogger.js b/splunklogger.js index 35e5bba..70470fd 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -174,7 +174,7 @@ SplunkLogger.prototype._disableTimer = function() { * Configures an interval timer to flush any events in * this.contextQueue at the specified interval. * - * param {Number} interval - The batch interval. + * param {Number} interval - The batch interval in milliseconds. * @private */ SplunkLogger.prototype._enableTimer = function(interval) { From ae5d79be5f60d03790730269861e66ed8fa9f386 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Fri, 13 Nov 2015 15:47:27 -0800 Subject: [PATCH 18/46] Disable maxBatchSize and maxBatchCount by default. All batch settings will be ignored when they are <= 0. --- splunklogger.js | 18 +++++++++--------- test/test_config.js | 2 +- test/test_send.js | 1 + 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/splunklogger.js b/splunklogger.js index 70470fd..ea4b9c1 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -89,11 +89,11 @@ function _defaultEventFormatter(message, severity) { * [SplunkLogger.levels]{@link SplunkLogger#levels} for common levels. * @param {bool} [config.autoFlush=true] - Send events immediately or not. * @param {number} [config.batchInterval=0] - If config.autoFlush === true, automatically flush events after this many milliseconds. - * When set to a non-positive value, events will be sent one by one. - * @param {number} [config.maxBatchSize=10240] - If config.autoFlush === true, automatically flush events after the size of queued - * events exceeds this many bytes. - * @param {number} [config.maxBatchCount=10] - If config.autoFlush === true, automatically flush events after this many - * events have been queued. + * When set to a non-positive value, events will be sent one by one. This setting is ignored when non-positive. + * @param {number} [config.maxBatchSize=0] - If config.autoFlush === true, automatically flush events after the size of queued + * events exceeds this many bytes. This setting is ignored when non-positive. + * @param {number} [config.maxBatchCount=0] - If config.autoFlush === true, automatically flush events after this many + * events have been queued. This setting is ignored when non-positive. * @constructor * @throws Will throw an error if the config parameter is malformed. */ @@ -147,8 +147,8 @@ var defaultConfig = { autoFlush: true, maxRetries: 0, batchInterval: 0, - maxBatchSize: 10 * 1024, - maxBatchCount: 10 + maxBatchSize: 0, + maxBatchCount: 0 }; var defaultRequestOptions = { @@ -595,8 +595,8 @@ SplunkLogger.prototype.send = function(context, callback) { this.contextQueue.push(context); this.eventsBatchSize += Buffer.byteLength(JSON.stringify(this._makeBody(context)), "utf8"); - var batchOverSize = this.eventsBatchSize > this.config.maxBatchSize; - var batchOverCount = this.contextQueue.length >= this.config.maxBatchCount; + var batchOverSize = this.eventsBatchSize > this.config.maxBatchSize && this.config.maxBatchSize > 0; + var batchOverCount = this.contextQueue.length >= this.config.maxBatchCount && this.config.maxBatchCount > 0; // Only flush if not using manual batching, and if the contextQueue is too large or has many events if (this.config.autoFlush && (batchOverSize || batchOverCount)) { diff --git a/test/test_config.js b/test/test_config.js index 686a78d..2ed342c 100644 --- a/test/test_config.js +++ b/test/test_config.js @@ -144,7 +144,7 @@ describe("SplunkLogger", function() { assert.strictEqual(true, logger.config.autoFlush); assert.strictEqual(0, logger.config.maxRetries); assert.strictEqual(0, logger.config.batchInterval); - assert.strictEqual(10240, logger.config.maxBatchSize); + assert.strictEqual(0, logger.config.maxBatchSize); var expectedRO = { json: true, diff --git a/test/test_send.js b/test/test_send.js index 9ae5513..8fa5b14 100644 --- a/test/test_send.js +++ b/test/test_send.js @@ -2050,6 +2050,7 @@ describe("SplunkLogger send (integration tests)", function() { logger.config.autoFlush = true; logger.config.batchInterval = 100; + logger._initializeConfig(logger.config); var payload2 = { message: "something else" From 331143c8ffe5fb901fc890c9fcff22c5625a7ab3 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Fri, 13 Nov 2015 16:25:20 -0800 Subject: [PATCH 19/46] Rename utils.validatePositiveInt() to validateNonNegativeInt() Added remaining tests for utils --- splunklogger.js | 12 +++---- test/test_utils.js | 79 +++++++++++++++++++++++++++++++++++++++++++++- utils.js | 4 +-- 3 files changed, 86 insertions(+), 9 deletions(-) diff --git a/splunklogger.js b/splunklogger.js index ea4b9c1..412a6b1 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -179,7 +179,7 @@ SplunkLogger.prototype._disableTimer = function() { */ SplunkLogger.prototype._enableTimer = function(interval) { // Only enable the timer if possible - interval = utils.validatePositiveInt(interval, "Batch interval"); + interval = utils.validateNonNegativeInt(interval, "Batch interval"); if (this._timerID) { this._disableTimer(); @@ -259,13 +259,13 @@ SplunkLogger.prototype._initializeConfig = function(config) { ret.path = utils.orByProp("path", config, ret, defaultConfig); ret.protocol = utils.orByProp("protocol", config, ret, defaultConfig); ret.port = utils.orByProp("port", config, ret, defaultConfig); - ret.port = utils.validatePositiveInt(ret.port, "Port"); + ret.port = utils.validateNonNegativeInt(ret.port, "Port"); if (ret.port < 1000 || ret.port > 65535) { throw new Error("Port must be an integer between 1000 and 65535, found: " + ret.port); } ret.maxRetries = utils.orByProp("maxRetries", config, ret, defaultConfig); - ret.maxRetries = utils.validatePositiveInt(ret.maxRetries, "Max retries"); + ret.maxRetries = utils.validateNonNegativeInt(ret.maxRetries, "Max retries"); ret.autoFlush = utils.orByBooleanProp("autoFlush", config, this.config, defaultConfig); @@ -281,9 +281,9 @@ SplunkLogger.prototype._initializeConfig = function(config) { ret.batchInterval = utils.orByProp("batchInterval", config, ret, defaultConfig); } - ret.maxBatchCount = utils.validatePositiveInt(ret.maxBatchCount, "Max batch count"); - ret.maxBatchSize = utils.validatePositiveInt(ret.maxBatchSize, "Max batch size"); - ret.batchInterval = utils.validatePositiveInt(ret.batchInterval, "Batch interval"); + ret.maxBatchCount = utils.validateNonNegativeInt(ret.maxBatchCount, "Max batch count"); + ret.maxBatchSize = utils.validateNonNegativeInt(ret.maxBatchSize, "Max batch size"); + ret.batchInterval = utils.validateNonNegativeInt(ret.batchInterval, "Batch interval"); // Error if autoFlush is off and any batching settings are set if (!ret.autoFlush && (ret.batchInterval || ret.maxBatchSize || ret.maxBatchCount)) { diff --git a/test/test_utils.js b/test/test_utils.js index 51a1d13..9f09eb5 100644 --- a/test/test_utils.js +++ b/test/test_utils.js @@ -537,5 +537,82 @@ describe("Utils", function() { }); }); - // TODO: test for orByProp, orByBooleanProp, validatePositiveInt + describe("orByProp", function() { + it("should pick first value of 2", function() { + var a = { + x: "x value" + }; + var b = { + y: "y value" + }; + + assert.strictEqual(utils.orByProp("x", a, b), "x value"); + assert.strictEqual(utils.orByProp("y", b, a), "y value"); + }); + it("should pick second value of 2, when first is undefined", function() { + var a = { + x: "x value" + }; + var b = { + y: "y value" + }; + + assert.strictEqual(utils.orByProp("x", b, a), "x value"); + assert.strictEqual(utils.orByProp("y", a, b), "y value"); + }); + }); + describe("orByBooleanProp", function() { + it("should pick first value of 2", function() { + var a = { + x: false + }; + var b = { + y: true + }; + + assert.strictEqual(utils.orByBooleanProp("x", a, b), false); + assert.strictEqual(utils.orByBooleanProp("y", b, a), true); + }); + it("should pick second value of 2", function() { + var a = { + x: false + }; + var b = { + y: true + }; + + assert.strictEqual(utils.orByBooleanProp("x", b, a), false); + assert.strictEqual(utils.orByBooleanProp("y", a, b), true); + }); + }); + describe("validateNonNegativeInt", function() { + it("should error when value is NaN", function() { + try { + utils.validateNonNegativeInt(null, "test"); + assert.ok(false, "Expected an error."); + } + catch (err) { + assert.ok(err); + assert.strictEqual(err.message, "test must be a number, found: NaN"); + } + }); + it("should error when value is negative", function() { + try { + utils.validateNonNegativeInt(-1, "test"); + assert.ok(false, "Expected an error."); + } + catch (err) { + assert.ok(err); + assert.strictEqual(err.message, "test must be a positive number, found: -1"); + } + }); + it("should return the value when it's 0", function() { + var valid = utils.validateNonNegativeInt(0, "test"); + assert.strictEqual(valid, 0); + }); + it("should return the value when it's positive", function() { + var valid = utils.validateNonNegativeInt(5, "test"); + assert.strictEqual(valid, 5); + }); + }); }); \ No newline at end of file diff --git a/utils.js b/utils.js index 3ec80ef..d158fe9 100644 --- a/utils.js +++ b/utils.js @@ -281,7 +281,7 @@ utils.orByBooleanProp = function(prop) { }; /** - * Tries to validate the value parameter as a positive + * Tries to validate the value parameter as a non-negative * integer. * * @param {number} [value] - Some value, expected to be a positive integer. @@ -291,7 +291,7 @@ utils.orByBooleanProp = function(prop) { * @throws Will throw an error if the value parameter cannot by parsed as an integer. * @static */ -utils.validatePositiveInt = function(value, label) { +utils.validateNonNegativeInt = function(value, label) { value = parseInt(value, 10); if (isNaN(value)) { throw new Error(label + " must be a number, found: " + value); From 974187f3eb8643c93f82211daf326d3ba3e7d04c Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Fri, 13 Nov 2015 16:41:42 -0800 Subject: [PATCH 20/46] Renamed batching example to manual_batching --- examples/{batching.js => manual_batching.js} | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) rename examples/{batching.js => manual_batching.js} (92%) diff --git a/examples/batching.js b/examples/manual_batching.js similarity index 92% rename from examples/batching.js rename to examples/manual_batching.js index 2aff407..d1ab3e8 100644 --- a/examples/batching.js +++ b/examples/manual_batching.js @@ -32,17 +32,12 @@ var SplunkLogger = require("../index").Logger; /** * Only the token property is required. * - * Here, autoFlush is set to false + * Here, autoFlush is set to false. */ var config = { token: "your-token-here", - host: "localhost", - path: "/services/collector/event/1.0", - protocol: "https", - port: 8088, - level: "info", - autoFlush: false, - maxRetries: 0 + url: "https://localhost:8088", + autoFlush: false // Manually flush events }; // Create a new logger From d50f926d92358efdb57e21aa3b7def4d29cfafda Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Fri, 13 Nov 2015 17:18:44 -0800 Subject: [PATCH 21/46] Add all batching settings example --- examples/all_batching.js | 102 ++++++++++++++++++++++++++++++++++++ examples/manual_batching.js | 5 +- 2 files changed, 103 insertions(+), 4 deletions(-) create mode 100644 examples/all_batching.js diff --git a/examples/all_batching.js b/examples/all_batching.js new file mode 100644 index 0000000..d0c0a06 --- /dev/null +++ b/examples/all_batching.js @@ -0,0 +1,102 @@ +/* + * Copyright 2015 Splunk, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"): you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +/** + * This example shows how to batch events with the + * SplunkLogger with all available settings. + * + * By default autoFlush is enabled. + * + * By disabling autoFlush, events will be queued + * until flush() is called. + */ + +// Change to require("splunk-logging").Logger; +var SplunkLogger = require("../index").Logger; + +/** + * Only the token property is required. + * + * Here, batchInterval is set to flush every 1 + * second, when 10 events are queued, or + * when the size of queued events totals + * more than 1kb. + */ +var config = { + token: "your-token", + url: "https://localhost:8088", + batchInterval: 1000, + maxBatchCount: 10, + maxBatchSize: 1024 // 1kb +}; + +// Create a new logger +var Logger = new SplunkLogger(config); + +Logger.error = function(err, context) { + // Handle errors here + console.log("error", err, "context", context); +}; + +// Define the payload to send to Splunk's Event Collector +var payload = { + // Message can be anything, doesn't have to be an object + message: { + temperature: "70F", + chickenCount: 500 + }, + // Metadata is optional + metadata: { + source: "chicken coop", + sourcetype: "httpevent", + index: "main", + host: "farm.local", + }, + // Severity is also optional + severity: "info" +}; + +console.log("Queuing payload", payload); +// Don't need a callback here +Logger.send(payload); + +var payload2 = { + message: { + temperature: "75F", + chickenCount: 600, + note: "New chickens have arrived" + }, + metadata: payload.metadata +}; + +console.log("Queuing second payload", payload2); +// Don't need a callback here +Logger.send(payload2); + +/** + * Since we've configured autoFlush, we don't need + * to do anything at this point. Events will + * will be sent to Splunk automatically based + * on the batching settings above. + */ + +// Kill the process +var p = process; +setTimeout(function() { + console.log("Events should be in Splunk! Exiting..."); + // p.exit(); + process.exit(); +}, 2000); \ No newline at end of file diff --git a/examples/manual_batching.js b/examples/manual_batching.js index d1ab3e8..3e2e991 100644 --- a/examples/manual_batching.js +++ b/examples/manual_batching.js @@ -17,10 +17,7 @@ /** * This example shows how to batch events with the * SplunkLogger by manually calling flush. - * - * By default autoFlush is enabled, this means - * an HTTP request is made each time send() - * is called. + * By default autoFlush is enabled. * * By disabling autoFlush, events will be queued * until flush() is called. From 064449752a0755d8b512c4986a66724149fe418c Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Fri, 13 Nov 2015 17:35:49 -0800 Subject: [PATCH 22/46] remove middleware and all references to it We have collectively decided that this feature does not provide enough value at this time. --- examples/all_batching.js | 2 - examples/middleware.js | 89 ----- splunklogger.js | 73 +---- test/test_config.js | 4 - test/test_send.js | 678 --------------------------------------- 5 files changed, 5 insertions(+), 841 deletions(-) delete mode 100644 examples/middleware.js diff --git a/examples/all_batching.js b/examples/all_batching.js index d0c0a06..fe7e278 100644 --- a/examples/all_batching.js +++ b/examples/all_batching.js @@ -94,9 +94,7 @@ Logger.send(payload2); */ // Kill the process -var p = process; setTimeout(function() { console.log("Events should be in Splunk! Exiting..."); - // p.exit(); process.exit(); }, 2000); \ No newline at end of file diff --git a/examples/middleware.js b/examples/middleware.js deleted file mode 100644 index 3745493..0000000 --- a/examples/middleware.js +++ /dev/null @@ -1,89 +0,0 @@ -/* - * Copyright 2015 Splunk, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"): you may - * not use this file except in compliance with the License. You may obtain - * a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the - * License for the specific language governing permissions and limitations - * under the License. - */ - -/** - * This example shows how to use middleware with the SplunkLogger. - */ - -// Change to require("splunk-logging").Logger; -var SplunkLogger = require("../index").Logger; - -/** - * Only the token property is required. - * Defaults are listed explicitly. - * - * Alternatively, specify config.url like so: - * - * "https://localhost:8088/services/collector/event/1.0" - */ -var config = { - token: "your-token-here", - host: "localhost", - path: "/services/collector/event/1.0", - protocol: "https", - port: 8088, - level: "info", - autoFlush: true, - maxRetries: 0 -}; - -// Create a new logger -var Logger = new SplunkLogger(config); - -Logger.error = function(err, context) { - // Handle errors here - console.log("error", err, "context", context); -}; - -// Add a middleware function -Logger.use(function(context, next) { - console.log("Message before middleware", context.message); - - // Add a property to the message if it's an object - if (typeof context.message === "object") { - context.message.nestedValue = { - b00l: true, - another: "string" - }; - } - - console.log("Message after middleware", context.message); - next(null, context); -}); - -// Define the payload to send to Splunk's Event Collector -var payload = { - // Message can be anything, doesn't have to be an object - message: { - temperature: "70F", - chickenCount: 500 - }, - // Metadata is optional - metadata: { - source: "chicken coop", - sourcetype: "httpevent", - index: "main", - host: "farm.local" - }, - // Severity is also optional - severity: "info" -}; - -console.log("Sending payload", payload); -Logger.send(payload, function(err, resp, body) { - // If successful, body will be { text: 'Success', code: 0 } - console.log("Response from Splunk", body); -}); \ No newline at end of file diff --git a/splunklogger.js b/splunklogger.js index 412a6b1..2e3edc5 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -27,6 +27,7 @@ var utils = require("./utils"); * @param {object} [context] - The context of an event. * @private */ +/* istanbul ignore next*/ function _err(err, context) { console.log("ERROR:", err, " CONTEXT", context); } @@ -67,7 +68,6 @@ function _defaultEventFormatter(message, severity) { * @property {object} config - Configuration settings for this SplunkLogger instance. * @param {object} requestOptions - Options to pass to {@link https://github.com/request/request#requestpost|request.post()}. * See the {@link http://github.com/request/request|request documentation} for all available options. - * @property {function[]} middlewares - Middleware functions to run before sending data to Splunk. * @property {object[]} contextQueue - Queue of context objects to be sent to Splunk. * @property {function} eventFormatter - Formats events, returning an event as a string, function(message, severity). * Can be overwritten, the default event formatter will display event and severity as properties in a JSON object. @@ -102,7 +102,6 @@ var SplunkLogger = function(config) { this._timerDuration = 0; this.config = this._initializeConfig(config); this.requestOptions = this._initializeRequestOptions(); - this.middlewares = []; this.contextQueue = []; this.eventsBatchSize = 0; this.eventFormatter = _defaultEventFormatter; @@ -116,7 +115,6 @@ var SplunkLogger = function(config) { this._initializeMetadata = utils.bind(this, this._initializeMetadata); this._initializeContext = utils.bind(this, this._initializeContext); this._makeBody = utils.bind(this, this._makeBody); - this.use = utils.bind(this, this.use); this._post = utils.bind(this, this._post); this._sendEvents = utils.bind(this, this._sendEvents); this.send = utils.bind(this, this.send); @@ -421,37 +419,6 @@ SplunkLogger.prototype._makeBody = function(context) { return body; }; -/** - * Adds an express-like middleware function to run before sending the - * data to Splunk. - * Multiple middleware functions can be used, they will be executed - * in the order they are added. - * - * This function is a wrapper around this.middlewares.push(). - * - * @example - * var SplunkLogger = require("splunk-logging").Logger; - * - * var Logger = new SplunkLogger({token: "your-token-here"}); - * Logger.use(function(context, next) { - * context.message.additionalProperty = "Add this before sending the data"; - * next(null, context); - * }); - * - * @param {function} middleware - A middleware function: function(context, next). - * It must call next(error, context) to continue. - * @public - * @throws Will throw an error if middleware is not a function. - */ -SplunkLogger.prototype.use = function(middleware) { - if (!middleware || typeof middleware !== "function") { - throw new Error("Middleware must be a function."); - } - else { - this.middlewares.push(middleware); - } -}; - /** * Makes an HTTP POST to the configured server. * @@ -539,7 +506,7 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { } ); }; - + /** * Sends or queues data to be sent based on this.config.autoFlush. * Default behavior is to send immediately. @@ -567,6 +534,7 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { * } * }; * + * // The callback is only used if autoFlush is set to false. * logger.send(payload, function(err, resp, body) { * if (err) { * console.log("error:", err); @@ -605,8 +573,7 @@ SplunkLogger.prototype.send = function(context, callback) { }; /** - * Manually send all events in this.contextQueue to Splunk, after - * chaining any functions in this.middlewares. + * Manually send all events in this.contextQueue to Splunk. * * @param {function} [callback] - A callback function: function(err, response, body). * @public @@ -625,37 +592,7 @@ SplunkLogger.prototype.flush = function(callback) { data += JSON.stringify(this._makeBody(queue[i])); } context.message = data; - - // Initialize the context before invoking middleware functions - context = this._initializeContext(context); - - // Copy over the middlewares - var callbacks = utils.copyArray(this.middlewares); - - // Send the data to the first middleware - callbacks.unshift(function(cb) { - cb(null, context); - }); - - // After running all, if any, middlewares send the events - var that = this; - utils.chain(callbacks, function(err, passedContext) { - // The passedContext parameter could be named context to - // do this automatically, but the || notation adds a bit of clarity. - context = passedContext || context; - - // Errors from any of the middleware callbacks will fall through to here. - // If the context is modified at any point the error callback will get it also - // event if next("error"); is called w/o the context parameter! - // This works because context inside & outside the scope of this function - // point to the same memory block. - if (err) { - that.error(err, context); - } - else { - that._sendEvents(context, callback); - } - }); + this._sendEvents(context, callback); }; module.exports = SplunkLogger; \ No newline at end of file diff --git a/test/test_config.js b/test/test_config.js index 2ed342c..acb1804 100644 --- a/test/test_config.js +++ b/test/test_config.js @@ -903,7 +903,6 @@ describe("SplunkLogger", function() { assert.strictEqual(Logger.config.protocol, expected.protocol); assert.strictEqual(Logger.config.level, expected.level); assert.strictEqual(Logger.config.port, expected.port); - assert.strictEqual(Logger.middlewares.length, 0); Logger._initializeConfig({}); assert.strictEqual(Logger.config.token, expected.token); @@ -913,7 +912,6 @@ describe("SplunkLogger", function() { assert.strictEqual(Logger.config.protocol, expected.protocol); assert.strictEqual(Logger.config.level, expected.level); assert.strictEqual(Logger.config.port, expected.port); - assert.strictEqual(Logger.middlewares.length, 0); }); it("token in constructor, then init with full config", function() { var config = { @@ -939,7 +937,6 @@ describe("SplunkLogger", function() { assert.strictEqual(Logger.config.protocol, expected.protocol); assert.strictEqual(Logger.config.level, expected.level); assert.strictEqual(Logger.config.port, expected.port); - assert.strictEqual(Logger.middlewares.length, 0); expected.token = "a-different-token"; Logger.config = Logger._initializeConfig(expected); @@ -950,7 +947,6 @@ describe("SplunkLogger", function() { assert.strictEqual(Logger.config.protocol, expected.protocol); assert.strictEqual(Logger.config.level, expected.level); assert.strictEqual(Logger.config.port, expected.port); - assert.strictEqual(Logger.middlewares.length, 0); }); it("should set autoFlush property to true after initially false", function() { Object.prototype.something = "ignore"; diff --git a/test/test_send.js b/test/test_send.js index 8fa5b14..61333fd 100644 --- a/test/test_send.js +++ b/test/test_send.js @@ -41,30 +41,6 @@ var noDataBody = { code: 5 }; -// TODO: add a test that gets this response -// var incorrectIndexBody = { -// text: "Incorrect index", -// code: 7, -// "invalid-event-number": 1 -// }; - -// Backup console.log so we can restore it later -var ___log = console.log; -/** - * Silences console.log - * Undo this effect by calling unmute(). - */ -function mute() { - console.log = function(){}; -} - -/** - * Un-silences console.log - */ -function unmute() { - console.log = ___log; -} - describe("SplunkLogger _makeBody", function() { it("should error with no args", function() { try { @@ -751,660 +727,6 @@ describe("SplunkLogger send (integration tests)", function() { }); }); }); - describe("using custom middleware", function() { - it("should error with non-function middleware", function() { - var config = { - token: "a-token-goes-here-usually" - }; - try { - var logger = new SplunkLogger(config); - logger.use("not a function"); - assert.fail(false, "Expected an error."); - } - catch (err) { - assert.ok(err); - assert.strictEqual(err.message, "Middleware must be a function."); - } - }); - it("should succeed using non-default middleware", function(done) { - var config = { - token: "token-goes-here", - maxBatchCount: 1 - }; - - var middlewareCount = 0; - - function middleware(context, next) { - middlewareCount++; - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - assert.strictEqual(Object.keys(body).length, 2); - var event = body.event; - assert.strictEqual(event.message, "something"); - next(null, context); - } - - var logger = new SplunkLogger(config); - logger.use(middleware); - - logger._sendEvents = function(context, next) { - var response = { - headers: { - "content-type": "application/json; charset=UTF-8", - isCustom: true - }, - body: successBody - }; - next(null, response, successBody); - }; - - var initialData = "something"; - var context = { - message: initialData - }; - - logger.send(context, function(err, resp, body) { - assert.ok(!err); - assert.strictEqual(resp.body, body); - assert.strictEqual(body.code, successBody.code); - assert.strictEqual(body.text, successBody.text); - assert.strictEqual(middlewareCount, 1); - done(); - }); - }); - it("should succeed using non-default middleware with manual batching", function(done) { - var config = { - token: "token-goes-here", - autoFlush: false - }; - - var initialData = "something"; - var sentContext = { - message: initialData - }; - - var middlewareCount = 0; - - function middleware(context, next) { - middlewareCount++; - var expectedString = JSON.stringify(logger._makeBody(sentContext)); - - assert.strictEqual(expectedString.length * 3, context.message.length); - - // Search the payload, we should have 3 identical events - for (var i = 1; i <= 3; i++) { - var start = (i - 1) * expectedString.length; - var end = start + expectedString.length; - - // Don't compare timestamps - var expected = JSON.parse(expectedString); - var actual = JSON.parse(context.message.substring(start, end)); - assert.ok(expected.hasOwnProperty("event")); - assert.ok(expected.hasOwnProperty("time")); - assert.ok(actual.hasOwnProperty("event")); - assert.ok(actual.hasOwnProperty("time")); - assert.strictEqual(JSON.stringify(expected.event), JSON.stringify(actual.event)); - } - - next(null, context); - } - - var logger = new SplunkLogger(config); - logger.use(middleware); - - logger._sendEvents = function(context, next) { - var response = { - headers: { - "content-type": "application/json; charset=UTF-8", - isCustom: true - }, - body: successBody - }; - next(null, response, successBody); - }; - - logger.send(sentContext); - logger.send(sentContext); - logger.send(sentContext); - - logger.flush(function(err, resp, body) { - assert.ok(!err); - assert.strictEqual(resp.body, body); - assert.strictEqual(body.code, successBody.code); - assert.strictEqual(body.text, successBody.text); - assert.strictEqual(middlewareCount, 1); - done(); - }); - }); - it("should succeed using non-default middleware, without passing the context through", function(done) { - var config = { - token: "token-goes-here", - maxBatchCount: 1 - }; - - var middlewareCount = 0; - - function middleware(context, next) { - middlewareCount++; - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - assert.strictEqual(Object.keys(body).length, 2); - var event = body.event; - assert.strictEqual(event.message, "something"); - next(null); - } - - var logger = new SplunkLogger(config); - logger.use(middleware); - - logger._sendEvents = function(context, next) { - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - assert.strictEqual(Object.keys(body).length, 2); - var event = body.event; - assert.strictEqual(event.message, initialContext.message); - assert.strictEqual(event.severity, "info"); - - var response = { - headers: { - "content-type": "application/json; charset=UTF-8", - isCustom: true - }, - body: successBody - }; - next(null, response, successBody); - }; - - var initialData = "something"; - var initialContext = { - message: initialData - }; - - logger.send(initialContext, function(err, resp, body) { - assert.ok(!err); - assert.strictEqual(resp.body, body); - assert.strictEqual(body.code, successBody.code); - assert.strictEqual(body.text, successBody.text); - assert.strictEqual(middlewareCount, 1); - done(); - }); - }); - it("should succeed using 2 middlewares", function(done) { - var config = { - token: "token-goes-here", - maxBatchCount: 1 - }; - - var middlewareCount = 0; - - function middleware(context, callback) { - middlewareCount++; - - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - assert.strictEqual(Object.keys(body).length, 2); - var event = body.event; - assert.strictEqual(event.message, "some??thing"); - - body.event.message = encodeURIComponent(event.message); - context.message = JSON.stringify(body); - callback(null, context); - } - - function middleware2(context, callback) { - middlewareCount++; - - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - assert.strictEqual(Object.keys(body).length, 2); - var event = body.event; - assert.strictEqual(event.message, "some%3F%3Fthing"); - assert.strictEqual(event.severity, "info"); - - callback(null, context); - } - - var logger = new SplunkLogger(config); - logger.use(middleware); - logger.use(middleware2); - - logger._sendEvents = function(context, next) { - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - assert.strictEqual(Object.keys(body).length, 2); - var event = body.event; - assert.ok(event.hasOwnProperty("message")); - assert.ok(event.hasOwnProperty("severity")); - assert.strictEqual(event.message, "some%3F%3Fthing"); - - var response = { - headers: { - "content-type": "application/json; charset=UTF-8", - isCustom: true - }, - body: successBody - }; - next(null, response, successBody); - }; - - var initialData = "some??thing"; - var context = { - message: initialData - }; - - logger.send(context, function(err, resp, body) { - assert.ok(!err); - assert.strictEqual(resp.body, body); - assert.strictEqual(body.code, successBody.code); - assert.strictEqual(body.text, successBody.text); - assert.strictEqual(middlewareCount, 2); - done(); - }); - }); - it("should succeed using 3 middlewares", function(done) { - var config = { - token: configurationFile.token, - maxBatchCount: 1 - }; - - var middlewareCount = 0; - - function middleware(context, next) { - middlewareCount++; - - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - var event = body.event; - assert.ok(event.hasOwnProperty("message")); - assert.ok(event.hasOwnProperty("severity")); - assert.strictEqual(event.message, "some??thing"); - assert.strictEqual(event.severity, "info"); - - body.event.message = encodeURIComponent(event.message); - context.message = JSON.stringify(body); - next(null, context); - } - - function middleware2(context, next) { - middlewareCount++; - - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - var event = body.event; - assert.ok(event.hasOwnProperty("message")); - assert.ok(event.hasOwnProperty("severity")); - assert.strictEqual(event.message, "some%3F%3Fthing"); - assert.strictEqual(event.severity, "info"); - - body.event.message = decodeURIComponent(event.message) + " changed"; - context.message = JSON.stringify(body); - next(null, context); - } - - function middleware3(context, next) { - middlewareCount++; - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - var event = body.event; - assert.ok(event.hasOwnProperty("message")); - assert.ok(event.hasOwnProperty("severity")); - assert.strictEqual(event.message, "some??thing changed"); - next(null, context); - } - - var logger = new SplunkLogger(config); - logger.use(middleware); - logger.use(middleware2); - logger.use(middleware3); - - var _sendEvents = logger._sendEvents; - logger._sendEvents = function(context, next) { - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - var event = body.event; - assert.ok(event.hasOwnProperty("message")); - assert.ok(event.hasOwnProperty("severity")); - assert.strictEqual(event.message, "some??thing changed"); - - _sendEvents(context, next); - }; - - var initialData = "some??thing"; - var context = { - message: initialData - }; - - logger.send(context, function(err, resp, body) { - assert.ok(!err); - assert.strictEqual(resp.body, body); - assert.strictEqual(body.code, successBody.code); - assert.strictEqual(body.text, successBody.text); - assert.strictEqual(middlewareCount, 3); - done(); - }); - }); - it("should succeed using 3 middlewares with data object", function(done) { - var config = { - token: configurationFile.token, - maxBatchCount: 1 - }; - - var middlewareCount = 0; - - function middleware(context, next) { - middlewareCount++; - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - var event = body.event; - assert.ok(event.hasOwnProperty("message")); - assert.ok(event.hasOwnProperty("severity")); - - assert.strictEqual(Object.keys(event.message).length, Object.keys(initialData).length); - assert.strictEqual(event.message.property, initialData.property); - assert.strictEqual(event.message.nested.object, initialData.nested.object); - assert.strictEqual(event.message.number, initialData.number); - assert.strictEqual(event.message.bool, initialData.bool); - - body.event.message.property = "new"; - body.event.message.bool = true; - context.message = JSON.stringify(body); - next(null, context); - } - - function middleware2(context, next) { - middlewareCount++; - - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - var event = body.event; - assert.ok(event.hasOwnProperty("message")); - assert.ok(event.hasOwnProperty("severity")); - - assert.strictEqual(Object.keys(event.message).length, Object.keys(initialData).length); - assert.strictEqual(event.message.property, "new"); - assert.strictEqual(event.message.nested.object, initialData.nested.object); - assert.strictEqual(event.message.number, initialData.number); - assert.strictEqual(event.message.bool, true); - - body.event.message.number = 789; - context.message = JSON.stringify(body); - next(null, context); - } - - function middleware3(context, next) { - middlewareCount++; - - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - var event = body.event; - assert.ok(event.hasOwnProperty("message")); - assert.ok(event.hasOwnProperty("severity")); - - assert.strictEqual(Object.keys(event.message).length, Object.keys(initialData).length); - assert.strictEqual(event.message.property, "new"); - assert.strictEqual(event.message.nested.object, initialData.nested.object); - assert.strictEqual(event.message.number, 789); - assert.strictEqual(event.message.bool, true); - - next(null, context); - } - - var logger = new SplunkLogger(config); - logger.use(middleware); - logger.use(middleware2); - logger.use(middleware3); - - var _sendEvents = logger._sendEvents; - logger._sendEvents = function(context, next) { - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - var event = body.event; - assert.ok(event.hasOwnProperty("message")); - assert.ok(event.hasOwnProperty("severity")); - - assert.strictEqual(event.message.property, "new"); - assert.strictEqual(event.message.nested.object, initialData.nested.object); - assert.strictEqual(event.message.number, 789); - assert.strictEqual(event.message.bool, true); - - _sendEvents(context, next); - }; - - var initialData = { - property: "one", - nested: { - object: "value" - }, - number: 1234, - bool: false - }; - var context = { - message: initialData - }; - - logger.send(context, function(err, resp, body) { - assert.ok(!err); - assert.strictEqual(resp.body, body); - assert.strictEqual(body.code, successBody.code); - assert.strictEqual(body.text, successBody.text); - assert.strictEqual(middlewareCount, 3); - done(); - }); - }); - }); - describe("error handlers", function() { - it("should get error and context using default error handler, without passing context to next()", function(done) { - var config = { - token: "token-goes-here", - maxBatchCount: 1 - }; - - var middlewareCount = 0; - - function middleware(context, next) { - middlewareCount++; - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - var event = body.event; - assert.ok(event.hasOwnProperty("message")); - assert.ok(event.hasOwnProperty("severity")); - assert.strictEqual(event.message, "something"); - - - body.event.message = "something else"; - context.message = JSON.stringify(body); - next(new Error("error!")); - } - - var logger = new SplunkLogger(config); - logger.use(middleware); - - var initialData = "something"; - var initialContext = { - message: initialData - }; - - var run = false; - - // Wrap the default error callback for code coverage - var errCallback = logger.error; - logger.error = function(err, context) { - run = true; - assert.ok(err); - assert.ok(context); - assert.strictEqual(err.message, "error!"); - - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - var event = body.event; - assert.ok(event.hasOwnProperty("message")); - assert.ok(event.hasOwnProperty("severity")); - - assert.strictEqual(event.message, "something else"); - - mute(); - errCallback(err, context); - unmute(); - - done(); - }; - - // Fire & forget, the callback won't be called anyways due to the error - logger.send(initialContext); - - assert.ok(run); - assert.strictEqual(middlewareCount, 1); - }); - it("should get error and context using default error handler", function(done) { - var config = { - token: "token-goes-here", - maxBatchCount: 1 - }; - - var middlewareCount = 0; - - function middleware(context, next) { - middlewareCount++; - - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - var event = body.event; - assert.ok(event.hasOwnProperty("message")); - assert.ok(event.hasOwnProperty("severity")); - - assert.strictEqual(event.message, "something"); - - body.event.message = "something else"; - context.message = JSON.stringify(body); - next(new Error("error!"), context); - } - - var logger = new SplunkLogger(config); - logger.use(middleware); - - var initialData = "something"; - var initialContext = { - message: initialData - }; - - var run = false; - - // Wrap the default error callback for code coverage - var errCallback = logger.error; - logger.error = function(err, context) { - run = true; - assert.ok(err); - assert.ok(context); - assert.strictEqual(err.message, "error!"); - - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - var event = body.event; - assert.ok(event.hasOwnProperty("message")); - assert.ok(event.hasOwnProperty("severity")); - - body.event.message = "something else"; - context.message = JSON.stringify(body); - - mute(); - errCallback(err, context); - unmute(); - - done(); - }; - - // Fire & forget, the callback won't be called anyways due to the error - logger.send(initialContext); - - assert.ok(run); - assert.strictEqual(middlewareCount, 1); - }); - it("should get error and context sending twice using default error handler", function(done) { - var config = { - token: "token-goes-here", - maxBatchCount: 1 - }; - - var middlewareCount = 0; - - function middleware(context, next) { - middlewareCount++; - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - var event = body.event; - assert.ok(event.hasOwnProperty("message")); - assert.ok(event.hasOwnProperty("severity")); - - assert.strictEqual(event.message, "something"); - assert.strictEqual(event.severity, "info"); - - body.event.message = "something else"; - context.message = JSON.stringify(body); - next(new Error("error!"), context); - } - - var logger = new SplunkLogger(config); - logger.use(middleware); - - var initialData = "something"; - var context1 = { - message: initialData - }; - - // Wrap the default error callback for code coverage - var errCallback = logger.error; - logger.error = function(err, context) { - assert.ok(err); - assert.strictEqual(err.message, "error!"); - assert.ok(context); - - var body = JSON.parse(context.message); - assert.ok(body.hasOwnProperty("time")); - assert.ok(body.hasOwnProperty("event")); - var event = body.event; - assert.ok(event.hasOwnProperty("message")); - assert.ok(event.hasOwnProperty("severity")); - - assert.strictEqual(event.message, "something else"); - - mute(); - errCallback(err, context); - unmute(); - - if (middlewareCount === 2) { - done(); - } - }; - - // Fire & forget, the callback won't be called anyways due to the error - logger.send(context1); - // Reset the data, hopefully this doesn't explode - var context2 = JSON.parse(JSON.stringify(context1)); - context2.message = "something"; - logger.send(context2); - - assert.strictEqual(middlewareCount, 2); - }); - }); describe("using retry", function() { it("should retry exactly 0 times (send once only)", function(done) { var config = { From bd45dfa7dc97e5c158b6c1c5e63e6b39eb073cf9 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Sun, 15 Nov 2015 22:58:26 -0800 Subject: [PATCH 23/46] Update tests for 100% coverage --- splunklogger.js | 2 +- test/test_config.js | 42 ++++++++++++++++++++++++++++++++++++++++++ test/test_send.js | 42 ++++++++++++++++++++++++++++++++++++++++++ test/test_utils.js | 11 +++++++++++ 4 files changed, 96 insertions(+), 1 deletion(-) diff --git a/splunklogger.js b/splunklogger.js index 2e3edc5..1267141 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -447,7 +447,7 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { var requestOptions = this._initializeRequestOptions(this.requestOptions); requestOptions.body = this._validateMessage(context.message); requestOptions.headers["Authorization"] = "Splunk " + this.config.token; - // Manually set the content-type header for batched requests, default is application/json + // Manually set the content-type header, the default is application/json // since json is set to true. requestOptions.headers["Content-Type"] = "application/x-www-form-urlencoded"; requestOptions.url = this.config.protocol + "://" + this.config.host + ":" + this.config.port + this.config.path; diff --git a/test/test_config.js b/test/test_config.js index acb1804..6a78a38 100644 --- a/test/test_config.js +++ b/test/test_config.js @@ -221,6 +221,48 @@ describe("SplunkLogger", function() { assert.strictEqual("Batch interval must be a positive number, found: -1", err.message); } }); + it("should change the timer via _enableTimer()", function() { + var config = { + token: "a-token-goes-here-usually" + }; + + var logger = new SplunkLogger(config); + logger._enableTimer(1); + assert.strictEqual(logger._timerDuration, 1); + logger._enableTimer(2); + assert.strictEqual(logger._timerDuration, 2); + logger._disableTimer(); + }); + it("should disable the timer via _initializeConfig()", function() { + var config = { + token: "a-token-goes-here-usually" + }; + + var logger = new SplunkLogger(config); + logger._enableTimer(1); + assert.strictEqual(logger._timerDuration, 1); + logger._enableTimer(2); + assert.strictEqual(logger._timerDuration, 2); + + logger.config.autoFlush = false; + logger.config.batchInterval = 0; + + logger._initializeConfig(logger.config); + assert.ok(!logger._timerDuration); + assert.ok(!logger._timerID); + }); + it("should be noop when _disableTimer() is called when no timer is configured", function() { + var config = { + token: "a-token-goes-here-usually" + }; + + var logger = new SplunkLogger(config); + var old = logger._timerDuration; + assert.ok(!logger._timerDuration); + logger._disableTimer(); + assert.ok(!logger._timerDuration); + assert.strictEqual(logger._timerDuration, old); + }); it("should set a batch interval timer with autoFlush on, & batchInterval set", function() { var config = { token: "a-token-goes-here-usually", diff --git a/test/test_send.js b/test/test_send.js index 61333fd..9691483 100644 --- a/test/test_send.js +++ b/test/test_send.js @@ -1839,4 +1839,46 @@ describe("SplunkLogger send (integration tests)", function() { }, 300); }); }); + describe("using custom eventFormatter", function() { + it("should use custom event formatter, instead of the default", function(done) { + var config = { + token: configurationFile.token, + maxBatchCount: 1 + }; + var logger = new SplunkLogger(config); + + logger.eventFormatter = function(message, severity) { + var ret = "[" + severity + "]"; + for (var key in message) { + if (message.hasOwnProperty(key)) { + ret += key + "=" + message[key] + ", "; + } + } + return ret; + }; + + var post = logger._post; + logger._post = function(opts, callback) { + var expected = "[info]some=data, asObject=true, num=123, "; + + assert.ok(opts); + assert.ok(opts.hasOwnProperty("body")); + console.log(typeof opts.body); + var body = JSON.parse(opts.body); + assert.ok(body.hasOwnProperty("event")); + assert.ok(body.hasOwnProperty("time")); + + assert.strictEqual(body.event, expected); + + post(opts, callback); + }; + + logger.send({message: {some: "data", asObject: true, num: 123}}, function(err, resp, body) { + assert.ok(!err); + assert.strictEqual(body.code, successBody.code); + assert.strictEqual(body.text, successBody.text); + done(); + }); + }); + }); }); diff --git a/test/test_utils.js b/test/test_utils.js index 9f09eb5..9aa92dc 100644 --- a/test/test_utils.js +++ b/test/test_utils.js @@ -560,6 +560,17 @@ describe("Utils", function() { assert.strictEqual(utils.orByProp("x", b, a), "x value"); assert.strictEqual(utils.orByProp("y", a, b), "y value"); }); + it("should ignore a null argument", function() { + var a = { + x: "x value" + }; + var b = { + y: "y value" + }; + + assert.strictEqual(utils.orByProp("x", null, b, a), "x value"); + assert.strictEqual(utils.orByProp("y", null, a, b), "y value"); + }); }); describe("orByBooleanProp", function() { it("should pick first value of 2", function() { From 1a0c5bb6cfd93d74ae34d7a687f961a5bfc777f0 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Sun, 15 Nov 2015 22:58:45 -0800 Subject: [PATCH 24/46] Added new example for custom event format --- examples/all_batching.js | 5 +- examples/basic.js | 28 +++++----- examples/custom_format.js | 111 ++++++++++++++++++++++++++++++++++++++ examples/retry.js | 4 +- 4 files changed, 129 insertions(+), 19 deletions(-) create mode 100644 examples/custom_format.js diff --git a/examples/all_batching.js b/examples/all_batching.js index fe7e278..6d88802 100644 --- a/examples/all_batching.js +++ b/examples/all_batching.js @@ -30,9 +30,8 @@ var SplunkLogger = require("../index").Logger; /** * Only the token property is required. * - * Here, batchInterval is set to flush every 1 - * second, when 10 events are queued, or - * when the size of queued events totals + * Here, batchInterval is set to flush every 1 second, when + * 10 events are queued, or when the size of queued events totals * more than 1kb. */ var config = { diff --git a/examples/basic.js b/examples/basic.js index a457b4d..b6747cd 100644 --- a/examples/basic.js +++ b/examples/basic.js @@ -66,23 +66,23 @@ console.log("Sending payload", payload); * * https://localhost:8088/services/collector/event/1.0 * - * with the following data + * with the following body * - * "{ - * metadata: { - * source: "chicken coop", - * sourcetype: "httpevent", - * index: "main", - * host: "farm.local" + * { + * "metadata": { + * "source": "chicken coop", + * "sourcetype": "httpevent", + * "index": "main", + * "host": "farm.local" * }, - * event: { - * message: { - * temperature: "70F", - * chickenCount: 500 + * "event": { + * "message": { + * "temperature": "70F", + * "chickenCount": 500 * }, - * severity: "info" - * }, - * }" + * "severity": "info" + * } + * } * */ Logger.send(payload, function(err, resp, body) { diff --git a/examples/custom_format.js b/examples/custom_format.js new file mode 100644 index 0000000..a09beb6 --- /dev/null +++ b/examples/custom_format.js @@ -0,0 +1,111 @@ +/* + * Copyright 2015 Splunk, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"): you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +/** + * This example shows how to use a custom event format for SplunkLogger. + */ + +// Change to require("splunk-logging").Logger; +var SplunkLogger = require("../index").Logger; + +/** + * Only the token property is required. + */ +var config = { + token: "your-token-here", + url: "https://localhost:8088", + maxBatchCount: 1 // Send events 1 at a time +}; + +// Create a new logger +var Logger = new SplunkLogger(config); + +Logger.error = function(err, context) { + // Handle errors here + console.log("error", err, "context", context); +}; + +/** + * Override the default eventFormatter() function, + * which takes a message and severity, returning + * any type - string or object are recommended. + * + * The message parameter can be any type. It will + * be whatever was passed to Logger.send(). + * Severity will always be a string. + * + * In this example, we're + */ +Logger.eventFormatter = function(message, severity) { + var event = "[" + severity + "]"; + + if (typeof message === "object") { + for (var key in message) { + event += key + "=" + message[key] + " "; + } + } + else { + event += "message=" + message + " "; + } + + return event; +}; + +// Define the payload to send to Splunk's Event Collector +var payload = { + // Message can be anything, it doesn't have to be an object + message: { + temperature: "70F", + chickenCount: 500 + }, + // Metadata is optional + metadata: { + source: "chicken coop", + sourcetype: "httpevent", + index: "main", + host: "farm.local", + }, + // Severity is also optional + severity: "info" +}; + +console.log("Sending payload", payload); + +/** + * Since maxBatchCount is set to 1, calling send + * will immediately send the payload. + * + * The underlying HTTP POST request is made to + * + * https://localhost:8088/services/collector/event/1.0 + * + * with the following body + * + * { + * "metadata": { + * "source": "chicken coop", + * "sourcetype": "httpevent", + * "index": "main", + * "host": "farm.local" + * }, + * "event": "[info]temperature=70F, chickenCount=500" + * } + * + */ +Logger.send(payload, function(err, resp, body) { + // If successful, body will be { text: 'Success', code: 0 } + console.log("Response from Splunk", body); +}); \ No newline at end of file diff --git a/examples/retry.js b/examples/retry.js index 72d053b..0f9c3f8 100644 --- a/examples/retry.js +++ b/examples/retry.js @@ -25,8 +25,8 @@ var SplunkLogger = require("../index").Logger; * Only the token property is required. * * Here we've set maxRetries to 5, - * If there are any connection errors the request - * to Splunk will be retried up to 5 times. + * If there are any connection errors the request to Splunk will + * be retried up to 5 times. * The default is 0. */ var config = { From d9ce2d61f76c804fa0fbece86fd86a2dfc7c38d5 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Mon, 16 Nov 2015 14:22:29 -0800 Subject: [PATCH 25/46] Correct the payload comment in the basic example --- examples/basic.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/examples/basic.js b/examples/basic.js index b6747cd..05477b8 100644 --- a/examples/basic.js +++ b/examples/basic.js @@ -69,12 +69,10 @@ console.log("Sending payload", payload); * with the following body * * { - * "metadata": { - * "source": "chicken coop", - * "sourcetype": "httpevent", - * "index": "main", - * "host": "farm.local" - * }, + * "source": "chicken coop", + * "sourcetype": "httpevent", + * "index": "main", + * "host": "farm.local", * "event": { * "message": { * "temperature": "70F", From e2f98813dc9804bb61996c48a80b26c7980242f0 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Mon, 16 Nov 2015 14:24:21 -0800 Subject: [PATCH 26/46] Correct comment in all batching example --- examples/all_batching.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/all_batching.js b/examples/all_batching.js index 6d88802..f56c07f 100644 --- a/examples/all_batching.js +++ b/examples/all_batching.js @@ -86,7 +86,7 @@ console.log("Queuing second payload", payload2); Logger.send(payload2); /** - * Since we've configured autoFlush, we don't need + * Since we've configured batching, we don't need * to do anything at this point. Events will * will be sent to Splunk automatically based * on the batching settings above. From 8fa15ab88c39342c72dd86ba04a2cc61b7d58e92 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Mon, 16 Nov 2015 16:06:20 -0800 Subject: [PATCH 27/46] Refactor contextQueue to serializedContextQueue This change optimizes the flush() function since events will only be serialized once after passed to send(). Without middleware, we no longer need to store un-serialized objects. --- splunklogger.js | 42 ++++++++++++++-------------- test/test_send.js | 70 +++++++++++++++++++++++------------------------ 2 files changed, 57 insertions(+), 55 deletions(-) diff --git a/splunklogger.js b/splunklogger.js index 1267141..6997647 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -68,7 +68,7 @@ function _defaultEventFormatter(message, severity) { * @property {object} config - Configuration settings for this SplunkLogger instance. * @param {object} requestOptions - Options to pass to {@link https://github.com/request/request#requestpost|request.post()}. * See the {@link http://github.com/request/request|request documentation} for all available options. - * @property {object[]} contextQueue - Queue of context objects to be sent to Splunk. + * @property {object[]} serializedContextQueue - Queue of serialized context objects to be sent to Splunk. * @property {function} eventFormatter - Formats events, returning an event as a string, function(message, severity). * Can be overwritten, the default event formatter will display event and severity as properties in a JSON object. * @property {function} error - A callback function for errors: function(err, context). @@ -102,7 +102,7 @@ var SplunkLogger = function(config) { this._timerDuration = 0; this.config = this._initializeConfig(config); this.requestOptions = this._initializeRequestOptions(); - this.contextQueue = []; + this.serializedContextQueue = []; this.eventsBatchSize = 0; this.eventFormatter = _defaultEventFormatter; this.error = _err; @@ -142,11 +142,11 @@ var defaultConfig = { protocol: "https", port: 8088, level: SplunkLogger.prototype.levels.INFO, - autoFlush: true, + autoFlush: true, // TODO: remove all refs to this property, !autoFlush = all batch settings === 0 maxRetries: 0, batchInterval: 0, maxBatchSize: 0, - maxBatchCount: 0 + maxBatchCount: 0 // TODO: set to 1 }; var defaultRequestOptions = { @@ -168,9 +168,10 @@ SplunkLogger.prototype._disableTimer = function() { } }; +// TODO: update doc /** * Configures an interval timer to flush any events in - * this.contextQueue at the specified interval. + * this.serializedContextQueue at the specified interval. * * param {Number} interval - The batch interval in milliseconds. * @private @@ -192,7 +193,7 @@ SplunkLogger.prototype._enableTimer = function(interval) { var that = this; this._timerID = setInterval(function() { - if (that.contextQueue.length > 0) { + if (that.serializedContextQueue.length > 0) { that.flush(); } }, interval); @@ -560,20 +561,21 @@ SplunkLogger.prototype.send = function(context, callback) { context = this._initializeContext(context); // Store the context, and its estimated length - this.contextQueue.push(context); - this.eventsBatchSize += Buffer.byteLength(JSON.stringify(this._makeBody(context)), "utf8"); + var currentEvent = JSON.stringify(this._makeBody(context)); + this.serializedContextQueue.push(currentEvent); + this.eventsBatchSize += Buffer.byteLength(currentEvent, "utf8"); var batchOverSize = this.eventsBatchSize > this.config.maxBatchSize && this.config.maxBatchSize > 0; - var batchOverCount = this.contextQueue.length >= this.config.maxBatchCount && this.config.maxBatchCount > 0; + var batchOverCount = this.serializedContextQueue.length >= this.config.maxBatchCount && this.config.maxBatchCount > 0; - // Only flush if not using manual batching, and if the contextQueue is too large or has many events + // Only flush if not using manual batching, and if the queue is too large or has many events if (this.config.autoFlush && (batchOverSize || batchOverCount)) { this.flush(callback || function(){}); } }; /** - * Manually send all events in this.contextQueue to Splunk. + * Manually send all events in this.serializedContextQueue to Splunk. * * @param {function} [callback] - A callback function: function(err, response, body). * @public @@ -581,17 +583,17 @@ SplunkLogger.prototype.send = function(context, callback) { SplunkLogger.prototype.flush = function(callback) { callback = callback || function(){}; - var context = {}; - // Empty the queue, reset the eventsBatchSize - var queue = this.contextQueue; - this.contextQueue = []; + var queue = this.serializedContextQueue; + this.serializedContextQueue = []; this.eventsBatchSize = 0; - var data = ""; - for (var i = 0; i < queue.length; i++) { - data += JSON.stringify(this._makeBody(queue[i])); - } - context.message = data; + + // Send all queued events + var data = queue.join(""); + var context = { + message: data + }; + this._sendEvents(context, callback); }; diff --git a/test/test_send.js b/test/test_send.js index 9691483..d0fa2c8 100644 --- a/test/test_send.js +++ b/test/test_send.js @@ -212,11 +212,11 @@ describe("SplunkLogger send (integration tests)", function() { message: data }; - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); logger.send(context); setTimeout(function() { - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); done(); }, 500); @@ -606,7 +606,7 @@ describe("SplunkLogger send (integration tests)", function() { logger.send(context2); setTimeout(function() { - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); assert.strictEqual(sent, 2); done(); @@ -632,7 +632,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(errContext); }; - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); logger.flush(function(err, resp, body) { assert.ok(!err); @@ -641,7 +641,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.strictEqual(resp.body, body); assert.strictEqual(body.text, noDataBody.text); assert.strictEqual(body.code, noDataBody.code); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); done(); }); @@ -663,10 +663,10 @@ describe("SplunkLogger send (integration tests)", function() { }; // Nothing should be sent if queue is empty - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); logger.flush(); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); }); it("should flush a batch of 1 event with valid token", function(done) { @@ -684,7 +684,7 @@ describe("SplunkLogger send (integration tests)", function() { logger.send(context); - assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.serializedContextQueue.length, 1); assert.ok(logger.eventsBatchSize > 50); logger.flush(function(err, resp, body) { assert.ok(!err); @@ -692,7 +692,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.strictEqual(resp.body, body); assert.strictEqual(body.text, successBody.text); assert.strictEqual(body.code, successBody.code); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); done(); }); @@ -713,7 +713,7 @@ describe("SplunkLogger send (integration tests)", function() { logger.send(context); logger.send(context); - assert.strictEqual(logger.contextQueue.length, 2); + assert.strictEqual(logger.serializedContextQueue.length, 2); assert.ok(logger.eventsBatchSize > 100); logger.flush(function(err, resp, body) { assert.ok(!err); @@ -721,7 +721,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.strictEqual(resp.body, body); assert.strictEqual(body.text, successBody.text); assert.strictEqual(body.code, successBody.code); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); done(); }); @@ -1196,7 +1196,7 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.strictEqual(logger._timerDuration, 100); assert.strictEqual(posts, 0); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); // Clean up the timer @@ -1234,7 +1234,7 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.strictEqual(logger._timerDuration, 100); assert.strictEqual(posts, 1); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); // Clean up the timer @@ -1276,7 +1276,7 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.strictEqual(logger._timerDuration, 100); assert.strictEqual(posts, 1); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); // Clean up the timer logger._disableTimer(); @@ -1319,7 +1319,7 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.strictEqual(logger._timerDuration, 200); assert.strictEqual(posts, 1); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); // Clean up the timer @@ -1381,7 +1381,7 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.strictEqual(posts, 1); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); // Clean up the timer @@ -1428,7 +1428,7 @@ describe("SplunkLogger send (integration tests)", function() { logger.send(payload2); logger.send(payload2); - assert.strictEqual(logger.contextQueue.length, 2); + assert.strictEqual(logger.serializedContextQueue.length, 2); assert.ok(logger.eventsBatchSize > 150); logger.send(payload2); // This should trigger a flush run = true; @@ -1437,7 +1437,7 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.ok(run); assert.strictEqual(posts, 2); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); // Clean up the timer @@ -1479,7 +1479,7 @@ describe("SplunkLogger send (integration tests)", function() { }; logger.send(payload2); - assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.serializedContextQueue.length, 1); assert.ok(logger.eventsBatchSize > 50); run = true; }, 150); @@ -1488,7 +1488,7 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.ok(run); assert.strictEqual(posts, 2); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); // Clean up the timer @@ -1516,7 +1516,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 1); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); assert.ok(!err); @@ -1550,7 +1550,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 1); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); assert.ok(!err); @@ -1569,7 +1569,7 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 0); - assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.serializedContextQueue.length, 1); assert.ok(logger.eventsBatchSize > 50); logger.send(payload); @@ -1578,7 +1578,7 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 1); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); }, 400); }); @@ -1612,7 +1612,7 @@ describe("SplunkLogger send (integration tests)", function() { // Make sure the event wasn't flushed yet setTimeout(function() { - assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.serializedContextQueue.length, 1); assert.ok(logger.eventsBatchSize > 50); }, 150); @@ -1622,7 +1622,7 @@ describe("SplunkLogger send (integration tests)", function() { logger._disableTimer(); assert.strictEqual(posts, 1); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); done(); }, 250); @@ -1657,7 +1657,7 @@ describe("SplunkLogger send (integration tests)", function() { // Event should be sent before the interval timer runs the first time setTimeout(function() { - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(posts, 1); }, 150); @@ -1667,7 +1667,7 @@ describe("SplunkLogger send (integration tests)", function() { logger._disableTimer(); assert.strictEqual(posts, 1); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); done(); }, 250); }); @@ -1691,7 +1691,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 1); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); assert.ok(!err); @@ -1737,7 +1737,7 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 0); - assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.serializedContextQueue.length, 1); assert.ok(logger.eventsBatchSize > 50); done(); @@ -1761,7 +1761,7 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 1); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); assert.ok(!err); @@ -1780,7 +1780,7 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 0); - assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.serializedContextQueue.length, 1); assert.ok(logger.eventsBatchSize > 50); logger.send(payload); @@ -1789,7 +1789,7 @@ describe("SplunkLogger send (integration tests)", function() { setTimeout(function() { assert.ok(!logger._timerID); assert.strictEqual(posts, 1); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); }, 400); }); @@ -1823,7 +1823,7 @@ describe("SplunkLogger send (integration tests)", function() { // Make sure the event wasn't flushed yet setTimeout(function() { - assert.strictEqual(logger.contextQueue.length, 1); + assert.strictEqual(logger.serializedContextQueue.length, 1); assert.ok(logger.eventsBatchSize > 50); }, 150); @@ -1833,7 +1833,7 @@ describe("SplunkLogger send (integration tests)", function() { logger._disableTimer(); assert.strictEqual(posts, 1); - assert.strictEqual(logger.contextQueue.length, 0); + assert.strictEqual(logger.serializedContextQueue.length, 0); assert.strictEqual(logger.eventsBatchSize, 0); done(); }, 300); From 4faf82787b03f8a9a9e2e1ec1b9953508c7e86bb Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Mon, 16 Nov 2015 20:49:36 -0800 Subject: [PATCH 28/46] Remove the config.autoFlush property. To disable autoFlush behavior, set all batching settings to 0. It is still enabled by default - flushing events 1 by 1. Modify this behavior through the config object. --- splunklogger.js | 55 +++++++++--------------- test/test_config.js | 76 ++++----------------------------- test/test_send.js | 101 ++++++++++++++++---------------------------- test/test_utils.js | 10 ++--- utils.js | 8 ++-- 5 files changed, 73 insertions(+), 177 deletions(-) diff --git a/splunklogger.js b/splunklogger.js index 6997647..40799e7 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -59,8 +59,7 @@ function _defaultEventFormatter(message, severity) { * var config = { * token: "your-token-here", * name: "my application", - * host: "splunk.local", - * autoFlush: true + * url: "https://splunk.local:8088" * }; * * var logger = new SplunkLogger(config); @@ -87,13 +86,12 @@ function _defaultEventFormatter(message, severity) { * the corresponding property is set on config. * @param {string} [config.level=info] - Logging level to use, will show up as the severity field of an event, see * [SplunkLogger.levels]{@link SplunkLogger#levels} for common levels. - * @param {bool} [config.autoFlush=true] - Send events immediately or not. - * @param {number} [config.batchInterval=0] - If config.autoFlush === true, automatically flush events after this many milliseconds. + * @param {number} [config.batchInterval=0] - Automatically flush events after this many milliseconds. * When set to a non-positive value, events will be sent one by one. This setting is ignored when non-positive. - * @param {number} [config.maxBatchSize=0] - If config.autoFlush === true, automatically flush events after the size of queued + * @param {number} [config.maxBatchSize=0] - Automatically flush events after the size of queued * events exceeds this many bytes. This setting is ignored when non-positive. - * @param {number} [config.maxBatchCount=0] - If config.autoFlush === true, automatically flush events after this many - * events have been queued. This setting is ignored when non-positive. + * @param {number} [config.maxBatchCount=1] - Automatically flush events after this many + * events have been queued. Defaults to flush immediately on sending an event. This setting is ignored when non-positive. * @constructor * @throws Will throw an error if the config parameter is malformed. */ @@ -142,11 +140,10 @@ var defaultConfig = { protocol: "https", port: 8088, level: SplunkLogger.prototype.levels.INFO, - autoFlush: true, // TODO: remove all refs to this property, !autoFlush = all batch settings === 0 maxRetries: 0, batchInterval: 0, maxBatchSize: 0, - maxBatchCount: 0 // TODO: set to 1 + maxBatchCount: 1 }; var defaultRequestOptions = { @@ -266,40 +263,25 @@ SplunkLogger.prototype._initializeConfig = function(config) { ret.maxRetries = utils.orByProp("maxRetries", config, ret, defaultConfig); ret.maxRetries = utils.validateNonNegativeInt(ret.maxRetries, "Max retries"); - ret.autoFlush = utils.orByBooleanProp("autoFlush", config, this.config, defaultConfig); - - // If manual batching, don't complain about batch settings from defaultConfig - if (!ret.autoFlush) { - ret.maxBatchCount = utils.orByProp("maxBatchCount", config, ret) || 0; - ret.maxBatchSize = utils.orByProp("maxBatchSize", config, ret) || 0; - ret.batchInterval = utils.orByProp("batchInterval", config, ret) || 0; - } - else { - ret.maxBatchCount = utils.orByProp("maxBatchCount", config, ret, defaultConfig); - ret.maxBatchSize = utils.orByProp("maxBatchSize", config, ret, defaultConfig); - ret.batchInterval = utils.orByProp("batchInterval", config, ret, defaultConfig); - } - + // Batching settings + ret.maxBatchCount = utils.orByFalseyProp("maxBatchCount", config, ret, defaultConfig); ret.maxBatchCount = utils.validateNonNegativeInt(ret.maxBatchCount, "Max batch count"); + ret.maxBatchSize = utils.orByFalseyProp("maxBatchSize", config, ret, defaultConfig); ret.maxBatchSize = utils.validateNonNegativeInt(ret.maxBatchSize, "Max batch size"); + ret.batchInterval = utils.orByFalseyProp("batchInterval", config, ret, defaultConfig); ret.batchInterval = utils.validateNonNegativeInt(ret.batchInterval, "Batch interval"); - // Error if autoFlush is off and any batching settings are set - if (!ret.autoFlush && (ret.batchInterval || ret.maxBatchSize || ret.maxBatchCount)) { - throw new Error("Autoflush is disabled, cannot configure batching settings."); - } - // Has the interval timer not started, and needs to be started? - var startTimer = !this._timerID && ret.autoFlush && ret.batchInterval > 0; + var startTimer = !this._timerID && ret.batchInterval > 0; // Has the interval timer already started, and the interval changed to a different duration? - var changeTimer = this._timerID && ret.autoFlush && this._timerDuration !== ret.batchInterval && ret.batchInterval > 0; + var changeTimer = this._timerID && this._timerDuration !== ret.batchInterval && ret.batchInterval > 0; // Upsert the timer if (startTimer || changeTimer) { this._enableTimer(ret.batchInterval); } // Disable timer - there is currently a timer, but config says we no longer need a timer - else if (this._timerID && (this._timerDuration < 0 || !ret.autoFlush)) { + else if (this._timerID && (ret.batchInterval <= 0 || this._timerDuration < 0)) { this._disableTimer(); } } @@ -509,7 +491,7 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { }; /** - * Sends or queues data to be sent based on this.config.autoFlush. + * Sends or queues data to be sent based on batching settings. * Default behavior is to send immediately. * * @example @@ -527,7 +509,7 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { * chickenCount: 500 * }, * severity: "info", - * { + * metadata: { * source: "chicken coop", * sourcetype: "httpevent", * index: "main", @@ -535,7 +517,8 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { * } * }; * - * // The callback is only used if autoFlush is set to false. + * // The callback is only used if maxBatchCount=1, or + * // batching thresholds have been exceeded. * logger.send(payload, function(err, resp, body) { * if (err) { * console.log("error:", err); @@ -568,8 +551,8 @@ SplunkLogger.prototype.send = function(context, callback) { var batchOverSize = this.eventsBatchSize > this.config.maxBatchSize && this.config.maxBatchSize > 0; var batchOverCount = this.serializedContextQueue.length >= this.config.maxBatchCount && this.config.maxBatchCount > 0; - // Only flush if not using manual batching, and if the queue is too large or has many events - if (this.config.autoFlush && (batchOverSize || batchOverCount)) { + // Only flush if the queue's byte size is too large, or has too many events + if (batchOverSize || batchOverCount) { this.flush(callback || function(){}); } }; diff --git a/test/test_config.js b/test/test_config.js index 6a78a38..f6f7ecc 100644 --- a/test/test_config.js +++ b/test/test_config.js @@ -141,7 +141,6 @@ describe("SplunkLogger", function() { assert.strictEqual("info", logger.config.level); assert.strictEqual(logger.levels.INFO, logger.config.level); assert.strictEqual(8088, logger.config.port); - assert.strictEqual(true, logger.config.autoFlush); assert.strictEqual(0, logger.config.maxRetries); assert.strictEqual(0, logger.config.batchInterval); assert.strictEqual(0, logger.config.maxBatchSize); @@ -157,11 +156,10 @@ describe("SplunkLogger", function() { assert.strictEqual(expectedRO.strictSSL, logger.requestOptions.strictSSL); assert.strictEqual(Object.keys(expectedRO.headers).length, Object.keys(logger.requestOptions.headers).length); }); - it("should set remaining defaults when setting config with token, autoFlush off, & level", function() { + it("should set remaining defaults when setting config with token, batching off, & level", function() { var config = { token: "a-token-goes-here-usually", - level: "important", - autoFlush: false + level: "important" }; var logger = new SplunkLogger(config); @@ -172,7 +170,6 @@ describe("SplunkLogger", function() { assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("https", logger.config.protocol); assert.strictEqual("important", logger.config.level); - assert.strictEqual(false, logger.config.autoFlush); assert.strictEqual(8088, logger.config.port); assert.strictEqual(0, logger.config.maxRetries); }); @@ -233,6 +230,7 @@ describe("SplunkLogger", function() { assert.strictEqual(logger._timerDuration, 2); logger._disableTimer(); }); + // TODO: fix this test it("should disable the timer via _initializeConfig()", function() { var config = { token: "a-token-goes-here-usually" @@ -244,8 +242,8 @@ describe("SplunkLogger", function() { logger._enableTimer(2); assert.strictEqual(logger._timerDuration, 2); - logger.config.autoFlush = false; logger.config.batchInterval = 0; + logger.config.maxBatchCount = 0; logger._initializeConfig(logger.config); assert.ok(!logger._timerDuration); @@ -263,11 +261,10 @@ describe("SplunkLogger", function() { assert.ok(!logger._timerDuration); assert.strictEqual(logger._timerDuration, old); }); - it("should set a batch interval timer with autoFlush on, & batchInterval set", function() { + it("should set a batch interval timer with batching on, & batchInterval set", function() { var config = { token: "a-token-goes-here-usually", - batchInterval: 100, - autoFlush: true + batchInterval: 100 }; var logger = new SplunkLogger(config); @@ -280,15 +277,13 @@ describe("SplunkLogger", function() { assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("https", logger.config.protocol); assert.strictEqual("info", logger.config.level); - assert.strictEqual(true, logger.config.autoFlush); assert.strictEqual(100, logger.config.batchInterval); assert.strictEqual(8088, logger.config.port); assert.strictEqual(0, logger.config.maxRetries); }); - it("should not set a batch interval timer with autoFlush on, & default batchInterval", function() { + it("should not set a batch interval timer with batching on, & default batchInterval", function() { var config = { - token: "a-token-goes-here-usually", - autoFlush: true + token: "a-token-goes-here-usually" }; var logger = new SplunkLogger(config); @@ -301,27 +296,10 @@ describe("SplunkLogger", function() { assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("https", logger.config.protocol); assert.strictEqual("info", logger.config.level); - assert.strictEqual(true, logger.config.autoFlush); assert.strictEqual(0, logger.config.batchInterval); assert.strictEqual(8088, logger.config.port); assert.strictEqual(0, logger.config.maxRetries); }); - it("should error trying set a batch interval timer with autoFlush off, & batchInterval set", function() { - var config = { - token: "a-token-goes-here-usually", - batchInterval: 100, - autoFlush: false - }; - - try { - var logger = new SplunkLogger(config); - assert.ok(!logger, "Expected an error."); - } - catch (err) { - assert.ok(err); - assert.strictEqual(err.message, "Autoflush is disabled, cannot configure batching settings."); - } - }); it("should error when maxBatchCount=NaN", function() { var config = { token: "a-token-goes-here-usually", @@ -536,7 +514,6 @@ describe("SplunkLogger", function() { assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("https", logger.config.protocol); assert.strictEqual("info", logger.config.level); - assert.strictEqual(true, logger.config.autoFlush); assert.strictEqual(8088, logger.config.port); assert.strictEqual(0, logger.config.maxRetries); }); @@ -554,7 +531,6 @@ describe("SplunkLogger", function() { assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("https", logger.config.protocol); assert.strictEqual("info", logger.config.level); - assert.strictEqual(true, logger.config.autoFlush); assert.strictEqual(8088, logger.config.port); assert.strictEqual(10, logger.config.maxRetries); }); @@ -990,41 +966,5 @@ describe("SplunkLogger", function() { assert.strictEqual(Logger.config.level, expected.level); assert.strictEqual(Logger.config.port, expected.port); }); - it("should set autoFlush property to true after initially false", function() { - Object.prototype.something = "ignore"; - var config = { - token: "a-token-goes-here-usually", - url: "splunk.local", - autoFlush: false - }; - - var logger = new SplunkLogger(config); - var loggerConfig = logger.config; - - assert.ok(loggerConfig); - assert.ok(!loggerConfig.hasOwnProperty("something")); - assert.strictEqual(config.token, loggerConfig.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", loggerConfig.name); - assert.strictEqual("splunk.local", loggerConfig.host); - assert.strictEqual("/services/collector/event/1.0", loggerConfig.path); - assert.strictEqual("https", loggerConfig.protocol); - assert.strictEqual("info", loggerConfig.level); - assert.strictEqual(8088, loggerConfig.port); - assert.strictEqual(0, loggerConfig.maxRetries); - assert.strictEqual(false, loggerConfig.autoFlush); - - config.autoFlush = true; - loggerConfig = logger._initializeConfig(config); - - assert.strictEqual(config.token, loggerConfig.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", loggerConfig.name); - assert.strictEqual("splunk.local", loggerConfig.host); - assert.strictEqual("/services/collector/event/1.0", loggerConfig.path); - assert.strictEqual("https", loggerConfig.protocol); - assert.strictEqual("info", loggerConfig.level); - assert.strictEqual(8088, loggerConfig.port); - assert.strictEqual(0, loggerConfig.maxRetries); - assert.strictEqual(true, loggerConfig.autoFlush); - }); }); }); diff --git a/test/test_send.js b/test/test_send.js index d0fa2c8..6308d8b 100644 --- a/test/test_send.js +++ b/test/test_send.js @@ -156,11 +156,10 @@ describe("SplunkLogger _makeBody", function() { }); }); describe("SplunkLogger send (integration tests)", function() { - describe("using no middleware", function () { + describe("normal", function () { it("should error with bad token", function(done) { var config = { - token: "token-goes-here", - maxBatchCount: 1 + token: "token-goes-here" }; var logger = new SplunkLogger(config); @@ -200,8 +199,7 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should send without callback", function(done) { var config = { - token: configurationFile.token, - maxBatchCount: 1 + token: configurationFile.token }; var logger = new SplunkLogger(config); @@ -223,8 +221,7 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed with valid token", function(done) { var config = { - token: configurationFile.token, - maxBatchCount: 1 + token: configurationFile.token }; var logger = new SplunkLogger(config); @@ -246,8 +243,7 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed with valid token, using custom time", function(done) { var config = { - token: configurationFile.token, - maxBatchCount: 1 + token: configurationFile.token }; var logger = new SplunkLogger(config); @@ -273,8 +269,7 @@ describe("SplunkLogger send (integration tests)", function() { // TODO: test unsuccessfully sending to another index with specific index token settings it("should succeed with valid token, sending to a different index", function(done) { var config = { - token: configurationFile.token, - maxBatchCount: 1 + token: configurationFile.token }; var logger = new SplunkLogger(config); @@ -303,8 +298,7 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed with valid token, changing source", function(done) { var config = { - token: configurationFile.token, - maxBatchCount: 1 + token: configurationFile.token }; var logger = new SplunkLogger(config); @@ -329,8 +323,7 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed with valid token, changing sourcetype", function(done) { var config = { - token: configurationFile.token, - maxBatchCount: 1 + token: configurationFile.token }; var logger = new SplunkLogger(config); @@ -355,8 +348,7 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed with valid token, changing host", function(done) { var config = { - token: configurationFile.token, - maxBatchCount: 1 + token: configurationFile.token }; var logger = new SplunkLogger(config); @@ -381,8 +373,7 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed with valid token", function(done) { var config = { - token: configurationFile.token, - maxBatchCount: 1 + token: configurationFile.token }; var logger = new SplunkLogger(config); @@ -404,8 +395,7 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should succeed without token passed through context", function(done) { var config = { - token: configurationFile.token, - maxBatchCount: 1 + token: configurationFile.token }; var logger = new SplunkLogger(config); @@ -430,8 +420,7 @@ describe("SplunkLogger send (integration tests)", function() { it("should fail on wrong protocol (assumes HTTP is invalid)", function(done) { var config = { token: configurationFile.token, - protocol: "http", - maxBatchCount: 1 + protocol: "http" }; var logger = new SplunkLogger(config); @@ -472,8 +461,7 @@ describe("SplunkLogger send (integration tests)", function() { it("should fail on wrong Splunk server", function(done) { var config = { token: configurationFile.token, - url: "https://something-so-invalid-that-it-should-never-exist.xyz:12345/junk", - maxBatchCount: 1 + url: "https://something-so-invalid-that-it-should-never-exist.xyz:12345/junk" }; var logger = new SplunkLogger(config); @@ -514,8 +502,7 @@ describe("SplunkLogger send (integration tests)", function() { it("should succeed with valid token, using non-default url", function(done) { var config = { token: configurationFile.token, - url: "https://localhost:8088/services/collector/event/1.0", - maxBatchCount: 1 + url: "https://localhost:8088/services/collector/event/1.0" }; var logger = new SplunkLogger(config); @@ -536,8 +523,7 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should error with valid token, using strict SSL", function(done) { var config = { - token: configurationFile.token, - maxBatchCount: 1 + token: configurationFile.token }; var logger = new SplunkLogger(config); @@ -579,8 +565,7 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should send 2 events with valid token, w/o callbacks", function(done) { var config = { - token: configurationFile.token, - maxBatchCount: 1 + token: configurationFile.token }; var logger = new SplunkLogger(config); @@ -613,11 +598,10 @@ describe("SplunkLogger send (integration tests)", function() { }, 1000); }); }); - describe("without autoFlush", function () { + describe("default batching settings", function () { it("should get no data response when flushing empty batch with valid token", function(done) { var config = { - token: configurationFile.token, - autoFlush: false + token: configurationFile.token }; var logger = new SplunkLogger(config); @@ -648,8 +632,7 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should be noop when flushing empty batch, without callback, with valid token", function(done) { var config = { - token: configurationFile.token, - autoFlush: false + token: configurationFile.token }; var logger = new SplunkLogger(config); @@ -672,7 +655,7 @@ describe("SplunkLogger send (integration tests)", function() { it("should flush a batch of 1 event with valid token", function(done) { var config = { token: configurationFile.token, - autoFlush: false + maxBatchCount: 0 // Use manual batching }; var logger = new SplunkLogger(config); @@ -700,7 +683,7 @@ describe("SplunkLogger send (integration tests)", function() { it("should flush a batch of 2 events with valid token", function(done) { var config = { token: configurationFile.token, - autoFlush: false + maxBatchCount: 0 }; var logger = new SplunkLogger(config); @@ -731,8 +714,7 @@ describe("SplunkLogger send (integration tests)", function() { it("should retry exactly 0 times (send once only)", function(done) { var config = { token: configurationFile.token, - maxRetries: 0, - maxBatchCount: 1 + maxRetries: 0 }; var logger = new SplunkLogger(config); @@ -1174,7 +1156,6 @@ describe("SplunkLogger send (integration tests)", function() { it("should not make a POST request if contextQueue is always empty", function(done) { var config = { token: configurationFile.token, - autoFlush: true, batchInterval: 100 }; var logger = new SplunkLogger(config); @@ -1207,8 +1188,8 @@ describe("SplunkLogger send (integration tests)", function() { it("should only make 1 POST request for 1 event", function(done) { var config = { token: configurationFile.token, - autoFlush: true, - batchInterval: 100 + batchInterval: 100, + maxBatchCount: 10 }; var logger = new SplunkLogger(config); @@ -1245,8 +1226,8 @@ describe("SplunkLogger send (integration tests)", function() { it("should only make 1 POST request for 2 events", function(done) { var config = { token: configurationFile.token, - autoFlush: true, - batchInterval: 100 + batchInterval: 100, + maxBatchCount: 10 }; var logger = new SplunkLogger(config); @@ -1286,7 +1267,6 @@ describe("SplunkLogger send (integration tests)", function() { it("should only make 1 POST request for 5 events", function(done) { var config = { token: configurationFile.token, - autoFlush: true, batchInterval: 200, maxBatchSize: 5000, maxBatchCount: 10 @@ -1329,8 +1309,7 @@ describe("SplunkLogger send (integration tests)", function() { }); it("should error when trying to set batchInterval to a negative value after logger creation", function() { var config = { - token: configurationFile.token, - autoFlush: false + token: configurationFile.token }; var logger = new SplunkLogger(config); @@ -1343,10 +1322,10 @@ describe("SplunkLogger send (integration tests)", function() { assert.strictEqual(err.message, "Batch interval must be a positive number, found: -1"); } }); - it("should flush a stale event after enabling autoFlush and batchInterval", function(done) { + it("should flush a stale event after enabling batching and batchInterval", function(done) { var config = { token: configurationFile.token, - autoFlush: false + maxBatchCount: 10 }; var logger = new SplunkLogger(config); @@ -1370,7 +1349,6 @@ describe("SplunkLogger send (integration tests)", function() { logger.send(payload); assert.strictEqual(logger._timerDuration, 0); - logger.config.autoFlush = true; logger.config.batchInterval = 100; logger._initializeConfig(logger.config); @@ -1392,7 +1370,6 @@ describe("SplunkLogger send (integration tests)", function() { it("should flush an event with batchInterval, then set batchInterval=0 and maxBatchCount=3 for manual batching", function(done) { var config = { token: configurationFile.token, - autoFlush: true, batchInterval: 100, maxBatchSize: 100000, maxBatchCount: 3 @@ -1448,8 +1425,8 @@ describe("SplunkLogger send (integration tests)", function() { it("should flush an event with batchInterval=100", function(done) { var config = { token: configurationFile.token, - autoFlush: true, - batchInterval: 100 + batchInterval: 100, + maxBatchCount: 10 }; var logger = new SplunkLogger(config); @@ -1500,9 +1477,7 @@ describe("SplunkLogger send (integration tests)", function() { describe("using max batch size", function() { it("should flush first event immediately with maxBatchSize=1", function(done) { var config = { - token: configurationFile.token, - autoFlush: true, - maxBatchSize: 1 + token: configurationFile.token }; var logger = new SplunkLogger(config); @@ -1535,7 +1510,7 @@ describe("SplunkLogger send (integration tests)", function() { it("should flush first 2 events after maxBatchSize>100", function(done) { var config = { token: configurationFile.token, - autoFlush: true, + maxBatchCount: 10, maxBatchSize: 100 }; var logger = new SplunkLogger(config); @@ -1585,9 +1560,9 @@ describe("SplunkLogger send (integration tests)", function() { it("should flush first event after 200ms, with maxBatchSize=200", function(done) { var config = { token: configurationFile.token, - autoFlush: true, maxBatchSize: 200, - batchInterval: 200 + batchInterval: 200, + maxBatchCount: 10 }; var logger = new SplunkLogger(config); @@ -1630,9 +1605,9 @@ describe("SplunkLogger send (integration tests)", function() { it("should flush first event before 200ms, with maxBatchSize=1", function(done) { var config = { token: configurationFile.token, - autoFlush: true, maxBatchSize: 1, - batchInterval: 200 + batchInterval: 200, + maxBatchCount: 10 }; var logger = new SplunkLogger(config); @@ -1796,7 +1771,6 @@ describe("SplunkLogger send (integration tests)", function() { it("should flush first event after 200ms, with maxBatchCount=10", function(done) { var config = { token: configurationFile.token, - autoFlush: true, maxBatchCount: 10, batchInterval: 200 }; @@ -1863,7 +1837,6 @@ describe("SplunkLogger send (integration tests)", function() { assert.ok(opts); assert.ok(opts.hasOwnProperty("body")); - console.log(typeof opts.body); var body = JSON.parse(opts.body); assert.ok(body.hasOwnProperty("event")); assert.ok(body.hasOwnProperty("time")); diff --git a/test/test_utils.js b/test/test_utils.js index 9aa92dc..3f2126b 100644 --- a/test/test_utils.js +++ b/test/test_utils.js @@ -572,7 +572,7 @@ describe("Utils", function() { assert.strictEqual(utils.orByProp("y", null, a, b), "y value"); }); }); - describe("orByBooleanProp", function() { + describe("orByFalseyProp", function() { it("should pick first value of 2", function() { var a = { x: false @@ -581,8 +581,8 @@ describe("Utils", function() { y: true }; - assert.strictEqual(utils.orByBooleanProp("x", a, b), false); - assert.strictEqual(utils.orByBooleanProp("y", b, a), true); + assert.strictEqual(utils.orByFalseyProp("x", a, b), false); + assert.strictEqual(utils.orByFalseyProp("y", b, a), true); }); it("should pick second value of 2", function() { var a = { @@ -592,8 +592,8 @@ describe("Utils", function() { y: true }; - assert.strictEqual(utils.orByBooleanProp("x", b, a), false); - assert.strictEqual(utils.orByBooleanProp("y", a, b), true); + assert.strictEqual(utils.orByFalseyProp("x", b, a), false); + assert.strictEqual(utils.orByFalseyProp("y", a, b), true); }); }); describe("validateNonNegativeInt", function() { diff --git a/utils.js b/utils.js index d158fe9..83f82a4 100644 --- a/utils.js +++ b/utils.js @@ -260,15 +260,16 @@ utils.orByProp = function(prop) { }; /** - * Like utisl.orByProp() but for a boolean property. + * Like utils.orByProp() but for a falsey property. * The first argument after prop with that property * defined will be returned. + * Useful for booleans and numbers. * * @param {string} [prop] - property name for other arguments. * @returns {boolean} * @static */ -utils.orByBooleanProp = function(prop) { +utils.orByFalseyProp = function(prop) { var ret = null; // Logic is reversed here, first value wins for (var i = arguments.length - 1; i > 0; i--) { @@ -276,8 +277,7 @@ utils.orByBooleanProp = function(prop) { ret = arguments[i][prop]; } } - // Convert whatever we have to a boolean - return !!ret; + return ret; }; /** From dab006bfe6cbb6916255dc9662cab2652511f207 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Mon, 16 Nov 2015 21:05:19 -0800 Subject: [PATCH 29/46] Update examples following refactor --- examples/all_batching.js | 8 ++------ examples/basic.js | 9 ++++----- examples/custom_format.js | 27 +++++++++++++++------------ examples/manual_batching.js | 11 +++++------ examples/retry.js | 2 +- splunklogger.js | 1 - 6 files changed, 27 insertions(+), 31 deletions(-) diff --git a/examples/all_batching.js b/examples/all_batching.js index f56c07f..14865aa 100644 --- a/examples/all_batching.js +++ b/examples/all_batching.js @@ -16,12 +16,8 @@ /** * This example shows how to batch events with the - * SplunkLogger with all available settings. - * - * By default autoFlush is enabled. - * - * By disabling autoFlush, events will be queued - * until flush() is called. + * SplunkLogger with all available settings: + * batchInterval, maxBatchCount, & maxBatchSize. */ // Change to require("splunk-logging").Logger; diff --git a/examples/basic.js b/examples/basic.js index 05477b8..ca82cd0 100644 --- a/examples/basic.js +++ b/examples/basic.js @@ -26,8 +26,7 @@ var SplunkLogger = require("../index").Logger; */ var config = { token: "your-token-here", - url: "https://localhost:8088", - maxBatchCount: 1 // Send events 1 at a time + url: "https://localhost:8088" }; // Create a new logger @@ -50,7 +49,7 @@ var payload = { source: "chicken coop", sourcetype: "httpevent", index: "main", - host: "farm.local", + host: "farm.local" }, // Severity is also optional severity: "info" @@ -59,8 +58,8 @@ var payload = { console.log("Sending payload", payload); /** - * Since maxBatchCount is set to 1, calling send - * will immediately send the payload. + * Since maxBatchCount is set to 1 by default, + * calling send will immediately send the payload. * * The underlying HTTP POST request is made to * diff --git a/examples/custom_format.js b/examples/custom_format.js index a09beb6..48266d7 100644 --- a/examples/custom_format.js +++ b/examples/custom_format.js @@ -47,7 +47,12 @@ Logger.error = function(err, context) { * be whatever was passed to Logger.send(). * Severity will always be a string. * - * In this example, we're + * In this example, we're building up a string + * of key=value pairs if message is an object, + * otherwise the message value is as value for + * the message key. + * This string is prefixed with the event + * severity in square brackets. */ Logger.eventFormatter = function(message, severity) { var event = "[" + severity + "]"; @@ -58,7 +63,7 @@ Logger.eventFormatter = function(message, severity) { } } else { - event += "message=" + message + " "; + event += "message=" + message; } return event; @@ -76,7 +81,7 @@ var payload = { source: "chicken coop", sourcetype: "httpevent", index: "main", - host: "farm.local", + host: "farm.local" }, // Severity is also optional severity: "info" @@ -85,8 +90,8 @@ var payload = { console.log("Sending payload", payload); /** - * Since maxBatchCount is set to 1, calling send - * will immediately send the payload. + * Since maxBatchCount is set to 1 by default, + * calling send will immediately send the payload. * * The underlying HTTP POST request is made to * @@ -95,13 +100,11 @@ console.log("Sending payload", payload); * with the following body * * { - * "metadata": { - * "source": "chicken coop", - * "sourcetype": "httpevent", - * "index": "main", - * "host": "farm.local" - * }, - * "event": "[info]temperature=70F, chickenCount=500" + * "source": "chicken coop", + * "sourcetype": "httpevent", + * "index": "main", + * "host": "farm.local", + * "event": "[info]temperature=70F chickenCount=500 " * } * */ diff --git a/examples/manual_batching.js b/examples/manual_batching.js index 3e2e991..3720c36 100644 --- a/examples/manual_batching.js +++ b/examples/manual_batching.js @@ -17,9 +17,8 @@ /** * This example shows how to batch events with the * SplunkLogger by manually calling flush. - * By default autoFlush is enabled. * - * By disabling autoFlush, events will be queued + * By setting maxbatchCount=0, events will be queued * until flush() is called. */ @@ -29,12 +28,12 @@ var SplunkLogger = require("../index").Logger; /** * Only the token property is required. * - * Here, autoFlush is set to false. + * Here, maxBatchCount is set to 0. */ var config = { token: "your-token-here", url: "https://localhost:8088", - autoFlush: false // Manually flush events + maxBatchCount: 0 // Manually flush events }; // Create a new logger @@ -57,7 +56,7 @@ var payload = { source: "chicken coop", sourcetype: "httpevent", index: "main", - host: "farm.local", + host: "farm.local" }, // Severity is also optional severity: "info" @@ -81,7 +80,7 @@ console.log("Queuing second payload", payload2); Logger.send(payload2); /** - * Since autoFlush is disabled, call flush manually. + * Call flush manually. * This will send both payloads in a single * HTTP request. * diff --git a/examples/retry.js b/examples/retry.js index 0f9c3f8..fd3a2f1 100644 --- a/examples/retry.js +++ b/examples/retry.js @@ -56,7 +56,7 @@ var payload = { source: "chicken coop", sourcetype: "httpevent", index: "main", - host: "farm.local", + host: "farm.local" }, // Severity is also optional severity: "info" diff --git a/splunklogger.js b/splunklogger.js index 40799e7..965f1b6 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -165,7 +165,6 @@ SplunkLogger.prototype._disableTimer = function() { } }; -// TODO: update doc /** * Configures an interval timer to flush any events in * this.serializedContextQueue at the specified interval. From a157135ab6a8b2459e90ef77a55eb2a23ce7550d Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Mon, 16 Nov 2015 21:24:07 -0800 Subject: [PATCH 30/46] Update version number, readme, changelog --- CHANGELOG.md | 24 +++++++++++++++++++++++ README.md | 7 ++++--- package.json | 2 +- splunklogger.js | 4 ++-- test/test_config.js | 48 ++++++++++++++++++++++----------------------- utils.js | 1 + 6 files changed, 56 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 21f0941..933af21 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,29 @@ # Splunk Logging Library for JavaScript (beta) +## v0.9.0 - beta + +### New Features & APIs + +* Added to configure automated batching with 3 settings: `batchInterval`, `maxBatchCount`, & `maxBatchSize`. +* Added the ability to retry sending to Splunk in the case of network errors with the `maxRetries` configuration setting. +* Added the ability to configure a custom Splunk event format by overriding `eventFormatter(message, severity)`. + +### Breaking Changes + +* Removed the `autoFlush` configuration setting. To achieve the same effect, set `config.maxBatchCount` to `0`. +* Removed support for middleware functions. +* The `context` object has been simplified, `config` and `requestOptions` can no longer be specified there - please use those settings directly on the logger. + +### Examples + +* Removed the `middleware.js` example. +* Renamed the `batching.js` example to `manual_batching`. +* Added the `all_batching.js`, `custom_format.js`, `retry.js` examples. + +### Minor changes + +* Significant refactor of internal functions. + ## v0.8.0 - beta * Beta release. \ No newline at end of file diff --git a/README.md b/README.md index 5ddfb64..fd5145c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Splunk Logging Library for JavaScript (beta) -#### Version 0.8.0 +#### Version 0.9.0 This project provides a simple interface for logging to Splunk's Event Collector. @@ -18,9 +18,10 @@ If you already have Node.js and npm installed, simply run: `npm install --save s See the `examples` folder for more examples: +* `all_batching.js`: shows how to configure a logger with the 3 batching settings: `batchInterval`, `maxBatchCount`, & `maxBatchSize`. +* `custom_format.js`: shows how to configure a logger so log message to Splunk using a custom format. * `basic.js`: shows how to configure a logger and send a log message to Splunk. -* `batching.js`: shows how to queue log messages, and send them in batches. -* `middleware.js`: shows how to add an express-like middleware function to be called before sending log messages to Splunk. +* `manual_batching.js`: shows how to queue log messages, and send them in batches by manually calling `flush()`. * `retry.js`: shows how to configure retries on errors. ### Basic example diff --git a/package.json b/package.json index b91f364..6ae8c13 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "splunk-logging", - "version": "0.8.0", + "version": "0.9.0", "description": "Splunk HTTP Event Collector logging interface", "homepage": "http://dev.splunk.com", "main": "index.js", diff --git a/splunklogger.js b/splunklogger.js index 965f1b6..2794aae 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -75,7 +75,7 @@ function _defaultEventFormatter(message, severity) { * * @param {object} config - Configuration settings for a new [SplunkLogger]{@link SplunkLogger}. * @param {string} config.token - Splunk HTTP Event Collector token, required. - * @param {string} [config.name=splunk-javascript-logging/0.8.0] - Name for this logger. + * @param {string} [config.name=splunk-javascript-logging/0.9.0] - Name for this logger. * @param {string} [config.host=localhost] - Hostname or IP address of Splunk server. * @param {string} [config.maxRetries=0] - How many times to retry when HTTP POST to Splunk fails. * @param {string} [config.path=/services/collector/event/1.0] - URL path to send data to on the Splunk server. @@ -134,7 +134,7 @@ SplunkLogger.prototype.levels = { }; var defaultConfig = { - name: "splunk-javascript-logging/0.8.0", + name: "splunk-javascript-logging/0.9.0", host: "localhost", path: "/services/collector/event/1.0", protocol: "https", diff --git a/test/test_config.js b/test/test_config.js index f6f7ecc..7eb5d0f 100644 --- a/test/test_config.js +++ b/test/test_config.js @@ -134,7 +134,7 @@ describe("SplunkLogger", function() { assert.ok(logger); assert.strictEqual(config.token, logger.config.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", logger.config.name); assert.strictEqual("localhost", logger.config.host); assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("https", logger.config.protocol); @@ -165,7 +165,7 @@ describe("SplunkLogger", function() { assert.ok(logger); assert.strictEqual(config.token, logger.config.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", logger.config.name); assert.strictEqual("localhost", logger.config.host); assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("https", logger.config.protocol); @@ -272,7 +272,7 @@ describe("SplunkLogger", function() { assert.ok(logger._timerID); assert.strictEqual(config.token, logger.config.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", logger.config.name); assert.strictEqual("localhost", logger.config.host); assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("https", logger.config.protocol); @@ -291,7 +291,7 @@ describe("SplunkLogger", function() { assert.ok(!logger._timerID); assert.strictEqual(config.token, logger.config.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", logger.config.name); assert.strictEqual("localhost", logger.config.host); assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("https", logger.config.protocol); @@ -372,7 +372,7 @@ describe("SplunkLogger", function() { assert.ok(!logger._timerID); assert.strictEqual(config.token, logger.config.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", logger.config.name); assert.strictEqual("localhost", logger.config.host); assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("http", logger.config.protocol); @@ -389,7 +389,7 @@ describe("SplunkLogger", function() { assert.ok(logger); assert.strictEqual(config.token, logger.config.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", logger.config.name); assert.strictEqual("localhost", logger.config.host); assert.strictEqual(config.path, logger.config.path); assert.strictEqual("https", logger.config.protocol); @@ -406,7 +406,7 @@ describe("SplunkLogger", function() { assert.ok(logger); assert.strictEqual(config.token, logger.config.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", logger.config.name); assert.strictEqual("localhost", logger.config.host); assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("https", logger.config.protocol); @@ -423,7 +423,7 @@ describe("SplunkLogger", function() { assert.ok(logger); assert.strictEqual(config.token, logger.config.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", logger.config.name); assert.strictEqual("localhost", logger.config.host); assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("https", logger.config.protocol); @@ -441,7 +441,7 @@ describe("SplunkLogger", function() { assert.ok(logger); assert.strictEqual(config.token, logger.config.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", logger.config.name); assert.strictEqual("splunk.local", logger.config.host); assert.strictEqual("/services/collector/different/1.0", logger.config.path); assert.strictEqual("http", logger.config.protocol); @@ -458,7 +458,7 @@ describe("SplunkLogger", function() { assert.ok(logger); assert.strictEqual(config.token, logger.config.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", logger.config.name); assert.strictEqual("localhost", logger.config.host); assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("http", logger.config.protocol); @@ -475,7 +475,7 @@ describe("SplunkLogger", function() { assert.ok(logger); assert.strictEqual(config.token, logger.config.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", logger.config.name); assert.strictEqual("splunk.local", logger.config.host); assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("http", logger.config.protocol); @@ -492,7 +492,7 @@ describe("SplunkLogger", function() { assert.ok(logger); assert.strictEqual(config.token, logger.config.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", logger.config.name); assert.strictEqual("splunk.local", logger.config.host); assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("http", logger.config.protocol); @@ -509,7 +509,7 @@ describe("SplunkLogger", function() { assert.ok(logger); assert.strictEqual(config.token, logger.config.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", logger.config.name); assert.strictEqual("splunk.local", logger.config.host); assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("https", logger.config.protocol); @@ -526,7 +526,7 @@ describe("SplunkLogger", function() { assert.ok(logger); assert.strictEqual(config.token, logger.config.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", logger.config.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", logger.config.name); assert.strictEqual("localhost", logger.config.host); assert.strictEqual("/services/collector/event/1.0", logger.config.path); assert.strictEqual("https", logger.config.protocol); @@ -658,7 +658,7 @@ describe("SplunkLogger", function() { assert.ok(loggerConfig); assert.strictEqual(config.token, loggerConfig.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", loggerConfig.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", loggerConfig.name); assert.strictEqual("localhost", loggerConfig.host); assert.strictEqual("/services/collector/event/1.0", loggerConfig.path); assert.strictEqual("https", loggerConfig.protocol); @@ -675,7 +675,7 @@ describe("SplunkLogger", function() { var loggerConfig = SplunkLogger.prototype._initializeConfig(config); assert.strictEqual(config.token, loggerConfig.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", loggerConfig.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", loggerConfig.name); assert.strictEqual("localhost", loggerConfig.host); assert.strictEqual("/services/collector/event/1.0", loggerConfig.path); assert.strictEqual("http", loggerConfig.protocol); @@ -692,7 +692,7 @@ describe("SplunkLogger", function() { assert.ok(loggerConfig); assert.strictEqual(config.token, loggerConfig.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", loggerConfig.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", loggerConfig.name); assert.strictEqual("localhost", loggerConfig.host); assert.strictEqual(config.path, loggerConfig.path); assert.strictEqual("https", loggerConfig.protocol); @@ -709,7 +709,7 @@ describe("SplunkLogger", function() { assert.ok(loggerConfig); assert.strictEqual(config.token, loggerConfig.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", loggerConfig.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", loggerConfig.name); assert.strictEqual("localhost", loggerConfig.host); assert.strictEqual("/services/collector/event/1.0", loggerConfig.path); assert.strictEqual("https", loggerConfig.protocol); @@ -726,7 +726,7 @@ describe("SplunkLogger", function() { assert.ok(loggerConfig); assert.strictEqual(config.token, loggerConfig.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", loggerConfig.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", loggerConfig.name); assert.strictEqual("splunk.local", loggerConfig.host); assert.strictEqual("/services/collector/different/1.0", loggerConfig.path); assert.strictEqual("http", loggerConfig.protocol); @@ -743,7 +743,7 @@ describe("SplunkLogger", function() { assert.ok(loggerConfig); assert.strictEqual(config.token, loggerConfig.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", loggerConfig.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", loggerConfig.name); assert.strictEqual("localhost", loggerConfig.host); assert.strictEqual("/services/collector/event/1.0", loggerConfig.path); assert.strictEqual("http", loggerConfig.protocol); @@ -760,7 +760,7 @@ describe("SplunkLogger", function() { assert.ok(loggerConfig); assert.strictEqual(config.token, loggerConfig.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", loggerConfig.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", loggerConfig.name); assert.strictEqual("splunk.local", loggerConfig.host); assert.strictEqual("/services/collector/event/1.0", loggerConfig.path); assert.strictEqual("https", loggerConfig.protocol); @@ -779,7 +779,7 @@ describe("SplunkLogger", function() { assert.ok(loggerConfig); assert.ok(!loggerConfig.hasOwnProperty("something")); assert.strictEqual(config.token, loggerConfig.token); - assert.strictEqual("splunk-javascript-logging/0.8.0", loggerConfig.name); + assert.strictEqual("splunk-javascript-logging/0.9.0", loggerConfig.name); assert.strictEqual("splunk.local", loggerConfig.host); assert.strictEqual("/services/collector/event/1.0", loggerConfig.path); assert.strictEqual("https", loggerConfig.protocol); @@ -904,7 +904,7 @@ describe("SplunkLogger", function() { var expected = { token: config.token, - name: "splunk-javascript-logging/0.8.0", + name: "splunk-javascript-logging/0.9.0", host: "localhost", path: "/services/collector/event/1.0", protocol: "https", @@ -938,7 +938,7 @@ describe("SplunkLogger", function() { var expected = { token: config.token, - name: "splunk-javascript-logging/0.8.0", + name: "splunk-javascript-logging/0.9.0", host: "localhost", path: "/services/collector/event/1.0", protocol: "https", diff --git a/utils.js b/utils.js index 83f82a4..7a18d7a 100644 --- a/utils.js +++ b/utils.js @@ -79,6 +79,7 @@ utils.toArray = function(iterable) { return Array.prototype.slice.call(iterable); }; +// TODO: this isn't used anymore, remove it /** * Run async function in a chain, like {@link https://github.com/caolan/async#waterfall|Async.waterfall}. * From 700a932f05e901d53915da5ced17d2cbb588c299 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Mon, 16 Nov 2015 21:26:23 -0800 Subject: [PATCH 31/46] Update all batching token placeholder --- examples/all_batching.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/all_batching.js b/examples/all_batching.js index 14865aa..75ca50a 100644 --- a/examples/all_batching.js +++ b/examples/all_batching.js @@ -31,7 +31,7 @@ var SplunkLogger = require("../index").Logger; * more than 1kb. */ var config = { - token: "your-token", + token: "your-token-here", url: "https://localhost:8088", batchInterval: 1000, maxBatchCount: 10, From 5052a73a681acc79e8a0fa865f37f08206d160e2 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Mon, 16 Nov 2015 22:46:24 -0800 Subject: [PATCH 32/46] Fix grammar in changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 933af21..f1841b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ ### New Features & APIs -* Added to configure automated batching with 3 settings: `batchInterval`, `maxBatchCount`, & `maxBatchSize`. +* Added the ability to configure automated batching with 3 settings: `batchInterval`, `maxBatchCount`, & `maxBatchSize`. * Added the ability to retry sending to Splunk in the case of network errors with the `maxRetries` configuration setting. * Added the ability to configure a custom Splunk event format by overriding `eventFormatter(message, severity)`. From 5fbeba8269761d89d333392aa4b09571be536115 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Tue, 17 Nov 2015 21:26:02 -0800 Subject: [PATCH 33/46] Correct config settings in custom_format example --- examples/custom_format.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/examples/custom_format.js b/examples/custom_format.js index 48266d7..89875bb 100644 --- a/examples/custom_format.js +++ b/examples/custom_format.js @@ -26,8 +26,7 @@ var SplunkLogger = require("../index").Logger; */ var config = { token: "your-token-here", - url: "https://localhost:8088", - maxBatchCount: 1 // Send events 1 at a time + url: "https://localhost:8088" }; // Create a new logger From f49155a1637e6bb3a8cd5e8f3b5d595f331de654 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Tue, 17 Nov 2015 21:38:28 -0800 Subject: [PATCH 34/46] alphabetize examples in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fd5145c..5ce3a46 100644 --- a/README.md +++ b/README.md @@ -19,8 +19,8 @@ If you already have Node.js and npm installed, simply run: `npm install --save s See the `examples` folder for more examples: * `all_batching.js`: shows how to configure a logger with the 3 batching settings: `batchInterval`, `maxBatchCount`, & `maxBatchSize`. -* `custom_format.js`: shows how to configure a logger so log message to Splunk using a custom format. * `basic.js`: shows how to configure a logger and send a log message to Splunk. +* `custom_format.js`: shows how to configure a logger so log message to Splunk using a custom format. * `manual_batching.js`: shows how to queue log messages, and send them in batches by manually calling `flush()`. * `retry.js`: shows how to configure retries on errors. From ba786a82ae73b4ad9db18c57215d3f39cf05e8e8 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Tue, 17 Nov 2015 21:41:46 -0800 Subject: [PATCH 35/46] Fix typos in readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5ce3a46..e1a6f74 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ See the `examples` folder for more examples: * `all_batching.js`: shows how to configure a logger with the 3 batching settings: `batchInterval`, `maxBatchCount`, & `maxBatchSize`. * `basic.js`: shows how to configure a logger and send a log message to Splunk. -* `custom_format.js`: shows how to configure a logger so log message to Splunk using a custom format. +* `custom_format.js`: shows how to configure a logger to log messages to Splunk using a custom format. * `manual_batching.js`: shows how to queue log messages, and send them in batches by manually calling `flush()`. * `retry.js`: shows how to configure retries on errors. From e566ac1b4a6248738b08cbec252e4dc2a76aec76 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Wed, 18 Nov 2015 13:09:57 -0800 Subject: [PATCH 36/46] Remove "beta" phrasing --- CHANGELOG.md | 4 ++-- README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1841b8..3578651 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ -# Splunk Logging Library for JavaScript (beta) +# Splunk Logging Library for JavaScript -## v0.9.0 - beta +## v0.9.0 ### New Features & APIs diff --git a/README.md b/README.md index e1a6f74..2459ddf 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Splunk Logging Library for JavaScript (beta) +# Splunk Logging Library for JavaScript #### Version 0.9.0 From bacfb4491059d53c8fcdb8c08847ac99b8052160 Mon Sep 17 00:00:00 2001 From: Matt Tevenan Date: Wed, 18 Nov 2015 15:39:03 -0800 Subject: [PATCH 37/46] Editorial changes. --- README.md | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 2459ddf..2368882 100644 --- a/README.md +++ b/README.md @@ -2,27 +2,33 @@ #### Version 0.9.0 -This project provides a simple interface for logging to Splunk's Event Collector. +This project provides a simple interface for logging to HTTP Event Collector in Splunk Enterprise and Splunk Cloud. ## Requirements -* Splunk 6.3+. -* An HTTP Event Collector token from your Splunk server. -* Node.js v0.10+. +* Node.js v0.10 or later. +* Splunk Enterprise 6.3.0 or later, or Splunk Cloud. +* An HTTP Event Collector token from your Splunk Enterprise server. ## Installation -If you already have Node.js and npm installed, simply run: `npm install --save splunk-logging`. +First, update npm to the latest version by running: + + sudo npm install npm -g + +Then run: + + npm install --save splunk-logging ## Usage -See the `examples` folder for more examples: +See the `examples` folder for usage examples: -* `all_batching.js`: shows how to configure a logger with the 3 batching settings: `batchInterval`, `maxBatchCount`, & `maxBatchSize`. -* `basic.js`: shows how to configure a logger and send a log message to Splunk. -* `custom_format.js`: shows how to configure a logger to log messages to Splunk using a custom format. -* `manual_batching.js`: shows how to queue log messages, and send them in batches by manually calling `flush()`. -* `retry.js`: shows how to configure retries on errors. +* `all_batching.js`: Shows how to configure a logger with the 3 batching settings: `batchInterval`, `maxBatchCount`, & `maxBatchSize`. +* `basic.js`: Shows how to configure a logger and send a log message to Splunk. +* `custom_format.js`: Shows how to configure a logger to log messages to Splunk using a custom format. +* `manual_batching.js`: Shows how to queue log messages, and send them in batches by manually calling `flush()`. +* `retry.js`: Shows how to configure retries on errors. ### Basic example @@ -37,7 +43,7 @@ var config = { var Logger = new SplunkLogger(config); var payload = { - // Message can be anything, doesn't have to be an object + // Message can be anything; doesn't have to be an object message: { temperature: "70F", chickenCount: 500 @@ -53,7 +59,7 @@ Logger.send(payload, function(err, resp, body) { ## Community -Stay connected with other developers building on Splunk. +Stay connected with other developers building on Splunk software. @@ -64,7 +70,7 @@ Stay connected with other developers building on Splunk. + @@ -86,7 +92,7 @@ Stay connected with other developers building on Splunk. ### Contact us -You can reach the Developer Platform team at _devinfo@splunk.com_. +You can reach the developer platform team at _devinfo@splunk.com_. ## License From 5f02cb38a9d44ff02a93b6be4dc57da355d6ca16 Mon Sep 17 00:00:00 2001 From: Matt Tevenan Date: Wed, 18 Nov 2015 15:40:04 -0800 Subject: [PATCH 38/46] Tweaks --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2368882..6f55e8f 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ -# Splunk Logging Library for JavaScript +# Splunk logging for JavaScript #### Version 0.9.0 -This project provides a simple interface for logging to HTTP Event Collector in Splunk Enterprise and Splunk Cloud. +This project provides a simple JavaScript interface for logging to HTTP Event Collector in Splunk Enterprise and Splunk Cloud. ## Requirements From 1d1ff482a314ae166c9f7c36181173c9c2cc6a5b Mon Sep 17 00:00:00 2001 From: Matt Tevenan Date: Wed, 18 Nov 2015 15:43:35 -0800 Subject: [PATCH 39/46] Editorial updates. --- CHANGELOG.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3578651..cd6a639 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,18 +1,18 @@ -# Splunk Logging Library for JavaScript +# Splunk logging for JavaScript ## v0.9.0 -### New Features & APIs +### New features & APIs * Added the ability to configure automated batching with 3 settings: `batchInterval`, `maxBatchCount`, & `maxBatchSize`. -* Added the ability to retry sending to Splunk in the case of network errors with the `maxRetries` configuration setting. -* Added the ability to configure a custom Splunk event format by overriding `eventFormatter(message, severity)`. +* Added the ability to retry sending to Splunk Enterprise or Splunk Cloud in the case of network errors with the `maxRetries` configuration setting. +* Added the ability to configure a custom Splunk Enterprise or Splunk Cloud event format by overriding `eventFormatter(message, severity)`. ### Breaking Changes * Removed the `autoFlush` configuration setting. To achieve the same effect, set `config.maxBatchCount` to `0`. * Removed support for middleware functions. -* The `context` object has been simplified, `config` and `requestOptions` can no longer be specified there - please use those settings directly on the logger. +* The `context` object has been simplified, `config` and `requestOptions` can no longer be specified there; please use those settings directly on the logger. ### Examples @@ -26,4 +26,4 @@ ## v0.8.0 - beta -* Beta release. \ No newline at end of file +* Beta release. From 65e6b8697126ab4ce288b41ed1540eb52e46f678 Mon Sep 17 00:00:00 2001 From: Matt Tevenan Date: Wed, 18 Nov 2015 15:58:04 -0800 Subject: [PATCH 40/46] Editorial updates. --- splunklogger.js | 52 ++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/splunklogger.js b/splunklogger.js index 2794aae..03700ac 100644 --- a/splunklogger.js +++ b/splunklogger.js @@ -33,9 +33,9 @@ function _err(err, context) { } /** - * The default format for Splunk events. + * The default format for Splunk Enterprise or Splunk Cloud events. * - * This function can be overwritten, and can return any type (string, object, array, etc.) + * This function can be overwritten, and can return any type (string, object, array, and so on). * * @param {anything} [message] - The event message. * @param {string} [severity] - The event severity. @@ -50,8 +50,9 @@ function _defaultEventFormatter(message, severity) { } /** - * Constructs a SplunkLogger, to send events to Splunk via the HTTP Event Collector. - * See defaultConfig for default configuration settings. + * Constructs a SplunkLogger, to send events to Splunk Enterprise or Splunk Cloud + * via HTTP Event Collector. See defaultConfig for default + * configuration settings. * * @example * var SplunkLogger = require("splunk-logging").Logger; @@ -67,20 +68,20 @@ function _defaultEventFormatter(message, severity) { * @property {object} config - Configuration settings for this SplunkLogger instance. * @param {object} requestOptions - Options to pass to {@link https://github.com/request/request#requestpost|request.post()}. * See the {@link http://github.com/request/request|request documentation} for all available options. - * @property {object[]} serializedContextQueue - Queue of serialized context objects to be sent to Splunk. + * @property {object[]} serializedContextQueue - Queue of serialized context objects to be sent to Splunk Enterprise or Splunk Cloud. * @property {function} eventFormatter - Formats events, returning an event as a string, function(message, severity). * Can be overwritten, the default event formatter will display event and severity as properties in a JSON object. * @property {function} error - A callback function for errors: function(err, context). * Defaults to console.log both values; * * @param {object} config - Configuration settings for a new [SplunkLogger]{@link SplunkLogger}. - * @param {string} config.token - Splunk HTTP Event Collector token, required. + * @param {string} config.token - HTTP Event Collector token, required. * @param {string} [config.name=splunk-javascript-logging/0.9.0] - Name for this logger. - * @param {string} [config.host=localhost] - Hostname or IP address of Splunk server. - * @param {string} [config.maxRetries=0] - How many times to retry when HTTP POST to Splunk fails. - * @param {string} [config.path=/services/collector/event/1.0] - URL path to send data to on the Splunk server. - * @param {string} [config.protocol=https] - Protocol used to communicate with the Splunk server, http or https. - * @param {number} [config.port=8088] - HTTP Event Collector port on the Splunk server. + * @param {string} [config.host=localhost] - Hostname or IP address of Splunk Enterprise or Splunk Cloud server. + * @param {string} [config.maxRetries=0] - How many times to retry when HTTP POST to Splunk Enterprise or Splunk Cloud fails. + * @param {string} [config.path=/services/collector/event/1.0] - URL path to send data to on the Splunk Enterprise or Splunk Cloud server. + * @param {string} [config.protocol=https] - Protocol used to communicate with the Splunk Enterprise or Splunk Cloud server, http or https. + * @param {number} [config.port=8088] - HTTP Event Collector port on the Splunk Enterprise or Splunk Cloud server. * @param {string} [config.url] - URL string to pass to {@link https://nodejs.org/api/url.html#url_url_parsing|url.parse}. This will try to set * host, path, protocol, port, url. Any of these values will be overwritten if * the corresponding property is set on config. @@ -147,7 +148,7 @@ var defaultConfig = { }; var defaultRequestOptions = { - json: true, // Sets the content-type header to application/json + json: true, // Sets the content-type header to application/json. strictSSL: false }; @@ -275,7 +276,7 @@ SplunkLogger.prototype._initializeConfig = function(config) { // Has the interval timer already started, and the interval changed to a different duration? var changeTimer = this._timerID && this._timerDuration !== ret.batchInterval && ret.batchInterval > 0; - // Upsert the timer + // Enable the timer if (startTimer || changeTimer) { this._enableTimer(ret.batchInterval); } @@ -323,7 +324,7 @@ SplunkLogger.prototype._validateMessage = function(message) { }; /** - * Initialized metadata, if context.metadata is falsey or empty, + * Initializes metadata. If context.metadata is false or empty, * return an empty object. * * @param {object} context @@ -413,7 +414,7 @@ SplunkLogger.prototype._post = function(requestOptions, callback) { }; /** - * Sends events to Splunk, optionally with retries on non-Splunk errors. + * Sends events to Splunk Enterprise or Splunk Cloud, optionally with retries on non-Splunk errors. * * @param context * @param {function} callback - A callback function: function(err, response, body) @@ -434,13 +435,12 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { requestOptions.headers["Content-Type"] = "application/x-www-form-urlencoded"; requestOptions.url = this.config.protocol + "://" + this.config.host + ":" + this.config.port + this.config.path; - // Initialize the context again, right before using it context = this._initializeContext(context); var that = this; - var splunkError = null; // Errors returned by Splunk + var splunkError = null; // Errors returned by Splunk Enterprise or Splunk Cloud var requestError = null; // Any non-Splunk errors // References so we don't have to deal with callback parameters @@ -462,7 +462,7 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { _response = resp; _body = body; - // Try to parse an error response from Splunk + // Try to parse an error response from Splunk Enterprise or Splunk Cloud if (!requestError && body && body.code.toString() !== "0") { splunkError = new Error(body.text); splunkError.code = body.code; @@ -501,7 +501,7 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { * * var logger = new SplunkLogger(config); * - * // Payload to send to Splunk's Event Collector + * // Payload to send to HTTP Event Collector. * var payload = { * message: { * temperature: "70F", @@ -530,11 +530,11 @@ SplunkLogger.prototype._sendEvents = function(context, callback) { * @param {(object|string|Array|number|bool)} context.message - Data to send to Splunk. * @param {string} [context.severity=info] - Severity level of this event. * @param {object} [context.metadata] - Metadata for this event. - * @param {string} [context.metadata.host] - If not specified, Splunk will decide the value. - * @param {string} [context.metadata.index] - The Splunk index to send data to. - * If not specified, Splunk will decide the value. - * @param {string} [context.metadata.source] - If not specified, Splunk will decide the value. - * @param {string} [context.metadata.sourcetype] - If not specified, Splunk will decide the value. + * @param {string} [context.metadata.host] - If not specified, Splunk Enterprise or Splunk Cloud will decide the value. + * @param {string} [context.metadata.index] - The Splunk Enterprise or Splunk Cloud index to send data to. + * If not specified, Splunk Enterprise or Splunk Cloud will decide the value. + * @param {string} [context.metadata.source] - If not specified, Splunk Enterprise or Splunk Cloud will decide the value. + * @param {string} [context.metadata.sourcetype] - If not specified, Splunk Enterprise or Splunk Cloud will decide the value. * @param {function} [callback] - A callback function: function(err, response, body). * @throws Will throw an error if the context parameter is malformed. * @public @@ -557,7 +557,7 @@ SplunkLogger.prototype.send = function(context, callback) { }; /** - * Manually send all events in this.serializedContextQueue to Splunk. + * Manually send all events in this.serializedContextQueue to Splunk Enterprise or Splunk Cloud. * * @param {function} [callback] - A callback function: function(err, response, body). * @public @@ -579,4 +579,4 @@ SplunkLogger.prototype.flush = function(callback) { this._sendEvents(context, callback); }; -module.exports = SplunkLogger; \ No newline at end of file +module.exports = SplunkLogger; From 94eab46d82795dba4a3bcce655edad60b6640e5d Mon Sep 17 00:00:00 2001 From: Matt Tevenan Date: Wed, 18 Nov 2015 15:59:43 -0800 Subject: [PATCH 41/46] Editorial updates. --- utils.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/utils.js b/utils.js index 7a18d7a..1ca3ff6 100644 --- a/utils.js +++ b/utils.js @@ -23,7 +23,7 @@ var utils = {}; /* Utility Functions */ /** - * Formats the time for Splunk as a the epoch time in seconds. + * Formats the time for Splunk Enterprise or Splunk Cloud as a the epoch time in seconds. * * @param {(string|number|date)} time - A date string, timestamp, or Date object. * @returns {number|null} Epoch time in seconds, or null if time is malformed. @@ -226,7 +226,7 @@ utils.copyObject = function(obj) { }; /** - * Copies all elements into a new array which is returned. + * Copies all elements into a new array, which is returned. * * @param {array} [arr] - Array to copy elements from. * @returns {array} @@ -261,10 +261,10 @@ utils.orByProp = function(prop) { }; /** - * Like utils.orByProp() but for a falsey property. + * Like utils.orByProp() but for a false property. * The first argument after prop with that property * defined will be returned. - * Useful for booleans and numbers. + * Useful for Booleans and numbers. * * @param {string} [prop] - property name for other arguments. * @returns {boolean} @@ -303,4 +303,4 @@ utils.validateNonNegativeInt = function(value, label) { return value; }; -module.exports = utils; \ No newline at end of file +module.exports = utils; From 93853fd0b89b7b4d9abcd17e2ade69ac7605fe16 Mon Sep 17 00:00:00 2001 From: Matt Tevenan Date: Wed, 18 Nov 2015 16:01:30 -0800 Subject: [PATCH 42/46] Editorial updates. --- examples/basic.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/basic.js b/examples/basic.js index ca82cd0..c1a3d76 100644 --- a/examples/basic.js +++ b/examples/basic.js @@ -37,7 +37,7 @@ Logger.error = function(err, context) { console.log("error", err, "context", context); }; -// Define the payload to send to Splunk's Event Collector +// Define the payload to send to HTTP Event Collector var payload = { // Message can be anything, it doesn't have to be an object message: { @@ -85,4 +85,4 @@ console.log("Sending payload", payload); Logger.send(payload, function(err, resp, body) { // If successful, body will be { text: 'Success', code: 0 } console.log("Response from Splunk", body); -}); \ No newline at end of file +}); From 2411218b2f3f6b85f2e845506f4ccd1ad11f94d2 Mon Sep 17 00:00:00 2001 From: Matt Tevenan Date: Wed, 18 Nov 2015 16:03:00 -0800 Subject: [PATCH 43/46] Editorial updates. --- examples/custom_format.js | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/examples/custom_format.js b/examples/custom_format.js index 89875bb..57e18c3 100644 --- a/examples/custom_format.js +++ b/examples/custom_format.js @@ -40,7 +40,7 @@ Logger.error = function(err, context) { /** * Override the default eventFormatter() function, * which takes a message and severity, returning - * any type - string or object are recommended. + * any type; string or object are recommended. * * The message parameter can be any type. It will * be whatever was passed to Logger.send(). @@ -50,6 +50,7 @@ Logger.error = function(err, context) { * of key=value pairs if message is an object, * otherwise the message value is as value for * the message key. + * * This string is prefixed with the event * severity in square brackets. */ @@ -68,7 +69,7 @@ Logger.eventFormatter = function(message, severity) { return event; }; -// Define the payload to send to Splunk's Event Collector +// Define the payload to send to HTTP Event Collector var payload = { // Message can be anything, it doesn't have to be an object message: { @@ -110,4 +111,4 @@ console.log("Sending payload", payload); Logger.send(payload, function(err, resp, body) { // If successful, body will be { text: 'Success', code: 0 } console.log("Response from Splunk", body); -}); \ No newline at end of file +}); From 812e0b2a554c687b9b8a4fd5993cb0301def0c35 Mon Sep 17 00:00:00 2001 From: Matt Tevenan Date: Wed, 18 Nov 2015 16:04:18 -0800 Subject: [PATCH 44/46] Editorial updates. --- examples/retry.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/retry.js b/examples/retry.js index fd3a2f1..7cffee7 100644 --- a/examples/retry.js +++ b/examples/retry.js @@ -44,7 +44,7 @@ Logger.error = function(err, context) { console.log("error", err, "context", context); }; -// Define the payload to send to Splunk's Event Collector +// Define the payload to send to HTTP Event Collector var payload = { // Message can be anything, doesn't have to be an object message: { @@ -66,4 +66,4 @@ console.log("Sending payload", payload); Logger.send(payload, function(err, resp, body) { // If successful, body will be { text: 'Success', code: 0 } console.log("Response from Splunk", body); -}); \ No newline at end of file +}); From a1cf6d63d808d9866b3a352bae838697bf5d94c7 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Wed, 18 Nov 2015 18:52:09 -0800 Subject: [PATCH 45/46] Fix typo in utils doc comment --- utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils.js b/utils.js index 1ca3ff6..f6b78ed 100644 --- a/utils.js +++ b/utils.js @@ -261,7 +261,7 @@ utils.orByProp = function(prop) { }; /** - * Like utils.orByProp() but for a false property. + * Like utils.orByProp() but for a falsey property. * The first argument after prop with that property * defined will be returned. * Useful for Booleans and numbers. From 0d3ea9f1fa630ce985c01eff6dd700c1e0d7bc08 Mon Sep 17 00:00:00 2001 From: Shakeel Mohamed Date: Wed, 9 Dec 2015 14:08:25 -0800 Subject: [PATCH 46/46] Change retry example to retry 10 times --- examples/retry.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/retry.js b/examples/retry.js index 7cffee7..b0bb26b 100644 --- a/examples/retry.js +++ b/examples/retry.js @@ -24,16 +24,16 @@ var SplunkLogger = require("../index").Logger; /** * Only the token property is required. * - * Here we've set maxRetries to 5, + * Here we've set maxRetries to 10, * If there are any connection errors the request to Splunk will - * be retried up to 5 times. + * be retried up to 10 times. * The default is 0. */ var config = { token: "your-token-here", url: "https://localhost:8088", level: "info", - maxRetries: 5 + maxRetries: 10 }; // Create a new logger
Issues -https://github.com/splunk/splunk-logging-javascript/issues/https://github.com/splunk/splunk-javascript-logging/issues/