Merge pull request #2 from ibm-cds-labs/i1

i1

Author: ptitzler

index.js

@@ -24,7 +24,7 @@ const path = require('path');
 const SimpleDataVis = require('simple-data-vis');
 
 // to enable debugging, set environment variable DEBUG to nps or *
-const debug = require('debug')('npmwd');
+const debug = require('debug')('npmjs');
 
 const init = require('./lib/initialize.js');
 
@@ -60,7 +60,7 @@ const init = require('./lib/initialize.js');
 	console.log('Service is initializing...');
 
 	// initialize Cloudant repository and load defaults
-	init(appEnv, function(err, dataRepository, metaRepository, collector) {
+	init(appEnv, function(err, dataRepository, metaRepository, collector, indexer) {
 
 		//
 		// dataRepository - cloudant handle for data database
@@ -71,23 +71,46 @@ const init = require('./lib/initialize.js');
 			process.exit(1);
 		}
 
-/*		var year = 2015;
-		//var month = null;
+		// asynchronously build index and collect missing statistics
+		debug('Building index ...');
+		indexer.buildIndex(function(err) {
+			if(err) {
+				console.error('Index build error: ' + err);
+			}
+			else {
+				debug('Index was built. Identifying gaps ...');
+				indexer.inspectIndex(null, 
+									 function(err, todolist) {
+									 	if(err) {
+											console.error('Index inspection error: ' + err);						 		
+									 	}
+									 	else {
+											debug('Collecting missing statistics ... ');
+											collector.collect(todolist,
+															  function(err, data) {
+															  		if(err) {
+															  			console.error('Statistics collector error: ' + err);
+															  		}
+															  		else {
+															  			debug('Statistics collection results: ' + JSON.stringify(data));
+															  		}
+															  });
+									 	}
+									 });
+			}
+		});
 
-		collector.collectYear(['ibm-graph-client', 'couchimport', 'cf-deployment-tracker-client', 'cachemachine', 'cloudant-envoy', 'nosqlimport','nosqlimport-couchdb','nosqlimport-mongodb', 'nosqlelasticsearch'], 
-						  	  year, 
-						  //month, 
-						  function (err) {
-						  	if(err) {
-						  		console.log("Collection error: " + err);
-						  	}
-						  	else {
-							  	console.log("Collection done");
-						  	}
 
+/* TODO remove
+		// seed statistics
+		collector.collectMonth('cloudant', 
+							   2015, 
+							   12, 
+							   function(err) {
+							   		console.log(err);
 		});
-
 */
+
 		var app = express();
 		app.use(bodyParser.urlencoded({extended: false}));
   		// use https://www.npmjs.com/package/express-handlebars as view engine
@@ -124,7 +147,7 @@ const init = require('./lib/initialize.js');
 			// render stats page
 			collector.getPackages(function(err, packages) {
 				if(err) {
-					console.log("/stats: error retrieving package list: " + err);
+					console.log('/stats: error retrieving package list: ' + err);
 					res.render('stats');
 				}
 				res.render('stats', {packages:packages});	

lib/collector.js

@@ -42,7 +42,7 @@ function NPMJSCollector(repository) {
 NPMJSCollector.prototype.getPackages = function(callback) {
 
 	if(! this.dataRepository) {
-		return callback("No data repository is defined.");
+		return callback('No data repository is defined.');
 	}
 
 	var options = {
@@ -85,31 +85,35 @@ NPMJSCollector.prototype.collectYear = function(packages, year, callback) {
 NPMJSCollector.prototype.collectMonth = function(packages, year, month, callback) {
 
 	if(! this.dataRepository) {
-		return callback("No data repository is defined.");
+		return callback('No data repository is defined.');
 	}
 
 	if(! packages) {
 		return callback(); // nothing to do
 	}
 
+	// TODO
+	// check year/month parms
+
+
 	if(! Array.isArray(packages)) {
 		packages = [packages];
 	}
 
 	var dateRangeStart = new Date(year, month - 1,1).toISOString().substring(0,10);
 	var dateRangeEnd =	new Date(year, month, 0).toISOString().substring(0,10);
 
-	debug("Collecting download statistics for " + packages.toString() + " for date range " + dateRangeStart + " and " + dateRangeEnd);
+	debug('Collecting download statistics for ' + packages.toString() + ' for date range ' + dateRangeStart + ' and ' + dateRangeEnd);
 
 	// submit npmjs request
 	// sample: https://api.npmjs.org/downloads/range/2016-01-01:2016-02-01/cloudant
-	this.npmjsClient.getAll({ dateRange: dateRangeStart + ":" + dateRangeEnd , packages: packages},
+	this.npmjsClient.getAll({ dateRange: dateRangeStart + ':' + dateRangeEnd , packages: packages},
 						    function(err, data) {
 						    	if(err) {
 						    		return callback(err);
 						    	}
 
-						    	if(data.hasOwnProperty("error")) {
+						    	if(data.hasOwnProperty('error')) {
 						    		if(data.error === 'no stats for this package for this range (0008)') {
 						    			return callback(); // nothing to do; don't return an error
 						    		}
@@ -118,22 +122,22 @@ NPMJSCollector.prototype.collectMonth = function(packages, year, month, callback
 						    		}
 						    	}
 
-						    	debug("Err: " + JSON.stringify(err));
-						    	debug("Data: " + JSON.stringify(data));
+						    	debug('Err: ' + JSON.stringify(err));
+						    	debug('Data: ' + JSON.stringify(data));
 
 						    	// iterate through packages and create a stats doc for each
 						    	var statsDocs = [];
 						    	var root = null;
 						    	var total = 0;
 						    	_.forEach(packages, 
 						    		      function(packageName) {
-						    		      	debug("Checking for download information for package " + packageName);
+						    		      	debug('Checking for download information for package ' + packageName);
 						    		      	root = null;
 						    		      	if(data.hasOwnProperty(packageName)) {
 						    		      		root = data[packageName];	
 						    		      	}
 						    		      	else {
-						    		      		if(data.hasOwnProperty("downloads")) {
+						    		      		if(data.hasOwnProperty('downloads')) {
 						    		      			root = data;
 						    		      		}
 						    		      	}	
@@ -145,10 +149,10 @@ NPMJSCollector.prototype.collectMonth = function(packages, year, month, callback
 							    		      	});
 
 												statsDocs.push({
-							    								_id: root["package"] + "_" + root.start.substring(0, 7),
-							    								type: "stats",
-							    								package: root["package"],
-							    								language: "javascript",
+							    								_id: root['package'] + '_' + root.start.substring(0, 7),
+							    								type: 'stats',
+							    								package: root['package'],
+							    								language: 'javascript',
 							    								month: root.start.substring(0, 7),
 							    								total: total,
 							    								range: {
@@ -174,4 +178,39 @@ NPMJSCollector.prototype.collectMonth = function(packages, year, month, callback
 						    }.bind(this));
 };
 
+/**
+ * Collects the statistics for todolist
+ * @param todolist
+ * @param callback invoked when processing is finished; returns (err, data)
+ */
+NPMJSCollector.prototype.collect = function(todolist, callback) { 
+	if(! todolist) {
+		// nothing to do
+		return callback();
+	}
+	// process TODO list
+	async.eachLimit(Object.keys(todolist),
+					1,
+				    function(year, asyncCallbackYear) {
+				  		async.eachLimit(
+					  					Object.keys(todolist[year]),
+					  					1,
+								  		function(month, asyncCallbackMonth) {
+											this.collectMonth(todolist[year][month], 
+															  year, 
+															  month,
+										            	      function(err, data) {
+										            	      	debug(data);
+										    					return asyncCallbackMonth(err);
+										                   	});					  		
+								  		}.bind(this),
+								  		function(err) {
+								  			return asyncCallbackYear(err);
+								  		});
+				  	}.bind(this),
+				  	function(err) {
+						return callback(err);
+				  	});
+};
+
 module.exports = NPMJSCollector;