Get all Projects

controllers/projects/controller.js

@@ -1,9 +1,9 @@
 const models  = require('../../models');
-const https = require('https');
 const PG_config = require('../../config/config');
 const moment = require('moment');
 const tz = require('moment-timezone');
 const Logger = require('../../config/logger');
+const request = require('request');
 
 // save to database
 function createProject(project) {
@@ -18,6 +18,18 @@ function createProject(project) {
 		defaults: { 
 			project_name: project.name,
 			project_id: pid,
+			custom_id: project.custom_id,
+			type: project.type,
+			status: project.status,
+			owner: project.owner,
+			start_date: project.start_date,
+			end_date: project.end_date,
+			street_1: project.street_1,
+			street_2: project.street_2,
+			city: project.city,
+			region: project.region,
+			postal_code: project.postal_code,
+			country: project.country,
 			updatedAt: now
 		},
 	})
@@ -27,50 +39,72 @@ function createProject(project) {
 }
 
 // iterator over list
-const iterateProjectlist = (projects) => {
-	let data = projects.data;
-	data.forEach(project => createProject(project));
+const iterateProjectlist = (result) => {
+
+	result.projects.data.forEach(response => {
+		if (response.status_code === 200){
+			let body = JSON.parse(response.body);
+			body.data.forEach(project => createProject(project));
+		}
+	})
+
+	// make batch call if total count > number count
+	let first = result.projects.data[0];
+	let totalCount = JSON.parse(first.body).total_count;
+	if (result.count < totalCount){
+		let batch = [];
+		let calls = Math.ceil(totalCount / 50);
+
+		for(let count = result.count / 50; count < calls; count++){
+			batch.push({
+				"method": "GET",
+				"path": "/projects?skip=" + (count * 50)
+			})
+		}
+
+		setTimeout(function(){
+			getProjects(batch, calls * 50);
+		}, 1000)
+	}
+
 }
 
 // get projects
-const getProjectsFromPlanGrid = () => {
+const getProjectsFromPlanGrid = (requests, count) => {
 
 	return new Promise((resolve, reject) => {
+		let batchRequest = requests || [{"method": "GET", "path": "/projects"}]
 		let options = {
-			hostname: PG_config.plangrid.url,
-			path: '/projects',
-			auth: PG_config.plangrid.key,
-			method: 'GET',
-			headers: PG_config.plangrid.headers
-		};
-
-		let req = https.request(options, (res) => {
-			let data = '';
-
-			res.setEncoding('utf8');
-			res.on('data', (chunk) => { data += chunk });
-			res.on('end', () => {
-				let jsonProjects = JSON.parse(data);
-				resolve(jsonProjects);
-			});
-		});
+			url: 'https://' + PG_config.plangrid.url + '/batch',
+			auth: { 'user' : PG_config.plangrid.key },
+			method: 'POST',
+			headers: PG_config.plangrid.headers,
+			json: batchRequest
+		}
 
-		req.on('error', (e) => {
-		  Logger.error(`problem with request: ${e.message}`);
-		  reject(e);
-		});
+		let data = '';
+		request(options)
+			.on('data', (chunk) => { data += chunk })
+			.on('end', (res) => {
+				let jsonProjects = JSON.parse(data);
+				resolve({"projects": jsonProjects, "count": count || 50});
+			})
+			.on('error', (e) => {
+			  Logger.error(`problem with request: ${e.message}`);
+			  reject(e);
+			})
 
-		req.end();
 	})
 }
 
 // Main function to poll the API and create projects
-exports.getProjects = () => {
-	getProjectsFromPlanGrid()
-	.then(projects => iterateProjectlist(projects))
+const getProjects = (requests, count) => {
+	getProjectsFromPlanGrid(requests, count)
+	.then(result => iterateProjectlist(result))
 	.then(result => Logger.info(result))
 	.catch(err => Logger.error(err))
 }
+exports.getProjects = getProjects;
 
 // Create a single Project
 exports.createProject = createProject;

migrations/20161003154622-project-migration.js

@@ -7,6 +7,18 @@ module.exports = {
       uid: Sequelize.STRING,
       project_name: Sequelize.STRING,
       project_id: Sequelize.STRING,
+      custom_id: Sequelize.STRING,
+      type: Sequelize.STRING,
+      status: Sequelize.STRING,
+      owner: Sequelize.STRING,
+      start_date: Sequelize.DATE,
+      end_date: Sequelize.DATE,
+      street_1: Sequelize.STRING,
+      street_2: Sequelize.STRING,
+      city: Sequelize.STRING,
+      region: Sequelize.STRING,
+      postal_code: Sequelize.STRING,
+      country: Sequelize.STRING,
       created_at: Sequelize.DATE, // date PG project was created
       updated_at: Sequelize.DATE, // date PG project was updated
       updatedAt: Sequelize.DATE,  // date this record was updated

models/project.js

@@ -15,6 +15,18 @@ module.exports = function(sequelize, DataTypes) {
     project_id: {
       type: DataTypes.STRING
     },
+    custom_id: DataTypes.STRING,
+    type: DataTypes.STRING,
+    status: DataTypes.STRING,
+    owner: DataTypes.STRING,
+    start_date: DataTypes.DATE,
+    end_date: DataTypes.DATE,
+    street_1: DataTypes.STRING,
+    street_2: DataTypes.STRING,
+    city: DataTypes.STRING,
+    region: DataTypes.STRING,
+    postal_code: DataTypes.STRING,
+    country: DataTypes.STRING,
     created_at: DataTypes.DATE,
     updated_at: DataTypes.DATE,
     createdAt: DataTypes.DATE,

package.json

@@ -16,6 +16,7 @@
     "moment-timezone": "^0.5.5",
     "node-schedule": "^1.2.0",
     "pg": "^6.1.0",
+    "request": "^2.81.0",
     "sequelize": "^3.24.3",
     "tedious": "^1.14.0",
     "winston": "^2.2.0",