From 0270f4a8ab5ecb577d6ef8d8ec3bb7be4affce88 Mon Sep 17 00:00:00 2001 From: Sophat Sam Date: Tue, 27 Jun 2017 12:40:03 -0400 Subject: [PATCH 1/6] switch to request instead of https.request so we can use `/batch` endpoint in preparation for requesting all projects via batch --- controllers/projects/controller.js | 49 ++++++++++++++++-------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/controllers/projects/controller.js b/controllers/projects/controller.js index 5c20cb7..600c05b 100644 --- a/controllers/projects/controller.js +++ b/controllers/projects/controller.js @@ -1,9 +1,9 @@ const models = require('../../models'); -const https = require('https'); const PG_config = require('../../config/config'); const moment = require('moment'); const tz = require('moment-timezone'); const Logger = require('../../config/logger'); +const request = require('request'); // save to database function createProject(project) { @@ -27,40 +27,43 @@ function createProject(project) { } // iterator over list -const iterateProjectlist = (projects) => { - let data = projects.data; - data.forEach(project => createProject(project)); +const iterateProjectlist = (projects, firstRequest) => { + projects.data.forEach(response => { + if (response.status_code === 200){ + var body = JSON.parse(response.body); + body.data.forEach(project => createProject(project)); + } + }) + } // get projects -const getProjectsFromPlanGrid = () => { +const getProjectsFromPlanGrid = (requests) => { return new Promise((resolve, reject) => { + + var batchRequest = requests || [{"method": "GET", "path": "/projects"}] let options = { - hostname: PG_config.plangrid.url, - path: '/projects', - auth: PG_config.plangrid.key, - method: 'GET', - headers: PG_config.plangrid.headers + url: 'https://' + PG_config.plangrid.url + '/batch', + auth: { 'user' : PG_config.plangrid.key }, + method: 'POST', + headers: PG_config.plangrid.headers, + json: batchRequest }; - let req = https.request(options, (res) => { - let data = ''; - - res.setEncoding('utf8'); - res.on('data', (chunk) => { data += chunk }); - res.on('end', () => { + let data = ''; + + request(options) + .on('data', (chunk) => { data += chunk }) + .on('end', (res) => { let jsonProjects = JSON.parse(data); resolve(jsonProjects); + }) + .on('error', (e) => { + Logger.error(`problem with request: ${e.message}`); + reject(e); }); - }); - - req.on('error', (e) => { - Logger.error(`problem with request: ${e.message}`); - reject(e); - }); - req.end(); }) } From 7764122ae788d44ffd76fad268a5219cb17ae58a Mon Sep 17 00:00:00 2001 From: Sophat Sam Date: Tue, 27 Jun 2017 16:16:52 -0400 Subject: [PATCH 2/6] use `/batch` endpoint to request all projects --- controllers/projects/controller.js | 45 +++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/controllers/projects/controller.js b/controllers/projects/controller.js index 600c05b..7281c24 100644 --- a/controllers/projects/controller.js +++ b/controllers/projects/controller.js @@ -27,53 +27,72 @@ function createProject(project) { } // iterator over list -const iterateProjectlist = (projects, firstRequest) => { - projects.data.forEach(response => { +const iterateProjectlist = (result) => { + + result.projects.data.forEach(response => { if (response.status_code === 200){ - var body = JSON.parse(response.body); + let body = JSON.parse(response.body); body.data.forEach(project => createProject(project)); } }) + // make batch call if total count > number count + let first = result.projects.data[0]; + let totalCount = JSON.parse(first.body).total_count; + if (result.count < totalCount){ + let batch = []; + let calls = Math.ceil(totalCount / 50); + + for(let count = result.count / 50; count < calls; count++){ + batch.push({ + "method": "GET", + "path": "/projects?skip=" + (count * 50) + }) + } + + setTimeout(function(){ + getProjects(batch, calls * 50); + }, 1000) + } + } // get projects -const getProjectsFromPlanGrid = (requests) => { +const getProjectsFromPlanGrid = (requests, count) => { return new Promise((resolve, reject) => { - - var batchRequest = requests || [{"method": "GET", "path": "/projects"}] + let batchRequest = requests || [{"method": "GET", "path": "/projects"}] let options = { url: 'https://' + PG_config.plangrid.url + '/batch', auth: { 'user' : PG_config.plangrid.key }, method: 'POST', headers: PG_config.plangrid.headers, json: batchRequest - }; + } let data = ''; - request(options) .on('data', (chunk) => { data += chunk }) .on('end', (res) => { let jsonProjects = JSON.parse(data); - resolve(jsonProjects); + resolve({"projects": jsonProjects, "count": count || 50}); }) .on('error', (e) => { Logger.error(`problem with request: ${e.message}`); reject(e); - }); + }) }) } // Main function to poll the API and create projects -exports.getProjects = () => { - getProjectsFromPlanGrid() - .then(projects => iterateProjectlist(projects)) +const getProjects = (requests, count) => { + getProjectsFromPlanGrid(requests, count) + .then(result => iterateProjectlist(result)) .then(result => Logger.info(result)) .catch(err => Logger.error(err)) } +exports.getProjects = getProjects; // Create a single Project exports.createProject = createProject; From da0f86321f8b7188805142d7e2cc642ee05494ef Mon Sep 17 00:00:00 2001 From: Sophat Sam Date: Wed, 28 Jun 2017 12:13:05 -0400 Subject: [PATCH 3/6] add request to npm package file --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 7dd2a96..4f9c8cf 100644 --- a/package.json +++ b/package.json @@ -16,6 +16,7 @@ "moment-timezone": "^0.5.5", "node-schedule": "^1.2.0", "pg": "^6.1.0", + "request": "^2.81.0", "sequelize": "^3.24.3", "tedious": "^1.14.0", "winston": "^2.2.0", From 2ac4b96048c5127b52f03ffe6b83c791f4e57b36 Mon Sep 17 00:00:00 2001 From: Sophat Sam Date: Mon, 10 Jul 2017 12:40:20 -0400 Subject: [PATCH 4/6] add new Project fields to model --- models/project.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/models/project.js b/models/project.js index a2d3bed..1a6fbf3 100644 --- a/models/project.js +++ b/models/project.js @@ -15,6 +15,18 @@ module.exports = function(sequelize, DataTypes) { project_id: { type: DataTypes.STRING }, + custom_id: DataTypes.STRING, + type: DataTypes.STRING, + status: DataTypes.STRING, + owner: DataTypes.STRING, + start_date: DataTypes.DATE, + end_date: DataTypes.DATE, + street_1: DataTypes.STRING, + street_2: DataTypes.STRING, + city: DataTypes.STRING, + region: DataTypes.STRING, + postal_code: DataTypes.STRING, + country: DataTypes.STRING, created_at: DataTypes.DATE, updated_at: DataTypes.DATE, createdAt: DataTypes.DATE, From 54c36e9bd69771a04ace4910d78b263fad9570b4 Mon Sep 17 00:00:00 2001 From: Sophat Sam Date: Mon, 10 Jul 2017 12:59:10 -0400 Subject: [PATCH 5/6] update Sequelize migration to include new project fields --- migrations/20161003154622-project-migration.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/migrations/20161003154622-project-migration.js b/migrations/20161003154622-project-migration.js index 78fc042..02a2065 100644 --- a/migrations/20161003154622-project-migration.js +++ b/migrations/20161003154622-project-migration.js @@ -7,6 +7,18 @@ module.exports = { uid: Sequelize.STRING, project_name: Sequelize.STRING, project_id: Sequelize.STRING, + custom_id: Sequelize.STRING, + type: Sequelize.STRING, + status: Sequelize.STRING, + owner: Sequelize.STRING, + start_date: Sequelize.DATE, + end_date: Sequelize.DATE, + street_1: Sequelize.STRING, + street_2: Sequelize.STRING, + city: Sequelize.STRING, + region: Sequelize.STRING, + postal_code: Sequelize.STRING, + country: Sequelize.STRING, created_at: Sequelize.DATE, // date PG project was created updated_at: Sequelize.DATE, // date PG project was updated updatedAt: Sequelize.DATE, // date this record was updated From 9b4698c2a3846cb6624e18f4fdf6fefdbeaf9e00 Mon Sep 17 00:00:00 2001 From: Sophat Sam Date: Mon, 10 Jul 2017 14:45:24 -0400 Subject: [PATCH 6/6] update Project model to include new fields --- controllers/projects/controller.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/controllers/projects/controller.js b/controllers/projects/controller.js index 7281c24..db6dc4f 100644 --- a/controllers/projects/controller.js +++ b/controllers/projects/controller.js @@ -18,6 +18,18 @@ function createProject(project) { defaults: { project_name: project.name, project_id: pid, + custom_id: project.custom_id, + type: project.type, + status: project.status, + owner: project.owner, + start_date: project.start_date, + end_date: project.end_date, + street_1: project.street_1, + street_2: project.street_2, + city: project.city, + region: project.region, + postal_code: project.postal_code, + country: project.country, updatedAt: now }, })