Skip to content

fix: implemented last fetched time of content-types to reduce subsequent build times #116

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
vkalta wants to merge 2 commits into
base: develop-archived
Choose a base branch
from
Open

fix: implemented last fetched time of content-types to reduce subsequent build times #116

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
634d77a
fix: implemented last fetched time of content-types to reduce subsequ…
vkalta Nov 26, 2021
414d080
fix: minor fix
vkalta Nov 26, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 56 additions & 42 deletions contenttype-data.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ var _interopRequireDefault = require("@babel/runtime/helpers/interopRequireDefau

var _regenerator = _interopRequireDefault(require("@babel/runtime/regenerator"));

var _defineProperty2 = _interopRequireDefault(require("@babel/runtime/helpers/defineProperty"));

var _inherits2 = _interopRequireDefault(require("@babel/runtime/helpers/inherits"));

var _possibleConstructorReturn2 = _interopRequireDefault(require("@babel/runtime/helpers/possibleConstructorReturn"));
Expand All @@ -16,6 +18,10 @@ var _classCallCheck2 = _interopRequireDefault(require("@babel/runtime/helpers/cl

var _createClass2 = _interopRequireDefault(require("@babel/runtime/helpers/createClass"));

function ownKeys(object, enumerableOnly) { var keys = Object.keys(object); if (Object.getOwnPropertySymbols) { var symbols = Object.getOwnPropertySymbols(object); if (enumerableOnly) { symbols = symbols.filter(function (sym) { return Object.getOwnPropertyDescriptor(object, sym).enumerable; }); } keys.push.apply(keys, symbols); } return keys; }

function _objectSpread(target) { for (var i = 1; i < arguments.length; i++) { var source = arguments[i] != null ? arguments[i] : {}; if (i % 2) { ownKeys(Object(source), true).forEach(function (key) { (0, _defineProperty2["default"])(target, key, source[key]); }); } else if (Object.getOwnPropertyDescriptors) { Object.defineProperties(target, Object.getOwnPropertyDescriptors(source)); } else { ownKeys(Object(source)).forEach(function (key) { Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key)); }); } } return target; }

function _createSuper(Derived) { var hasNativeReflectConstruct = _isNativeReflectConstruct(); return function _createSuperInternal() { var Super = (0, _getPrototypeOf2["default"])(Derived), result; if (hasNativeReflectConstruct) { var NewTarget = (0, _getPrototypeOf2["default"])(this).constructor; result = Reflect.construct(Super, arguments, NewTarget); } else { result = Super.apply(this, arguments); } return (0, _possibleConstructorReturn2["default"])(this, result); }; }

function _isNativeReflectConstruct() { if (typeof Reflect === "undefined" || !Reflect.construct) return false; if (Reflect.construct.sham) return false; if (typeof Proxy === "function") return true; try { Boolean.prototype.valueOf.call(Reflect.construct(Boolean, [], function () {})); return true; } catch (e) { return false; } }
Expand Down Expand Up @@ -63,15 +69,13 @@ var FetchDefaultContentTypes = /*#__PURE__*/function (_FetchContentTypes) {
(0, _createClass2["default"])(FetchDefaultContentTypes, [{
key: "getPagedData",
value: function () {
var _getPagedData2 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee2(url, config, responseKey, fn) {
var query, result;
var _getPagedData2 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee2(url, config, responseKey, fn, query) {
var result;
return _regenerator["default"].wrap(function _callee2$(_context2) {
while (1) {
switch (_context2.prev = _context2.next) {
case 0:
query = {
include_global_field_schema: true
};
query.query = JSON.stringify(query.query);
_context2.next = 3;
return fn.apply(null, [url, config, responseKey, query]);

Expand All @@ -87,7 +91,7 @@ var FetchDefaultContentTypes = /*#__PURE__*/function (_FetchContentTypes) {
}, _callee2);
}));

function getPagedData(_x, _x2, _x3, _x4) {
function getPagedData(_x, _x2, _x3, _x4, _x5) {
return _getPagedData2.apply(this, arguments);
}

Expand All @@ -110,22 +114,26 @@ var FetchSpecifiedContentTypes = /*#__PURE__*/function (_FetchContentTypes2) {
(0, _createClass2["default"])(FetchSpecifiedContentTypes, [{
key: "getPagedData",
value: function () {
var _getPagedData3 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee3(url, config, responseKey, fn) {
var query, contentTypes, referredContentTypes, referredContentTypesList, referredContentTypesData, result;
var _getPagedData3 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee3(url, config, responseKey, fn, query) {
var contentTypes, referredContentTypes, referredContentTypesList, referredContentTypesData, result;
return _regenerator["default"].wrap(function _callee3$(_context3) {
while (1) {
switch (_context3.prev = _context3.next) {
case 0:
query = {
query: JSON.stringify({
uid: {
$in: config.contentTypes
}
}),
include_global_field_schema: true
};
query.query.uid = {
$in: config.contentTypes
}; // const queryParams = {
Copy link
Copy Markdown
Contributor

@abhishek305 abhishek305 Nov 26, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remove comment if not required

Copy link
Copy Markdown
Contributor Author

@vkalta vkalta Nov 26, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done!

// query: JSON.stringify({
// uid: { $in: config.contentTypes },
// ...query.query,
// }),
// include_global_field_schema: true
// };

_context3.next = 3;
return fn.apply(null, [url, config, responseKey, query]);
return fn.apply(null, [url, config, responseKey, _objectSpread(_objectSpread({}, query), {}, {
query: JSON.stringify(query.query)
})]);

case 3:
contentTypes = _context3.sent;
Expand All @@ -138,13 +146,14 @@ var FetchSpecifiedContentTypes = /*#__PURE__*/function (_FetchContentTypes2) {
break;
}

query.query = JSON.stringify({
uid: {
$in: referredContentTypesList
}
});
// query.query = JSON.stringify({ uid: { $in: referredContentTypesList } });
Copy link
Copy Markdown
Contributor

@abhishek305 abhishek305 Nov 26, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remove comment if not required

Copy link
Copy Markdown
Contributor Author

@vkalta vkalta Nov 26, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done @abhishek305

query.query.uid = {
$in: referredContentTypesList
};
_context3.next = 11;
return fn.apply(null, [url, config, responseKey, query]);
return fn.apply(null, [url, config, responseKey, _objectSpread(_objectSpread({}, query), {}, {
query: JSON.stringify(query.query)
})]);

case 11:
referredContentTypesData = _context3.sent;
Expand All @@ -161,7 +170,7 @@ var FetchSpecifiedContentTypes = /*#__PURE__*/function (_FetchContentTypes2) {
}, _callee3);
}));

function getPagedData(_x5, _x6, _x7, _x8) {
function getPagedData(_x6, _x7, _x8, _x9, _x10) {
return _getPagedData3.apply(this, arguments);
}

Expand All @@ -184,22 +193,26 @@ var FetchUnspecifiedContentTypes = /*#__PURE__*/function (_FetchContentTypes3) {
(0, _createClass2["default"])(FetchUnspecifiedContentTypes, [{
key: "getPagedData",
value: function () {
var _getPagedData4 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee4(url, config, responseKey, fn) {
var query, contentTypes, referredContentTypes, referredContentTypesList, referredContentTypesData, result;
var _getPagedData4 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee4(url, config, responseKey, fn, query) {
var contentTypes, referredContentTypes, referredContentTypesList, referredContentTypesData, result;
return _regenerator["default"].wrap(function _callee4$(_context4) {
while (1) {
switch (_context4.prev = _context4.next) {
case 0:
query = {
query: JSON.stringify({
uid: {
$nin: config.excludeContentTypes
}
}),
include_global_field_schema: true
};
query.query.uid = {
$nin: config.excludeContentTypes
}; // const queryParams = {
Copy link
Copy Markdown
Contributor

@abhishek305 abhishek305 Nov 26, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remove comment if not required

// query: JSON.stringify({
// uid: { $nin: config.excludeContentTypes },
// ...query.query,
// }),
// include_global_field_schema: true
// };

_context4.next = 3;
return fn.apply(null, [url, config, responseKey, query]);
return fn.apply(null, [url, config, responseKey, _objectSpread(_objectSpread({}, query), {}, {
query: JSON.stringify(query.query)
})]);

case 3:
contentTypes = _context4.sent;
Expand All @@ -212,13 +225,14 @@ var FetchUnspecifiedContentTypes = /*#__PURE__*/function (_FetchContentTypes3) {
break;
}

query.query = JSON.stringify({
uid: {
$in: referredContentTypesList
}
});
// query.query = JSON.stringify({ uid: { $in: referredContentTypesList } });
Comment thread
vkalta marked this conversation as resolved.
Outdated
query.query.uid = {
$in: referredContentTypesList
};
_context4.next = 11;
return fn.apply(null, [url, config, responseKey, query]);
return fn.apply(null, [url, config, responseKey, _objectSpread(_objectSpread({}, query), {}, {
query: JSON.stringify(query.query)
})]);

case 11:
referredContentTypesData = _context4.sent;
Expand All @@ -235,7 +249,7 @@ var FetchUnspecifiedContentTypes = /*#__PURE__*/function (_FetchContentTypes3) {
}, _callee4);
}));

function getPagedData(_x9, _x10, _x11, _x12) {
function getPagedData(_x11, _x12, _x13, _x14, _x15) {
return _getPagedData4.apply(this, arguments);
}

Expand Down
12 changes: 9 additions & 3 deletions entry-data.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,7 +233,9 @@ var FetchSpecifiedContentTypesEntries = /*#__PURE__*/function (_FetchEntries2) {
switch (_context4.prev = _context4.next) {
case 0:
_context4.prev = 0;
syncData = {};
syncData = {
data: []
};
typePrefix = configOptions.type_prefix || 'Contentstack';
_context4.next = 5;
return cache.get(typePrefix);
Expand Down Expand Up @@ -397,7 +399,9 @@ var FetchSpecifiedLocalesEntries = /*#__PURE__*/function (_FetchEntries3) {
switch (_context7.prev = _context7.next) {
case 0:
_context7.prev = 0;
syncData = {};
syncData = {
data: []
};
typePrefix = configOptions.type_prefix || 'Contentstack';
locales = configOptions.locales;
i = 0;
Expand Down Expand Up @@ -557,7 +561,9 @@ var FetchSpecifiedLocalesAndContentTypesEntries = /*#__PURE__*/function (_FetchE
switch (_context10.prev = _context10.next) {
case 0:
_context10.prev = 0;
syncData = {};
syncData = {
data: []
};
typePrefix = configOptions.type_prefix || 'Contentstack';
_context10.next = 5;
return cache.get(typePrefix);
Expand Down
53 changes: 39 additions & 14 deletions fetch.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -98,26 +98,51 @@ exports.fetchData = /*#__PURE__*/function () {
}();

exports.fetchContentTypes = /*#__PURE__*/function () {
var _ref2 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee2(config, contentTypeOption) {
var url, responseKey, contentType, allContentTypes;
var _ref2 = (0, _asyncToGenerator2["default"])( /*#__PURE__*/_regenerator["default"].mark(function _callee2(config, contentTypeOption, cache) {
var typePrefix, lastFetchedTimeCacheKey, lastFetchedTime, query, currentFetchedTime, url, responseKey, contentType, allContentTypes;
return _regenerator["default"].wrap(function _callee2$(_context2) {
while (1) {
switch (_context2.prev = _context2.next) {
case 0:
_context2.prev = 0;
config.cdn = config.cdn ? config.cdn : 'https://cdn.contentstack.io/v3';
typePrefix = config.type_prefix || 'Contentstack';
lastFetchedTimeCacheKey = "".concat(typePrefix.toLowerCase(), "-content-type-fetch-time-").concat(config.api_key);
_context2.next = 6;
return cache.get(lastFetchedTimeCacheKey);

case 6:
lastFetchedTime = _context2.sent;

if (!lastFetchedTime) {
lastFetchedTime = new Date(null).toISOString();
}

query = {
query: {
updated_at: {
$gt: lastFetchedTime
}
}
}; // Any changes here after in the content-types will be re-fetched.

currentFetchedTime = new Date().toISOString();
_context2.next = 12;
return cache.set(lastFetchedTimeCacheKey, currentFetchedTime);

case 12:
url = 'content_types';
responseKey = 'content_types';
contentType = new OPTION_CLASS_MAPPING[contentTypeOption]();
_context2.next = 7;
return contentType.getPagedData(url, config, responseKey, getPagedData);
_context2.next = 17;
return contentType.getPagedData(url, config, responseKey, getPagedData, query);

case 7:
case 17:
allContentTypes = _context2.sent;
return _context2.abrupt("return", allContentTypes);

case 11:
_context2.prev = 11;
case 21:
_context2.prev = 21;
_context2.t0 = _context2["catch"](0);
reporter.panic({
id: CODES.SyncError,
Expand All @@ -127,15 +152,15 @@ exports.fetchContentTypes = /*#__PURE__*/function () {
error: _context2.t0
});

case 14:
case 24:
case "end":
return _context2.stop();
}
}
}, _callee2, null, [[0, 11]]);
}, _callee2, null, [[0, 21]]);
}));

return function (_x5, _x6) {
return function (_x5, _x6, _x7) {
return _ref2.apply(this, arguments);
};
}();
Expand Down Expand Up @@ -163,7 +188,7 @@ var fetchSyncData = /*#__PURE__*/function () {
}, _callee3);
}));

return function fetchSyncData(_x7, _x8) {
return function fetchSyncData(_x8, _x9) {
return _ref3.apply(this, arguments);
};
}();
Expand Down Expand Up @@ -211,7 +236,7 @@ var fetchCsData = /*#__PURE__*/function () {
}, _callee4);
}));

return function fetchCsData(_x9, _x10, _x11) {
return function fetchCsData(_x10, _x11, _x12) {
return _ref4.apply(this, arguments);
};
}();
Expand Down Expand Up @@ -265,7 +290,7 @@ var getPagedData = /*#__PURE__*/function () {
}, _callee5);
}));

return function getPagedData(_x12, _x13, _x14) {
return function getPagedData(_x13, _x14, _x15) {
return _ref5.apply(this, arguments);
};
}();
Expand Down Expand Up @@ -317,7 +342,7 @@ var getSyncData = /*#__PURE__*/function () {
}, _callee6);
}));

return function getSyncData(_x15, _x16, _x17, _x18) {
return function getSyncData(_x16, _x17, _x18, _x19) {
return _ref6.apply(this, arguments);
};
}();
2 changes: 1 addition & 1 deletion gatsby-node.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ exports.createSchemaCustomization = /*#__PURE__*/function () {
_context.prev = 3;
contentTypeOption = getContentTypeOption(configOptions);
_context.next = 7;
return fetchContentTypes(configOptions, contentTypeOption);
return fetchContentTypes(configOptions, contentTypeOption, cache);

case 7:
contentTypes = _context.sent;
Expand Down
2 changes: 1 addition & 1 deletion package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "gatsby-source-contentstack",
"version": "3.1.0",
"version": "3.1.1",
"description": "Gatsby source plugin for building websites using Contentstack as a data source",
"scripts": {
"prepublish": "npm run build",
Expand Down
Loading