diff --git a/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js b/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js index fb978b5dda412..608d8baf2b78c 100644 --- a/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js +++ b/packages/gatsby/src/internal-plugins/internal-data-bridge/gatsby-node.js @@ -224,10 +224,5 @@ emitter.on(`DELETE_PAGE`, action => { : swapToStagedDelete(deleteNodeActions) } - console.log(`deletePage stuff`, { - transactionId: action.transactionId, - deleteNodeActions, - }) - store.dispatch(deleteNodeActions) }) diff --git a/packages/gatsby/src/redux/actions/commit-staging-nodes.ts b/packages/gatsby/src/redux/actions/commit-staging-nodes.ts new file mode 100644 index 0000000000000..81ff8b3a77244 --- /dev/null +++ b/packages/gatsby/src/redux/actions/commit-staging-nodes.ts @@ -0,0 +1,51 @@ +import { ActionsUnion } from "../types" +import { internalCreateNodeWithoutValidation } from "./internal" +import { actions as publicActions } from "./public" +import { getNode } from "../../datastore" + +import { store } from "../index" + +export const commitStagingNodes = ( + transactionId: string +): Array => { + const transaction = store + .getState() + .nodesStaging.transactions.get(transactionId) + if (!transaction) { + return [] + } + + const actions: Array = [ + { + type: `COMMIT_STAGING_NODES`, + payload: { + transactionId, + }, + }, + ] + + const nodesState = new Map() + for (const action of transaction) { + if (action.type === `CREATE_NODE_STAGING`) { + nodesState.set(action.payload.id, action) + } else if (action.type === `DELETE_NODE_STAGING` && action.payload?.id) { + nodesState.set(action.payload.id, undefined) + } + } + for (const [id, actionOrDelete] of nodesState.entries()) { + if (actionOrDelete) { + actions.push( + ...internalCreateNodeWithoutValidation( + actionOrDelete.payload, + actionOrDelete.plugin, + actionOrDelete + ) + ) + } else { + // delete case + actions.push(publicActions.deleteNode(getNode(id))) + } + } + + return actions +} diff --git a/packages/gatsby/src/redux/actions/internal.ts b/packages/gatsby/src/redux/actions/internal.ts index f9d5f9cc46de3..79683f5f66a2b 100644 --- a/packages/gatsby/src/redux/actions/internal.ts +++ b/packages/gatsby/src/redux/actions/internal.ts @@ -1,4 +1,5 @@ import reporter from "gatsby-cli/lib/reporter" +import _ from "lodash" import { IGatsbyConfig, @@ -28,8 +29,11 @@ import { IDeleteNodeManifests, IClearGatsbyImageSourceUrlAction, ActionsUnion, + IGatsbyNode, + IDeleteNodeAction, } from "../types" +import { store } from "../index" import { gatsbyConfigSchema } from "../../joi-schemas/joi" import { didYouMean } from "../../utils/did-you-mean" import { @@ -39,9 +43,8 @@ import { getInProcessJobPromise, } from "../../utils/jobs/manager" import { getEngineContext } from "../../utils/engine-context" -import { store } from "../index" - import { getNode } from "../../datastore" +import { hasNodeChanged } from "../../utils/nodes" /** * Create a dependency between a page and data. Probably for @@ -450,62 +453,98 @@ export const clearGatsbyImageSourceUrls = } } -let publicActions -export const setPublicActions = (actions): void => { - publicActions = actions +// We add a counter to node.internal for fast comparisons/intersections +// of various node slices. The counter must increase even across builds. +export function getNextNodeCounter(): number { + const lastNodeCounter = store.getState().status.LAST_NODE_COUNTER ?? 0 + if (lastNodeCounter >= Number.MAX_SAFE_INTEGER) { + throw new Error( + `Could not create more nodes. Maximum node count is reached: ${lastNodeCounter}` + ) + } + return lastNodeCounter + 1 } -export const commitStagingNodes = ( - transactionId: string -): Array => { - const transaction = store - .getState() - .nodesStaging.transactions.get(transactionId) - if (!transaction) { - return [] - } +export const findChildren = (initialChildren: Array): Array => { + const children = [...initialChildren] + const queue = [...initialChildren] + const traversedNodes = new Set() - const actions: Array = [ - { - type: `COMMIT_STAGING_NODES`, - payload: { - transactionId, - }, - }, - ] - - const nodesState = new Map() - for (const action of transaction) { - if (action.type === `CREATE_NODE_STAGING`) { - nodesState.set(action.payload.id, action) - } else if (action.type === `DELETE_NODE_STAGING` && action.payload?.id) { - nodesState.set(action.payload.id, undefined) + while (queue.length > 0) { + const currentChild = getNode(queue.pop()!) + if (!currentChild || traversedNodes.has(currentChild.id)) { + continue + } + traversedNodes.add(currentChild.id) + const newChildren = currentChild.children + if (_.isArray(newChildren) && newChildren.length > 0) { + children.push(...newChildren) + queue.push(...newChildren) } } + return children +} - function sanitizeNode(node: any): any { - return { - ...node, - internal: { - ...node.internal, - owner: undefined, - }, +function isNode(node: any): node is IGatsbyNode { + return Boolean(node) +} + +export function internalCreateNodeWithoutValidation( + node: IGatsbyNode, + plugin?: IGatsbyPlugin, + actionOptions?: any +): Array { + let deleteActions: Array | undefined + let updateNodeAction + + const oldNode = getNode(node.id) + + // marking internal-data-bridge as owner of SitePage instead of plugin that calls createPage + if (oldNode && !hasNodeChanged(node.id, node.internal.contentDigest)) { + updateNodeAction = { + ...actionOptions, + plugin, + type: `TOUCH_NODE`, + typeName: node.internal.type, + payload: node.id, + } + } else { + // Remove any previously created descendant nodes as they're all due + // to be recreated. + if (oldNode) { + const createDeleteAction = (node: IGatsbyNode): IDeleteNodeAction => { + return { + ...actionOptions, + type: `DELETE_NODE`, + plugin, + payload: node, + isRecursiveChildrenDelete: true, + } + } + deleteActions = findChildren(oldNode.children) + .map(getNode) + .filter(isNode) + .map(createDeleteAction) } - } - for (const [id, actionOrDelete] of nodesState.entries()) { - if (actionOrDelete) { - actions.push( - publicActions.createNode( - sanitizeNode(actionOrDelete.payload), - actionOrDelete.plugin - ) - ) - } else { - // delete case - actions.push(publicActions.deleteNode(getNode(id), actionOrDelete.plugin)) + node.internal.counter = getNextNodeCounter() + + updateNodeAction = { + ...actionOptions, + type: `CREATE_NODE`, + plugin, + oldNode, + payload: node, } } + const actions: Array = [] + + if (deleteActions && deleteActions.length) { + actions.push(...deleteActions) + } + + actions.push(updateNodeAction) + return actions } diff --git a/packages/gatsby/src/redux/actions/public.js b/packages/gatsby/src/redux/actions/public.js index bad5ac5e538b7..cdbbe6455800d 100644 --- a/packages/gatsby/src/redux/actions/public.js +++ b/packages/gatsby/src/redux/actions/public.js @@ -30,10 +30,16 @@ const apiRunnerNode = require(`../../utils/api-runner-node`) const { getNonGatsbyCodeFrame } = require(`../../utils/stack-trace-utils`) const { getPageMode } = require(`../../utils/page-mode`) const normalizePath = require(`../../utils/normalize-path`).default -import { createJobV2FromInternalJob, setPublicActions } from "./internal" +import { + createJobV2FromInternalJob, + internalCreateNodeWithoutValidation, + getNextNodeCounter, + findChildren, +} from "./internal" import { maybeSendJobToMainProcess } from "../../utils/jobs/worker-messaging" import { reportOnce } from "../../utils/report-once" import { wrapNode } from "../../utils/detect-node-mutations" +import { shouldRunOnCreatePage, shouldRunOnCreateNode } from "../plugin-runner" const isNotTestEnv = process.env.NODE_ENV !== `test` const isTestEnv = process.env.NODE_ENV === `test` @@ -57,29 +63,6 @@ const ensureWindowsDriveIsUppercase = filePath => { : filePath } -const findChildren = initialChildren => { - const children = [...initialChildren] - const queue = [...initialChildren] - const traversedNodes = new Set() - - while (queue.length > 0) { - const currentChild = getNode(queue.pop()) - if (!currentChild || traversedNodes.has(currentChild.id)) { - continue - } - traversedNodes.add(currentChild.id) - const newChildren = currentChild.children - if (_.isArray(newChildren) && newChildren.length > 0) { - children.push(...newChildren) - queue.push(...newChildren) - } - } - return children -} - -import type { Plugin } from "./types" -import { shouldRunOnCreateNode } from "../plugin-runner" - type Job = { id: string, } @@ -482,14 +465,9 @@ ${reservedFields.map(f => ` * "${f}"`).join(`\n`)} } node.id = `SitePage ${internalPage.path}` - let deleteActions - let updateNodeAction - // const shouldCommitImmediately = - // // !shouldRunOnCreateNode() || - // !page.path.includes("hello-world") const transactionId = actionOptions?.transactionId ?? - (shouldRunOnCreateNode() ? node.internal.contentDigest : undefined) + (shouldRunOnCreatePage() ? node.internal.contentDigest : undefined) // Sanitize page object so we don't attempt to serialize user-provided objects that are not serializable later const sanitizedPayload = sanitizeNode(internalPage) @@ -517,53 +495,13 @@ ${reservedFields.map(f => ` * "${f}"`).join(`\n`)} return actions } - const oldNode = getNode(node.id) - - // marking internal-data-bridge as owner of SitePage instead of plugin that calls createPage - if (oldNode && !hasNodeChanged(node.id, node.internal.contentDigest)) { - updateNodeAction = { - ...actionOptions, - plugin: { name: `internal-data-bridge` }, - type: `TOUCH_NODE`, - typeName: node.internal.type, - payload: node.id, - } - } else { - // Remove any previously created descendant nodes as they're all due - // to be recreated. - if (oldNode) { - const createDeleteAction = node => { - return { - ...actionOptions, - type: `DELETE_NODE`, - plugin: { name: `internal-data-bridge` }, - payload: node, - isRecursiveChildrenDelete: true, - } - } - deleteActions = findChildren(oldNode.children) - .map(getNode) - .map(createDeleteAction) - } - - node.internal.counter = getNextNodeCounter() - - updateNodeAction = { - ...actionOptions, - type: `CREATE_NODE`, - plugin: { name: `internal-data-bridge` }, - oldNode, - payload: node, - } - } - - if (deleteActions && deleteActions.length) { - actions.push(...deleteActions) - } - - actions.push(updateNodeAction) + const upsertNodeActions = internalCreateNodeWithoutValidation( + node, + { name: `internal-data-bridge` }, + actionOptions + ) - return actions + return [...actions, ...upsertNodeActions] } /** @@ -605,18 +543,6 @@ actions.deleteNode = (node: any, plugin?: Plugin) => { } } -// We add a counter to node.internal for fast comparisons/intersections -// of various node slices. The counter must increase even across builds. -function getNextNodeCounter() { - const lastNodeCounter = store.getState().status.LAST_NODE_COUNTER ?? 0 - if (lastNodeCounter >= Number.MAX_SAFE_INTEGER) { - throw new Error( - `Could not create more nodes. Maximum node count is reached: ${lastNodeCounter}` - ) - } - return lastNodeCounter + 1 -} - // memberof notation is added so this code can be referenced instead of the wrapper. /** * Create a new node. @@ -861,7 +787,7 @@ actions.createNode = Array.isArray(actions) ? actions : [actions] ).find(action => action.type === `CREATE_NODE`) - if (!createNodeAction) { + if (!createNodeAction || !shouldRunOnCreateNode()) { return Promise.resolve(undefined) } @@ -1552,4 +1478,4 @@ actions.setRequestHeaders = ({ domain, headers }, plugin: Plugin) => { module.exports = { actions } -setPublicActions(actions) +// setPublicActions(actions) diff --git a/packages/gatsby/src/redux/plugin-runner.ts b/packages/gatsby/src/redux/plugin-runner.ts index 105c478da44d5..3f6bb380d15ec 100644 --- a/packages/gatsby/src/redux/plugin-runner.ts +++ b/packages/gatsby/src/redux/plugin-runner.ts @@ -2,8 +2,7 @@ import { Span } from "opentracing" import { emitter, store } from "./index" import apiRunnerNode from "../utils/api-runner-node" import { ActivityTracker } from "../../" -import { ICreateNodeStagingAction } from "./types" -import { commitStagingNodes } from "./actions/internal" +import { ICreateNodeAction, ICreateNodeStagingAction } from "./types" type Plugin = any // TODO @@ -54,8 +53,11 @@ export const startPluginRunner = (): void => { const pluginsImplementingOnCreateNode = plugins.filter(plugin => plugin.nodeAPIs.includes(`onCreateNode`) ) - if (pluginsImplementingOnCreatePage.length > 0) { - hasOnCreatePage = true + + hasOnCreatePage = pluginsImplementingOnCreatePage.length > 0 + hasOnCreateNode = pluginsImplementingOnCreateNode.length > 0 + + if (hasOnCreatePage) { emitter.on(`CREATE_PAGE`, (action: ICreatePageAction) => { const page = action.payload apiRunnerNode( @@ -73,23 +75,26 @@ export const startPluginRunner = (): void => { // We make page nodes outside of the normal action for speed so we manually // call onCreateNode here for SitePage nodes. - if (pluginsImplementingOnCreateNode.length > 0) { - hasOnCreateNode = true - emitter.on(`CREATE_NODE_STAGING`, (action: ICreateNodeStagingAction) => { + if (hasOnCreateNode) { + const createNodeMiddleware = ( + action: ICreateNodeStagingAction | ICreateNodeAction + ): void => { const node = action.payload if (node.internal.type === `SitePage`) { + const transactionId = + action.type === `CREATE_NODE` ? undefined : action.transactionId apiRunnerNode(`onCreateNode`, { node, parentSpan: action.parentSpan, traceTags: { nodeId: node.id, nodeType: node.internal.type }, - traceId: action.transactionId, - transactionId: action.transactionId, + traceId: transactionId, + transactionId, waitForCascadingActions: true, - }).then(() => { - store.dispatch(commitStagingNodes(action.transactionId)) }) } - }) + } + emitter.on(`CREATE_NODE`, createNodeMiddleware) + emitter.on(`CREATE_NODE_STAGING`, createNodeMiddleware) } } diff --git a/packages/gatsby/src/utils/api-runner-node.js b/packages/gatsby/src/utils/api-runner-node.js index 74c3b6fd699c6..a60c4c81c6985 100644 --- a/packages/gatsby/src/utils/api-runner-node.js +++ b/packages/gatsby/src/utils/api-runner-node.js @@ -34,6 +34,10 @@ import errorParser from "./api-runner-error-parser" import { wrapNode, wrapNodes } from "./detect-node-mutations" import { reportOnce } from "./report-once" +/** + * @type {import('../redux/actions/commit-staging-nodes').commitStagingNodes | undefined} + */ +let commitStagingNodes // Override createContentDigest to remove autogenerated data from nodes to // ensure consistent digests. function createContentDigest(node) { @@ -512,6 +516,7 @@ const runAPI = async (plugin, api, args, activity) => { const apisRunningById = new Map() const apisRunningByTraceId = new Map() let waitingForCasacadeToFinish = [] +const ongoingTransactions = new Map() function apiRunnerNode(api, args = {}, { pluginSource, activity } = {}) { const plugins = store.getState().flattenedPlugins @@ -537,7 +542,13 @@ function apiRunnerNode(api, args = {}, { pluginSource, activity } = {}) { } return new Promise(resolve => { - const { parentSpan, traceId, traceTags, waitForCascadingActions } = args + const { + parentSpan, + traceId, + traceTags, + waitForCascadingActions, + transactionId, + } = args const apiSpanArgs = parentSpan ? { childOf: parentSpan } : {} const apiSpan = tracer.startSpan(`run-api`, apiSpanArgs) @@ -578,6 +589,11 @@ function apiRunnerNode(api, args = {}, { pluginSource, activity } = {}) { } apiRunInstance.id = id + if (transactionId) { + const count = (ongoingTransactions.get(transactionId) ?? 0) + 1 + ongoingTransactions.set(transactionId, count) + } + if (waitForCascadingActions) { waitingForCasacadeToFinish.push(apiRunInstance) } @@ -721,6 +737,20 @@ function apiRunnerNode(api, args = {}, { pluginSource, activity } = {}) { emitter.emit(`API_RUNNING_QUEUE_EMPTY`) } + if (transactionId) { + const count = (ongoingTransactions.get(transactionId) ?? 0) - 1 + if (count <= 0) { + ongoingTransactions.delete(transactionId) + if (!commitStagingNodes) { + commitStagingNodes = + require(`../redux/actions/commit-staging-nodes`).commitStagingNodes + } + store.dispatch(commitStagingNodes(transactionId)) + } else { + ongoingTransactions.set(transactionId, count) + } + } + // Filter empty results apiRunInstance.results = results.filter(result => !_.isEmpty(result))