diff --git a/CHANGELOG.md b/CHANGELOG.md index a2c23387b..7c5c1a86e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Fixed + +- log fetch now repects the log API request limit (1 per second) +- log fetch and log tail default to ALL levels by default + ## [3.0.6] - 2025-06-18 ## [3.0.5] - 2025-04-07 diff --git a/package-lock.json b/package-lock.json index 6447981c0..e2602d366 100644 --- a/package-lock.json +++ b/package-lock.json @@ -16,7 +16,7 @@ "frodo": "dist/launch.cjs" }, "devDependencies": { - "@rockcarver/frodo-lib": "3.3.0", + "@rockcarver/frodo-lib": "3.3.1", "@types/colors": "^1.2.1", "@types/fs-extra": "^11.0.1", "@types/jest": "^29.2.3", @@ -1848,9 +1848,9 @@ } }, "node_modules/@rockcarver/frodo-lib": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/@rockcarver/frodo-lib/-/frodo-lib-3.3.0.tgz", - "integrity": "sha512-isPkll4aUeAbpYJMkUMKJiDzjsZ+LItdZYQrbTvdOa/g5xt6dKaI1fcMxNDyq/AnOVoNThcj7VxLWJwBChrZRw==", + "version": "3.3.1", + "resolved": "https://registry.npmjs.org/@rockcarver/frodo-lib/-/frodo-lib-3.3.1.tgz", + "integrity": "sha512-EqPcLQf8mKYpMmf1NqCofrt21g4t8XnN+e48aAYXclHmHXqkZzvXLmCxc8n8L6mDIgWIZQr9qzzPLDYbvTSHZg==", "dev": true, "license": "MIT", "engines": { diff --git a/package.json b/package.json index 6f0626e61..6828cd256 100644 --- a/package.json +++ b/package.json @@ -115,7 +115,7 @@ ] }, "devDependencies": { - "@rockcarver/frodo-lib": "3.3.0", + "@rockcarver/frodo-lib": "3.3.1", "@types/colors": "^1.2.1", "@types/fs-extra": "^11.0.1", "@types/jest": "^29.2.3", diff --git a/src/cli/log/log-fetch.ts b/src/cli/log/log-fetch.ts index 48d92eb22..2b32c5612 100644 --- a/src/cli/log/log-fetch.ts +++ b/src/cli/log/log-fetch.ts @@ -14,7 +14,6 @@ const { getConnectionProfile, saveConnectionProfile } = frodo.conn; const SECONDS_IN_30_DAYS = 2592000; const SECONDS_IN_1_HOUR = 3600; const LOG_TIME_WINDOW_MAX = SECONDS_IN_30_DAYS; -const LOG_TIME_WINDOW_INCREMENT = 1; const deploymentTypes = ['cloud']; @@ -38,11 +37,17 @@ export default function setup() { \n0, SEVERE, FATAL, or ERROR\n1, WARNING, WARN or CONFIG\ \n2, INFO or INFORMATION\n3, DEBUG, FINE, FINER or FINEST\ \n4 or ALL' - ).default('ERROR', `${resolveLevel('ERROR')}`) + ).default('ALL', `${resolveLevel('ALL')}`) ) .addOption( new Option('-t, --transaction-id ', 'Filter by transactionId') ) + .addOption( + new Option( + '-f, --query-filter ', + 'Filter using a query expression' + ) + ) .addOption( new Option( '-b, --begin-timestamp ', @@ -123,53 +128,64 @@ export default function setup() { if (foundCredentials) { const now = Date.now() / 1000; const nowString = new Date(now * 1000).toISOString(); + let beginTs = -1; + let endTs = -1; + if ( - typeof options.beginTimestamp === 'undefined' || - !options.beginTimestamp - ) { - // no beginTimestamp value specified, default is 1 hour ago - const tempStartDate = new Date(); - tempStartDate.setTime((now - SECONDS_IN_1_HOUR) * 1000); - options.beginTimestamp = tempStartDate.toISOString(); - // also override endTimestamp to now - const tempEndDate = new Date(); - tempEndDate.setTime(now * 1000); - options.endTimestamp = tempEndDate; - printMessage( - 'No timestamps specified, defaulting to logs from 1 hour ago', - 'info' - ); - } - if ( - typeof options.endTimestamp === 'undefined' || - !options.endTimestamp + (typeof options.beginTimestamp === 'undefined' || + !options.beginTimestamp) && + (typeof options.endTimestamp === 'undefined' || !options.endTimestamp) ) { - // no endTimestamp value specified, default is now - options.endTimestamp = nowString; - printMessage( - 'No end timestamp specified, defaulting end timestamp to "now"', - 'info' - ); - } - let beginTs = Date.parse(options.beginTimestamp) / 1000; - const endTs = Date.parse(options.endTimestamp) / 1000; - if (endTs < beginTs) { - printMessage( - 'End timestamp can not be before begin timestamp', - 'error' - ); - process.exitCode = 1; - return; - } - if (now - beginTs > LOG_TIME_WINDOW_MAX) { - printMessage( - 'Begin timestamp can not be more than 30 days in the past', - 'error' - ); - process.exitCode = 1; - return; + beginTs = -1; + endTs = -1; + } else { + if ( + typeof options.beginTimestamp === 'undefined' || + !options.beginTimestamp + ) { + // no beginTimestamp value specified, default is 1 hour ago + const tempStartDate = new Date(); + tempStartDate.setTime((now - SECONDS_IN_1_HOUR) * 1000); + options.beginTimestamp = tempStartDate.toISOString(); + // also override endTimestamp to now + const tempEndDate = new Date(); + tempEndDate.setTime(now * 1000); + options.endTimestamp = tempEndDate; + printMessage( + 'No begin timestamp specified, defaulting to logs from 1 hour ago', + 'info' + ); + } + if ( + typeof options.endTimestamp === 'undefined' || + !options.endTimestamp + ) { + // no endTimestamp value specified, default is now + options.endTimestamp = nowString; + printMessage( + 'No end timestamp specified, defaulting end timestamp to "now"', + 'info' + ); + } + beginTs = Date.parse(options.beginTimestamp) / 1000; + endTs = Date.parse(options.endTimestamp) / 1000; + if (endTs < beginTs) { + printMessage( + 'End timestamp can not be before begin timestamp', + 'error' + ); + process.exitCode = 1; + return; + } + if (now - beginTs > LOG_TIME_WINDOW_MAX) { + printMessage( + 'Begin timestamp can not be more than 30 days in the past', + 'error' + ); + process.exitCode = 1; + return; + } } - let intermediateEndTs = 0; printMessage( `Fetching ID Cloud logs from the following sources: ${ command.opts().sources @@ -178,24 +194,18 @@ export default function setup() { }...` ); - let timeIncrement = LOG_TIME_WINDOW_INCREMENT; - if (endTs - beginTs > 30) { - timeIncrement = timeIncrement * 30; - } - do { - intermediateEndTs = beginTs + timeIncrement; - await fetchLogs( - command.opts().sources, - new Date(beginTs * 1000).toISOString(), - new Date(intermediateEndTs * 1000).toISOString(), - resolveLevel(command.opts().level), - command.opts().transactionId, - command.opts().searchString, - null, - config.getNoiseFilters(options.defaults) - ); - beginTs = intermediateEndTs; - } while (intermediateEndTs < endTs); + await fetchLogs( + command.opts().sources, + beginTs == -1 ? null : new Date(beginTs * 1000).toISOString(), + endTs == -1 ? null : new Date(endTs * 1000).toISOString(), + resolveLevel(command.opts().level), + command.opts().transactionId, + command.opts().queryFilter, + command.opts().searchString, + null, + config.getNoiseFilters(options.defaults) + ); + // printMessage(`count: ${counter++}, ts: ${new Date().toISOString()}`); } // no log api credentials else { diff --git a/src/cli/log/log-tail.ts b/src/cli/log/log-tail.ts index 205b801e0..490d32de7 100644 --- a/src/cli/log/log-tail.ts +++ b/src/cli/log/log-tail.ts @@ -30,7 +30,7 @@ export default function setup() { \n0, SEVERE, FATAL, or ERROR\n1, WARNING, WARN or CONFIG\ \n2, INFO or INFORMATION\n3, DEBUG, FINE, FINER or FINEST\ \n4 or ALL' - ).default('ERROR', `${resolveLevel('ERROR')}`) + ).default('ALL', `${resolveLevel('ALL')}`) ) .addOption( new Option('-t, --transaction-id ', 'Filter by transactionId') diff --git a/src/ops/LogOps.ts b/src/ops/LogOps.ts index aacd4ab45..0d6657945 100644 --- a/src/ops/LogOps.ts +++ b/src/ops/LogOps.ts @@ -153,12 +153,20 @@ export async function fetchLogs( endTs: string, levels: string[], txid: string, + filter: string, ffString: string, cookie: string, nf: string[] ) { try { - const logsObject = await fetch(source, startTs, endTs, cookie); + const logsObject = await fetch( + source, + startTs, + endTs, + cookie, + txid, + filter + ); let filteredLogs = []; const noiseFilter = nf == null ? getDefaultNoiseFilter() : nf; if (Array.isArray(logsObject.result)) { @@ -168,12 +176,7 @@ export async function fetchLogs( (el.payload as LogEventPayloadSkeleton).logger ) && !noiseFilter.includes(el.type) && - (levels[0] === 'ALL' || levels.includes(resolvePayloadLevel(el))) && - (typeof txid === 'undefined' || - txid === null || - (el.payload as LogEventPayloadSkeleton).transactionId?.includes( - txid - )) + (levels[0] === 'ALL' || levels.includes(resolvePayloadLevel(el))) ); } @@ -188,12 +191,14 @@ export async function fetchLogs( } }); if (logsObject.pagedResultsCookie != null) { + await new Promise((resolve) => setTimeout(resolve, 1000)); await fetchLogs( source, startTs, endTs, levels, txid, + filter, ffString, logsObject.pagedResultsCookie, nf diff --git a/test/client_cli/en/__snapshots__/log-fetch.test.js.snap b/test/client_cli/en/__snapshots__/log-fetch.test.js.snap index 6c30f8d18..378545575 100644 --- a/test/client_cli/en/__snapshots__/log-fetch.test.js.snap +++ b/test/client_cli/en/__snapshots__/log-fetch.test.js.snap @@ -37,6 +37,7 @@ Options: additional output helpful for troubleshooting. -e, --end-timestamp End timestamp for period. Default: "now" + -f, --query-filter Filter using a query expression --flush-cache Flush token cache. -h, --help Help --idm-host IDM base URL, e.g.: @@ -60,7 +61,7 @@ Options: 1, WARNING, WARN or CONFIG 2, INFO or INFORMATION 3, DEBUG, FINE, FINER or FINEST - 4 or ALL (default: SEVERE,ERROR,FATAL) + 4 or ALL (default: ALL) --login-client-id Specify a custom OAuth2 client id to use a your own oauth2 client for IDM API calls in deployments of type "cloud" or @@ -164,6 +165,7 @@ Options: additional output helpful for troubleshooting. -e, --end-timestamp End timestamp for period. Default: "now" + -f, --query-filter Filter using a query expression --flush-cache Flush token cache. -h, --help Help --idm-host IDM base URL, e.g.: @@ -187,7 +189,7 @@ Options: 1, WARNING, WARN or CONFIG 2, INFO or INFORMATION 3, DEBUG, FINE, FINER or FINEST - 4 or ALL (default: SEVERE,ERROR,FATAL) + 4 or ALL (default: ALL) --login-client-id Specify a custom OAuth2 client id to use a your own oauth2 client for IDM API calls in deployments of type "cloud" or diff --git a/test/client_cli/en/__snapshots__/log-tail.test.js.snap b/test/client_cli/en/__snapshots__/log-tail.test.js.snap index ff92e884d..1385dd60e 100644 --- a/test/client_cli/en/__snapshots__/log-tail.test.js.snap +++ b/test/client_cli/en/__snapshots__/log-tail.test.js.snap @@ -51,7 +51,7 @@ Options: 1, WARNING, WARN or CONFIG 2, INFO or INFORMATION 3, DEBUG, FINE, FINER or FINEST - 4 or ALL (default: SEVERE,ERROR,FATAL) + 4 or ALL (default: ALL) --login-client-id Specify a custom OAuth2 client id to use a your own oauth2 client for IDM API calls in deployments of type "cloud" or @@ -167,7 +167,7 @@ Options: 1, WARNING, WARN or CONFIG 2, INFO or INFORMATION 3, DEBUG, FINE, FINER or FINEST - 4 or ALL (default: SEVERE,ERROR,FATAL) + 4 or ALL (default: ALL) --login-client-id Specify a custom OAuth2 client id to use a your own oauth2 client for IDM API calls in deployments of type "cloud" or