Add log group glob support. Fixes #3

Makefile

@@ -1,5 +1,14 @@
-release:
+.PHONY: build
+build:
 	npm run build
 
+.PHONY: test
+test:
+	npm test
+
+.PHONY: release
+release: build test
+
+.PHONY: publish
 publish:
 	npm publish

README.md

@@ -29,6 +29,8 @@ $ AWS_PROFILE=staging AWS_REGION=us-east-1 cwq --log-group MyLogGroup 'filter @m
 
 ## Examples
 
+### Time Ranges
+
 Find errors in a specific log group over the last hour:
 
 ```bash
@@ -53,6 +55,8 @@ Find errors within a specific time range (ISO 8601 format):
 $ cwq --log-group MyLogGroup --start 2021-05-08T06:00:00Z --end 2021-05-08T12:00:00Z 'filter @message like /ERROR/'
 ```
 
+### Formats
+
 Change output format to JSON for more advanced queries (Lambda memory example):
 
 ```bash
@@ -64,3 +68,22 @@ Pipe the CSV output into a markdown formatter for sharing with friends (using [`
 ```bash
 $ cwq --log-group MyLogGroup 'filter @type = "REPORT" | status max(@maxMemoryUsed / 1000 / 1000) as maxMemoryUsedMB by bin(5m)' | csvtomd
 ```
+
+### Log Group Matching
+
+You can provide a glob expression to the `--log-group` argument to match log group names by prefix:
+
+```bash
+$ cwq --log-group 'MyPrefix-*' 'filter @message like /ERROR/'
+```
+
+## IAM Policy Requirements
+
+In order to run this utility, the IAM entity associated with the call must `ALLOW` the following actions:
+
+```
+logs:StartQuery
+logs:GetQueryResults
+logs:StopQuery
+logs:DescribeLogGroups
+```

src/index.ts

@@ -4,66 +4,86 @@ import {
   GetQueryResultsCommandOutput,
   QueryStatus,
   StartQueryCommand,
-} from "@aws-sdk/client-cloudwatch-logs";
-import { Parser as CsvParser } from "json2csv";
-import * as yargs from "yargs";
+} from '@aws-sdk/client-cloudwatch-logs';
+import { Parser as CsvParser } from 'json2csv';
+import * as yargs from 'yargs';
 
-import * as time from "./time";
+import * as logs from './logs';
+import * as time from './time';
 
 const args = yargs
   .command(
-    "cwq",
-    "Executes a CloudWatch Logs Insights query and outputs the results"
+    'cwq',
+    'Executes a CloudWatch Logs Insights query and outputs the results'
   )
-  .option("log-group", {
-    alias: "l",
-    type: "string",
-    description: "The name of the log group",
+  .option('log-group', {
+    alias: 'l',
+    type: 'string',
+    description: 'The name of the log group',
   })
-  .option("format", {
-    alias: "f",
-    type: "string",
-    choices: ["csv", "json"],
-    default: "csv",
-    description: "The format of the results",
+  .option('format', {
+    alias: 'f',
+    type: 'string',
+    choices: ['csv', 'json'],
+    default: 'csv',
+    description: 'The format of the results',
   })
-  .option("start", {
-    alias: "s",
-    type: "string",
-    default: "1h",
-    description: "When to start the search (duration or ISO 8601 format)",
+  .option('start', {
+    alias: 's',
+    type: 'string',
+    default: '1h',
+    description: 'When to start the search (duration or ISO 8601 format)',
   })
-  .option("end", {
-    alias: "e",
-    type: "string",
-    default: "now",
-    description: "When to end the search (duration or ISO 8601 format)",
+  .option('end', {
+    alias: 'e',
+    type: 'string',
+    default: 'now',
+    description: 'When to end the search (duration or ISO 8601 format)',
   })
   .help()
-  .alias("help", "h").argv;
+  .alias('help', 'h').argv;
 
 (async function (): Promise<void> {
   // Parse arguments
   const logGroups = args.logGroup;
   if (!logGroups) {
-    throw new Error("A log group is required (specify with -l or --log-group)");
+    throw new Error('A log group is required (specify with -l or --log-group)');
   }
-  const logGroupNames = Array.isArray(logGroups) ? logGroups : [logGroups];
+
+  const inputLogGroupNames: string[] = Array.isArray(logGroups) ? logGroups : [logGroups];
 
   const startTime = Math.ceil(time.parseTimeOrDuration(args.start) / 1000);
   const endTime = Math.ceil(time.parseTimeOrDuration(args.end) / 1000);
 
   const query = args._[0];
   if (!query) {
-    throw new Error("No query provided");
+    throw new Error('No query provided');
   }
   const queryString = query.toString();
 
-  // Execute the query
   const logsClient = new CloudWatchLogsClient({});
+
+  // Expand log group names
+  const logGroupNames = new Set<string>();
+  for (const inputName of inputLogGroupNames) {
+    if (inputName.endsWith('*')) {
+      const matchingLogGroups = await logs.getLogGroupsByPrefix(inputName.replace('*', ''), logsClient);
+      matchingLogGroups.forEach(g => logGroupNames.add(g));
+    } else {
+      logGroupNames.add(inputName);
+    }
+  }
+
+  if (logGroupNames.size === 0) {
+    throw new Error('No explicit or matching log groups provided.');
+  }
+
+  console.error(`Querying ${logGroupNames.size} log group(s): ${JSON.stringify([...logGroupNames])}`);
+
+  // Execute the query
   const startOutput = await logsClient.send(
     new StartQueryCommand({
-      logGroupNames,
+      logGroupNames: [...logGroupNames],
       queryString,
       startTime,
       endTime,
@@ -96,29 +116,33 @@ const args = yargs
   }
 
   if (!queryResults.results) {
-    throw new Error("No results returned");
+    throw new Error('No results returned');
   }
 
   // Translate results into an array of key/value pairs, excluding the pointer field
   const results = queryResults.results.map((r) => {
     const result: { [key: string]: string } = {};
     for (const item of r) {
-      const field = item.field || "<no name>";
-      if (item.field === "@ptr") {
+      const field = item.field || '<no name>';
+      if (item.field === '@ptr') {
         continue;
       }
 
-      result[field] = item.value || "";
+      result[field] = item.value || '';
     }
     return result;
   });
 
+  if(results.length === 0) {
+    throw new Error('No results returned');
+  }
+
   // Output the results in the desired format
   switch (args.format) {
-    case "csv":
+    case 'csv':
       console.log(new CsvParser().parse(results));
       break;
-    case "json":
+    case 'json':
       console.log(JSON.stringify(results));
       break;
   }

src/logs.ts

@@ -0,0 +1,29 @@
+import {
+  CloudWatchLogsClient,
+  DescribeLogGroupsCommand,
+  DescribeLogGroupsCommandOutput
+} from "@aws-sdk/client-cloudwatch-logs";
+
+export const getLogGroupsByPrefix = async(logGroupPrefix: string, client: CloudWatchLogsClient): Promise<string[]> => {
+    const logGroups: string[] = [];
+
+    let response: DescribeLogGroupsCommandOutput = {
+        $metadata: {}
+    };
+    do {
+        response = await client.send(new DescribeLogGroupsCommand({
+            logGroupNamePrefix: logGroupPrefix,
+            nextToken: response.nextToken
+        }));
+
+        const matchingLogGroups = response.logGroups;
+        if (!matchingLogGroups) {
+            throw new Error(`No log groups found for prefix ${logGroupPrefix}`);
+        }
+
+        logGroups.push(...matchingLogGroups.map(g => g.logGroupName!));
+    } while (response.nextToken)
+
+    return logGroups;
+}
+

src/time.ts

@@ -1,9 +1,9 @@
-import parse from "parse-duration";
+import parse from 'parse-duration';
 
 // Attempts to parse a time as a duration or ISO 8601 format, returning a unix timestamp
 export const parseTimeOrDuration = (timeOrDuration: string): number => {
   // special case: `now` is now
-  if (timeOrDuration === "now") {
+  if (timeOrDuration === 'now') {
     return new Date().getTime();
   }
 
@@ -17,7 +17,7 @@ export const parseTimeOrDuration = (timeOrDuration: string): number => {
   }
 
   // otherwise, try to parse as a duration
-  const duration = parse(timeOrDuration, "ms");
+  const duration = parse(timeOrDuration, 'ms');
   if (!duration) {
     throw new Error(`Unable to parse time as a duration: ${timeOrDuration}`);
   }