Match on arbitrary header and value.

Matt Diehl · Matt Diehl · commit 4528a0a57ccd · 2022-08-06T13:35:38.000-07:00
Based on Issue #36, allows matching on any header and value. ``` (header X-Custom-Header /value/) ``` As we use `GmailMessage.getHeader` function, the header name used in the condition must be case-sensitive. Overall flow: - Rules are parsed, and a list of condition-requested-headers is created. - ThreadData reads in the message data, and all the headers that were requested in the rules (we have to pass SessionData to ThreadData so it knows which headers to search). - Condition.match checks what header to match with, and compares the header's value with the rule's value. In order to get the SessionData.mock.ts to work, SessionData.labels had to be made public. This seems to be a limitation we will continue to hit in testing if we want to test on Sheet Apps, since we cannot use other better libraries.
diff --git a/Condition.ts b/Condition.ts
@@ -15,12 +15,14 @@
  */
 
 import {MessageData} from './ThreadData';
+import {SessionData} from './SessionData';
+import Mocks from './Mocks';
 import Utils from './utils';
 
 const RE_FLAG_PATTERN = /^\/(.*)\/([gimuys]*)$/;
 
 enum ConditionType {
-    AND, OR, NOT, SUBJECT, FROM, TO, CC, BCC, LIST, SENDER, RECEIVER, BODY,
+    AND, OR, NOT, SUBJECT, FROM, TO, CC, BCC, LIST, SENDER, RECEIVER, BODY, HEADER,
 }
 
 /**
@@ -65,7 +67,7 @@ export default class Condition {
         return pattern.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
     }
 
-    private static parseRegExp(pattern: string, condition_str: string, matching_address: boolean): RegExp {
+    public static parseRegExp(pattern: string, condition_str: string, matching_address: boolean): RegExp {
         Utils.assert(pattern.length > 0, `Condition ${condition_str} should have value but not found`);
         const match = pattern.match(RE_FLAG_PATTERN);
         if (match !== null) {
@@ -85,6 +87,7 @@ export default class Condition {
     }
 
     private readonly type: ConditionType;
+    private readonly subtype: string;
     private readonly regexp: RegExp;
     private readonly sub_conditions: Condition[];
 
@@ -94,8 +97,9 @@ export default class Condition {
             `Condition ${condition_str} should be surrounded by ().`);
         const first_space = condition_str.indexOf(" ");
         const type_str = condition_str.substring(1, first_space).trim().toUpperCase();
-        const rest_str = condition_str.substring(first_space + 1, condition_str.length - 1).trim();
+        let rest_str = condition_str.substring(first_space + 1, condition_str.length - 1).trim();
         this.type = ConditionType[type_str as keyof typeof ConditionType];
+        this.subtype = "";
         switch (this.type) {
             case ConditionType.AND:
             case ConditionType.OR: {
@@ -119,6 +123,13 @@ export default class Condition {
                 this.regexp = Condition.parseRegExp(rest_str, condition_str, true);
                 break;
             }
+            case ConditionType.HEADER: {
+                const subtype_first_space = rest_str.indexOf(" ");
+                this.subtype = rest_str.substring(0, subtype_first_space).trim();
+                rest_str = rest_str.substring(subtype_first_space + 1, rest_str.length - 1).trim();
+                this.regexp = Condition.parseRegExp(rest_str, condition_str, false);
+                break;
+            }
             case ConditionType.SUBJECT:
             case ConditionType.BODY: {
                 this.regexp = Condition.parseRegExp(rest_str, condition_str, false);
@@ -177,6 +188,13 @@ export default class Condition {
             case ConditionType.BODY: {
                 return this.regexp.test(message_data.body);
             }
+            case ConditionType.HEADER: {
+                const headerData = message_data.headers.get(this.subtype);
+                if (headerData !== undefined) {
+                    return this.regexp.test(headerData);
+                }
+                return false;
+            }
         }
     }
 
@@ -191,6 +209,17 @@ export default class Condition {
         return `(${type_str} ${regexp_str} ${sub_str})`;
     }
 
+    getConditionHeaders(): string[] {
+        const headers = [];
+        if (this.type === ConditionType.HEADER) {
+            headers.push(this.subtype);
+        }
+        this.sub_conditions?.forEach((sub_condition) => {
+            headers.push(...sub_condition.getConditionHeaders());
+        });
+        return headers;
+    }
+
     public static testRegex(it: Function, expect: Function) {
 
         function test_regexp(condition_str: string, target_str: string, is_address: boolean) {
@@ -271,11 +300,18 @@ export default class Condition {
             getSubject: () => '',
             getPlainBody: () => '',
             getRawContent: () => '',
+            getHeader: (_name: string) => '',
         } as GoogleAppsScript.Gmail.GmailMessage;
 
-        function test_cond(condition_str: string, message: Partial<GoogleAppsScript.Gmail.GmailMessage>): boolean {
+        function test_cond(
+            condition_str: string,
+            message: Partial<GoogleAppsScript.Gmail.GmailMessage>,
+            session_data: Partial<SessionData> = {}): boolean {
             const condition = new Condition(condition_str);
-            const message_data = new MessageData(Object.assign({}, base_message, message));
+            const mock_session_data = Mocks.getMockSessionData(session_data);
+            const message_data = new MessageData(
+                mock_session_data,
+                Object.assign({}, base_message, message));
             return condition.match(message_data);
         }
 
@@ -324,5 +360,56 @@ export default class Condition {
                 getTo: () => 'abc+Def@bar.com',
             })).toBe(true)
         })
+
+        it('Matches custom header with value', () => {
+            expect(test_cond(`(header Sender abc@def.com)`,
+            {
+                getHeader: (name: string) => {
+                    if (name === 'Sender') {
+                        return 'abc@def.com';
+                    }
+                    return '';
+                },
+            },
+            {
+                requested_headers: ['Sender', 'List-Post'],
+            })).toBe(true)
+        })
+        it('Matches nested custom header with value', () => {
+            expect(test_cond(`(and
+                (from abc@gmail.com)
+                (and
+                  (header X-List mylist.gmail.com)
+                  (header Precedence /list/i)))`,
+            {
+                getFrom: () => 'DDD EEE <abc@gmail.com>',
+                getHeader: (name: string) => {
+                    if (name === 'X-List') {
+                        return 'mylist.gmail.com';
+                    }
+                    if (name === 'Precedence') {
+                        return 'bills list';
+                    }
+                    return '';
+                },
+            },
+            {
+                requested_headers: ['X-List', 'Precedence'],
+            })).toBe(true)
+        })
+        it('Does not match custom header with incorrect data', () => {
+            expect(test_cond(`(header MyHeader abc)`,
+            {
+                getHeader: (name: string) => {
+                    if (name === 'MyHeader') {
+                        return 'xyz';
+                    }
+                    return '';
+                },
+            },
+            {
+                requested_headers: ['MyHeader'],
+            })).toBe(false)
+        })
     }
 }
diff --git a/Config.ts b/Config.ts
@@ -57,6 +57,9 @@ export class Config implements Readonly<MutableConfig> {
 
         const values = Utils.withTimer("GetConfigValues", () => {
             const sheet = SpreadsheetApp.getActiveSpreadsheet().getSheetByName('configs');
+            if (sheet === null) {
+                throw new Error("Sheet 'configs' does not exist in this Spreadsheet.");
+            }
             const num_rows = sheet.getLastRow();
             return sheet.getRange(1, 1, num_rows, 2).getDisplayValues().map(row => row.map(cell => cell.trim()));
         });
diff --git a/Mocks.ts b/Mocks.ts
@@ -0,0 +1,36 @@
+import {SessionData} from "SessionData";
+import {Config} from "Config";
+
+
+export default class Mocks {
+
+    private static base_config: Config = {
+        auto_labeling_parent_label: "",
+        go_link: "",
+        hour_of_day_to_run_sanity_checking: 0,
+        max_threads: 50,
+        processed_label: "myProcessed",
+        processing_failed_label: "zFailed",
+        processing_frequency_in_minutes: 5,
+        unprocessed_label: "myUnprocessed",
+    };
+
+    public static getMockConfig = (overrides: Partial<Config> = {}) => (
+        Object.assign({}, Mocks.base_config, overrides)
+    );
+    
+    private static base_session_data: SessionData = {
+        user_email: "abc@gmail.com",
+        config: Mocks.getMockConfig(),
+        labels: {},
+        rules: [],
+        requested_headers: [],
+        processing_start_time: new Date(12345),
+        oldest_to_process: new Date(23456),
+        getOrCreateLabel: () => ({} as GoogleAppsScript.Gmail.GmailLabel),
+    };
+
+    public static getMockSessionData = (overrides: Partial<SessionData> = {}) => (
+        Object.assign({}, Mocks.base_session_data, overrides)
+    );
+}
diff --git a/README.md b/README.md
@@ -127,7 +127,7 @@ Click menu "Gmail Automata" -> "Stop auto processing" to remove auto triggering.
 3. Update the script id in ".clasp.json" file. To find the script id:
     1. Setup the script following the section [Setup](#Setup) above if you
     haven't do it.
-    2. In the spreadsheet, click menu "Extensions" -> "App Script".
+    2. In the spreadsheet, click menu "Extensions" -> "Apps Script".
     3. In the script editor, click menu "Project Settings" > "IDs" > "ScriptID".
 4. Login CLASP: `yarn claspLogin` and authorize the app in the browser.
 5. Deploy current version: `yarn deploy`.
diff --git a/Rule.ts b/Rule.ts
@@ -34,6 +34,10 @@ export class Rule {
         return this.condition.toString();
     }
 
+    getConditionHeaders(): string[] {
+        return this.condition.getConditionHeaders();
+    }
+
     private static parseBooleanValue(str: string): boolean {
         if (str.length === 0) {
             return false;
@@ -85,6 +89,15 @@ export class Rule {
         return result;
     }
 
+    public static getConditionHeaders(rules: Rule[]): string[] {
+        const headers: Set<string> = new Set<string>();
+        rules.forEach((rule) => {
+            const rule_headers = rule.getConditionHeaders();
+            rule_headers.forEach(item => headers.add(item))
+        });
+        return Array.from(headers.values())
+    }
+
     private static parseRules(values: string[][]): Rule[] {
         const row_num = values.length;
         const column_num = values[0].length;
@@ -150,6 +163,9 @@ export class Rule {
     public static getRules(): Rule[] {
         const values: string[][] = Utils.withTimer("GetRuleValues", () => {
             const sheet = SpreadsheetApp.getActiveSpreadsheet().getSheetByName('rules');
+            if(sheet === null) {
+                throw "Active sheet 'rules' not found";
+            }
             const column_num = sheet.getLastColumn();
             const row_num = sheet.getLastRow();
             return sheet.getRange(1, 1, row_num, column_num)
@@ -216,21 +232,81 @@ export class Rule {
         })
 
         it('Loads Simple Rule', () => {
-            const row: string[] = new_row({
-                conditions: '(body /to: me/i)',
-                add_labels: 'abc, xyz',
-                stage: "5",
-            });
             const sheet: string[][] = [
                 headers,
-                row,
+                new_row({
+                    conditions: '(body /to: me/i)',
+                    add_labels: 'abc, xyz',
+                    stage: "5",
+                }),
             ]
 
             const rules = Rule.parseRules(sheet);
+            const condition_headers = Rule.getConditionHeaders(rules);
 
             expect(rules.length).toBe(1);
             expect(rules[0].stage).toBe(5);
             expect(rules[0].thread_action.label_names.size).toBe(2);
+            expect(condition_headers).toEqual([]);
+        })
+
+        it('Loaded Rules are sorted by stage', () => {
+            const sheet: string[][] = [
+                headers,
+                new_row({
+                    conditions: '(body /to: me/i)',
+                    add_labels: 'abc, xyz',
+                    stage: "5",
+                }),
+                new_row({
+                    conditions: '(body /to: me/i)',
+                    add_labels: 'abc, xyz',
+                    stage: "15",
+                }),
+                new_row({
+                    conditions: '(body /to: me/i)',
+                    add_labels: 'abc, xyz',
+                    stage: "1",
+                }),
+            ]
+
+            const rules = Rule.parseRules(sheet);
+
+            expect(rules.length).toBe(3);
+            expect(rules[0].stage).toBe(1);
+            expect(rules[1].stage).toBe(5);
+            expect(rules[2].stage).toBe(15);
+        })
+
+        it('Loads rules with Headers', () => {
+            const sheet: string[][] = [
+                headers,
+                new_row({
+                    conditions: '(and (or (header Test1 /abc/i)' +
+                                '         (header h2 xyz@gmail.com))' +
+                                '     (and' + 
+                                '         (header X-List abcde)' +
+                                '         (header h3 /abcde/)))',
+                    add_labels: 'def, uvw',
+                    stage: "10",
+                }),
+                new_row({
+                    conditions: '(header h5 /asdf/)',
+                    add_labels: 'abc',
+                    stage: "15",
+                }),
+            ]
+
+            const rules = Rule.parseRules(sheet);
+            const condition_headers = Rule.getConditionHeaders(rules);
+
+            expect(rules.length).toBe(2);
+            expect(rules[0].stage).toBe(10);
+            expect(rules[0].thread_action.label_names).toEqual(new Set(['def', 'uvw']));
+            expect(rules[1].stage).toBe(15);
+            expect(rules[1].thread_action.label_names).toEqual(new Set(['abc']));
+            expect(condition_headers).toEqual(
+                ['Test1', 'h2', 'X-List', 'h3', 'h5']);
         })
     }
 }
diff --git a/SessionData.ts b/SessionData.ts
@@ -30,8 +30,9 @@ export class SessionData {
 
     public readonly user_email: string;
     public readonly config: Config;
-    private readonly labels: { [key: string]: GoogleAppsScript.Gmail.GmailLabel };
+    public readonly labels: { [key: string]: GoogleAppsScript.Gmail.GmailLabel };
     public readonly rules: Rule[];
+    public readonly requested_headers: string[];
 
     public readonly processing_start_time: Date;
     public readonly oldest_to_process: Date;
@@ -41,14 +42,15 @@ export class SessionData {
         this.config = Utils.withTimer("getConfigs", () => Config.getConfig());
         this.labels = Utils.withTimer("getLabels", () => SessionData.getLabelMap());
         this.rules = Utils.withTimer("getRules", () => Rule.getRules());
+        this.requested_headers = Utils.withTimer("getHeaders", () => Rule.getConditionHeaders(this.rules));
 
         this.processing_start_time = new Date();
         // Check back two processing intervals to make sure we checked all messages in the thread
         this.oldest_to_process = new Date(
             this.processing_start_time.getTime() - 2 * this.config.processing_frequency_in_minutes * 60 * 1000);
     }
 
-    getOrCreateLabel(name: string) {
+    getOrCreateLabel(name: string): GoogleAppsScript.Gmail.GmailLabel {
         name = name.trim();
         Utils.assert(name.length > 0, "Can't get empty label!");
 
diff --git a/ThreadData.ts b/ThreadData.ts
