Match on arbitrary header and value.

Based on Issue #36, allows matching on any header and value.
```
(header X-Custom-Header /value/)
```

As we use `GmailMessage.getHeader` function, the header name
used in the condition must be case-sensitive.

Overall flow:
 - Rules are parsed, and a list of condition-requested-headers is
   created.
 - ThreadData reads in the message data, and all the headers
   that were requested in the rules (we have to pass SessionData
   to ThreadData so it knows which headers to search).
 - Condition.match checks what header to match with, and compares
   the header's value with the rule's value.

In order to get the SessionData.mock.ts to work, SessionData.labels
had to be made public. This seems to be a limitation we will continue
to hit in testing if we want to test on Sheet Apps, since we cannot use
other better libraries.

Author: Matt Diehl

Condition.ts

@@ -15,12 +15,14 @@
  */
 
 import {MessageData} from './ThreadData';
+import {SessionData} from './SessionData';
+import Mocks from './Mocks';
 import Utils from './utils';
 
 const RE_FLAG_PATTERN = /^\/(.*)\/([gimuys]*)$/;
 
 enum ConditionType {
-    AND, OR, NOT, SUBJECT, FROM, TO, CC, BCC, LIST, SENDER, RECEIVER, BODY,
+    AND, OR, NOT, SUBJECT, FROM, TO, CC, BCC, LIST, SENDER, RECEIVER, BODY, HEADER,
 }
 
 /**
@@ -65,7 +67,7 @@ export default class Condition {
         return pattern.replace(/[-\/\\^$*+?.()|[\]{}]/g, '\\$&');
     }
 
-    private static parseRegExp(pattern: string, condition_str: string, matching_address: boolean): RegExp {
+    public static parseRegExp(pattern: string, condition_str: string, matching_address: boolean): RegExp {
         Utils.assert(pattern.length > 0, `Condition ${condition_str} should have value but not found`);
         const match = pattern.match(RE_FLAG_PATTERN);
         if (match !== null) {
@@ -85,6 +87,7 @@ export default class Condition {
     }
 
     private readonly type: ConditionType;
+    private readonly subtype: string;
     private readonly regexp: RegExp;
     private readonly sub_conditions: Condition[];
 
@@ -94,8 +97,9 @@ export default class Condition {
             `Condition ${condition_str} should be surrounded by ().`);
         const first_space = condition_str.indexOf(" ");
         const type_str = condition_str.substring(1, first_space).trim().toUpperCase();
-        const rest_str = condition_str.substring(first_space + 1, condition_str.length - 1).trim();
+        let rest_str = condition_str.substring(first_space + 1, condition_str.length - 1).trim();
         this.type = ConditionType[type_str as keyof typeof ConditionType];
+        this.subtype = "";
         switch (this.type) {
             case ConditionType.AND:
             case ConditionType.OR: {
@@ -119,6 +123,13 @@ export default class Condition {
                 this.regexp = Condition.parseRegExp(rest_str, condition_str, true);
                 break;
             }
+            case ConditionType.HEADER: {
+                const subtype_first_space = rest_str.indexOf(" ");
+                this.subtype = rest_str.substring(0, subtype_first_space).trim();
+                rest_str = rest_str.substring(subtype_first_space + 1, rest_str.length - 1).trim();
+                this.regexp = Condition.parseRegExp(rest_str, condition_str, false);
+                break;
+            }
             case ConditionType.SUBJECT:
             case ConditionType.BODY: {
                 this.regexp = Condition.parseRegExp(rest_str, condition_str, false);
@@ -177,6 +188,13 @@ export default class Condition {
             case ConditionType.BODY: {
                 return this.regexp.test(message_data.body);
             }
+            case ConditionType.HEADER: {
+                const headerData = message_data.headers.get(this.subtype);
+                if (headerData !== undefined) {
+                    return this.regexp.test(headerData);
+                }
+                return false;
+            }
         }
     }
 
@@ -191,6 +209,17 @@ export default class Condition {
         return `(${type_str} ${regexp_str} ${sub_str})`;
     }
 
+    getConditionHeaders(): string[] {
+        const headers = [];
+        if (this.type === ConditionType.HEADER) {
+            headers.push(this.subtype);
+        }
+        this.sub_conditions?.forEach((sub_condition) => {
+            headers.push(...sub_condition.getConditionHeaders());
+        });
+        return headers;
+    }
+
     public static testRegex(it: Function, expect: Function) {
 
         function test_regexp(condition_str: string, target_str: string, is_address: boolean) {
@@ -271,11 +300,18 @@ export default class Condition {
             getSubject: () => '',
             getPlainBody: () => '',
             getRawContent: () => '',
+            getHeader: (_name: string) => '',
         } as GoogleAppsScript.Gmail.GmailMessage;
 
-        function test_cond(condition_str: string, message: Partial<GoogleAppsScript.Gmail.GmailMessage>): boolean {
+        function test_cond(
+            condition_str: string,
+            message: Partial<GoogleAppsScript.Gmail.GmailMessage>,
+            session_data: Partial<SessionData> = {}): boolean {
             const condition = new Condition(condition_str);
-            const message_data = new MessageData(Object.assign({}, base_message, message));
+            const mock_session_data = Mocks.getMockSessionData(session_data);
+            const message_data = new MessageData(
+                mock_session_data,
+                Object.assign({}, base_message, message));
             return condition.match(message_data);
         }
 
@@ -324,5 +360,56 @@ export default class Condition {
                 getTo: () => 'abc+Def@bar.com',
             })).toBe(true)
         })
+
+        it('Matches custom header with value', () => {
+            expect(test_cond(`(header Sender abc@def.com)`,
+            {
+                getHeader: (name: string) => {
+                    if (name === 'Sender') {
+                        return 'abc@def.com';
+                    }
+                    return '';
+                },
+            },
+            {
+                requested_headers: ['Sender', 'List-Post'],
+            })).toBe(true)
+        })
+        it('Matches nested custom header with value', () => {
+            expect(test_cond(`(and
+                (from abc@gmail.com)
+                (and
+                  (header X-List mylist.gmail.com)
+                  (header Precedence /list/i)))`,
+            {
+                getFrom: () => 'DDD EEE <abc@gmail.com>',
+                getHeader: (name: string) => {
+                    if (name === 'X-List') {
+                        return 'mylist.gmail.com';
+                    }
+                    if (name === 'Precedence') {
+                        return 'bills list';
+                    }
+                    return '';
+                },
+            },
+            {
+                requested_headers: ['X-List', 'Precedence'],
+            })).toBe(true)
+        })
+        it('Does not match custom header with incorrect data', () => {
+            expect(test_cond(`(header MyHeader abc)`,
+            {
+                getHeader: (name: string) => {
+                    if (name === 'MyHeader') {
+                        return 'xyz';
+                    }
+                    return '';
+                },
+            },
+            {
+                requested_headers: ['MyHeader'],
+            })).toBe(false)
+        })
     }
 }

Mocks.ts

@@ -24,6 +24,7 @@ export default class Mocks {
         config: Mocks.getMockConfig(),
         labels: {},
         rules: [],
+        requested_headers: [],
         processing_start_time: new Date(12345),
         oldest_to_process: new Date(23456),
         getOrCreateLabel: () => ({} as GoogleAppsScript.Gmail.GmailLabel),

Rule.ts

@@ -35,6 +35,10 @@ export class Rule {
         return this.condition.toString();
     }
 
+    getConditionHeaders(): string[] {
+        return this.condition.getConditionHeaders();
+    }
+
     private static parseBooleanValue(str: string): boolean {
         if (str.length === 0) {
             return false;
@@ -86,6 +90,15 @@ export class Rule {
         return result;
     }
 
+    public static getConditionHeaders(rules: Rule[]): string[] {
+        const headers: Set<string> = new Set<string>();
+        rules.forEach((rule) => {
+            const rule_headers = rule.getConditionHeaders();
+            rule_headers.forEach(item => headers.add(item))
+        });
+        return Array.from(headers.values())
+    }
+
     public static parseRules(values: string[][]): Rule[] {
         const row_num = values.length;
         const column_num = values[0].length;
@@ -206,10 +219,12 @@ export class Rule {
                 }]);
 
             const rules = Rule.parseRules(sheet);
+            const condition_headers = Rule.getConditionHeaders(rules);
 
             expect(rules.length).toBe(1);
             expect(rules[0].stage).toBe(5);
             expect(rules[0].thread_action.label_names.size).toBe(2);
+            expect(condition_headers).toEqual([]);
         })
 
         it('Loaded Rules are sorted by stage', () => {
@@ -239,5 +254,34 @@ export class Rule {
             expect(rules[2].stage).toBe(15);
         })
 
+        it('Loads rules with Headers', () => {
+            const sheet = Mocks.getMockTestSheet([
+                {
+                    conditions: '(and (or (header Test1 /abc/i)' +
+                                '         (header h2 xyz@gmail.com))' +
+                                '     (and' + 
+                                '         (header X-List abcde)' +
+                                '         (header h3 /abcde/)))',
+                    add_labels: 'def, uvw',
+                    stage: "10",
+                },
+                {
+                    conditions: '(header h5 /asdf/)',
+                    add_labels: 'abc',
+                    stage: "15",
+                }
+            ]);
+
+            const rules = Rule.parseRules(sheet);
+            const condition_headers = Rule.getConditionHeaders(rules);
+
+            expect(rules.length).toBe(2);
+            expect(rules[0].stage).toBe(10);
+            expect(rules[0].thread_action.label_names).toEqual(new Set(['def', 'uvw']));
+            expect(rules[1].stage).toBe(15);
+            expect(rules[1].thread_action.label_names).toEqual(new Set(['abc']));
+            expect(condition_headers).toEqual(
+                ['Test1', 'h2', 'X-List', 'h3', 'h5']);
+        })
     }
 }

SessionData.ts

@@ -32,6 +32,7 @@ export class SessionData {
     public readonly config: Config;
     public readonly labels: { [key: string]: GoogleAppsScript.Gmail.GmailLabel };
     public readonly rules: Rule[];
+    public readonly requested_headers: string[];
 
     public readonly processing_start_time: Date;
     public readonly oldest_to_process: Date;
@@ -41,14 +42,15 @@ export class SessionData {
         this.config = Utils.withTimer("getConfigs", () => Config.getConfig());
         this.labels = Utils.withTimer("getLabels", () => SessionData.getLabelMap());
         this.rules = Utils.withTimer("getRules", () => Rule.getRules());
+        this.requested_headers = Utils.withTimer("getHeaders", () => Rule.getConditionHeaders(this.rules));
 
         this.processing_start_time = new Date();
         // Check back two processing intervals to make sure we checked all messages in the thread
         this.oldest_to_process = new Date(
             this.processing_start_time.getTime() - 2 * this.config.processing_frequency_in_minutes * 60 * 1000);
     }
 
-    getOrCreateLabel(name: string) {
+    getOrCreateLabel(name: string): GoogleAppsScript.Gmail.GmailLabel {
         name = name.trim();
         Utils.assert(name.length > 0, "Can't get empty label!");
 

ThreadData.ts

@@ -68,8 +68,9 @@ export class MessageData {
     public readonly receivers: string[];
     public readonly subject: string;
     public readonly body: string;
+    public readonly headers: Map<string, string>;
 
-    constructor(message: GoogleAppsScript.Gmail.GmailMessage) {
+    constructor(session_data: SessionData, message: GoogleAppsScript.Gmail.GmailMessage) {
         this.from = message.getFrom();
         this.to = MessageData.parseAddresses(message.getTo());
         this.cc = MessageData.parseAddresses(message.getCc());
@@ -79,6 +80,10 @@ export class MessageData {
         this.sender = ([] as string[]).concat(this.from, this.reply_to);
         this.receivers = ([] as string[]).concat(this.to, this.cc, this.bcc, this.list);
         this.subject = message.getSubject();
+        this.headers = new Map<string, string>();
+        session_data.requested_headers.forEach(header => {
+            this.headers.set(header, message.getHeader(header));
+        });
         // Potentially could be HTML, Plain, or RAW. But doesn't seem very useful other than Plain.
         let body = message.getPlainBody();
         // Truncate and log long messages.
@@ -111,7 +116,7 @@ export class ThreadData {
         if (newMessages.length === 0) {
             newMessages = [messages[messages.length - 1]];
         }
-        this.message_data_list = newMessages.map(message => new MessageData(message));
+        this.message_data_list = newMessages.map(message => new MessageData(session_data, message));
 
         // Log if any dropped.
         const numDropped = messages.length - newMessages.length;