[jinja] Add support for macros and | tojson (#692)

Adds support for Macros. See here for more information:
https://jinja.palletsprojects.com/en/3.1.x/templates/#macros

cc @Rocketknight1

Author: xenova

packages/jinja/src/ast.ts

@@ -30,13 +30,18 @@ export class If extends Statement {
 	}
 }
 
+/**
+ * Loop over each item in a sequence
+ * https://jinja.palletsprojects.com/en/3.0.x/templates/#for
+ */
 export class For extends Statement {
 	override type = "For";
 
 	constructor(
 		public loopvar: Identifier | TupleLiteral,
 		public iterable: Expression,
-		public body: Statement[]
+		public body: Statement[],
+		public defaultBlock: Statement[] // if no iteration took place
 	) {
 		super();
 	}
@@ -52,6 +57,18 @@ export class SetStatement extends Statement {
 	}
 }
 
+export class Macro extends Statement {
+	override type = "Macro";
+
+	constructor(
+		public name: Identifier,
+		public args: Expression[],
+		public body: Statement[]
+	) {
+		super();
+	}
+}
+
 /**
  * Expressions will result in a value at runtime (unlike statements).
  */
@@ -182,6 +199,21 @@ export class FilterExpression extends Expression {
 	}
 }
 
+/**
+ * An operation which filters a sequence of objects by applying a test to each object,
+ * and only selecting the objects with the test succeeding.
+ */
+export class SelectExpression extends Expression {
+	override type = "SelectExpression";
+
+	constructor(
+		public iterable: Expression,
+		public test: Expression
+	) {
+		super();
+	}
+}
+
 /**
  * An operation with two sides, separated by the "is" operator.
  */

packages/jinja/src/lexer.ts

@@ -44,6 +44,8 @@ export const TOKEN_TYPES = Object.freeze({
 	And: "And",
 	Or: "Or",
 	Not: "UnaryOperator",
+	Macro: "Macro",
+	EndMacro: "EndMacro",
 });
 
 export type TokenType = keyof typeof TOKEN_TYPES;
@@ -65,10 +67,19 @@ const KEYWORDS = Object.freeze({
 	or: TOKEN_TYPES.Or,
 	not: TOKEN_TYPES.Not,
 	"not in": TOKEN_TYPES.NotIn,
+	macro: TOKEN_TYPES.Macro,
+	endmacro: TOKEN_TYPES.EndMacro,
 
 	// Literals
 	true: TOKEN_TYPES.BooleanLiteral,
 	false: TOKEN_TYPES.BooleanLiteral,
+
+	// NOTE: According to the Jinja docs: The special constants true, false, and none are indeed lowercase.
+	// Because that caused confusion in the past, (True used to expand to an undefined variable that was considered false),
+	// all three can now also be written in title case (True, False, and None). However, for consistency, (all Jinja identifiers are lowercase)
+	// you should use the lowercase versions.
+	True: TOKEN_TYPES.BooleanLiteral,
+	False: TOKEN_TYPES.BooleanLiteral,
 });
 
 /**

packages/jinja/src/parser.ts

@@ -21,6 +21,8 @@ import {
 	SliceExpression,
 	KeywordArgumentExpression,
 	TupleLiteral,
+	Macro,
+	SelectExpression,
 } from "./ast";
 
 /**
@@ -90,6 +92,14 @@ export function parse(tokens: Token[]): Program {
 				expect(TOKEN_TYPES.CloseStatement, "Expected %} token");
 				break;
 
+			case TOKEN_TYPES.Macro:
+				++current;
+				result = parseMacroStatement();
+				expect(TOKEN_TYPES.OpenStatement, "Expected {% token");
+				expect(TOKEN_TYPES.EndMacro, "Expected endmacro token");
+				expect(TOKEN_TYPES.CloseStatement, "Expected %} token");
+				break;
+
 			case TOKEN_TYPES.For:
 				++current;
 				result = parseForStatement();
@@ -173,6 +183,25 @@ export function parse(tokens: Token[]): Program {
 		return new If(test, body, alternate);
 	}
 
+	function parseMacroStatement(): Macro {
+		const name = parsePrimaryExpression();
+		if (name.type !== "Identifier") {
+			throw new SyntaxError(`Expected identifier following macro statement`);
+		}
+		const args = parseArgs();
+		expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
+
+		// Body of macro
+		const body: Statement[] = [];
+
+		// Keep going until we hit {% endmacro
+		while (not(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.EndMacro)) {
+			body.push(parseAny());
+		}
+
+		return new Macro(name as Identifier, args, body);
+	}
+
 	function parseExpressionSequence(primary = false): Statement {
 		const fn = primary ? parsePrimaryExpression : parseExpression;
 		const expressions = [fn()];
@@ -189,7 +218,7 @@ export function parse(tokens: Token[]): Program {
 
 	function parseForStatement(): For {
 		// e.g., `message` in `for message in messages`
-		const loopVariable = parseExpressionSequence(true); // should be an identifier
+		const loopVariable = parseExpressionSequence(true); // should be an identifier/tuple
 		if (!(loopVariable instanceof Identifier || loopVariable instanceof TupleLiteral)) {
 			throw new SyntaxError(`Expected identifier/tuple for the loop variable, got ${loopVariable.type} instead`);
 		}
@@ -204,28 +233,48 @@ export function parse(tokens: Token[]): Program {
 		// Body of for loop
 		const body: Statement[] = [];
 
-		// Keep going until we hit {% endfor
-		while (not(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.EndFor)) {
+		// Keep going until we hit {% endfor or {% else
+		while (not(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.EndFor) && not(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.Else)) {
 			body.push(parseAny());
 		}
 
-		return new For(loopVariable, iterable, body);
+		// (Optional) else block
+		const alternative: Statement[] = [];
+		if (is(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.Else)) {
+			++current; // consume {%
+			++current; // consume else
+			expect(TOKEN_TYPES.CloseStatement, "Expected closing statement token");
+
+			// keep going until we hit {% endfor
+			while (not(TOKEN_TYPES.OpenStatement, TOKEN_TYPES.EndFor)) {
+				alternative.push(parseAny());
+			}
+		}
+
+		return new For(loopVariable, iterable, body, alternative);
 	}
 
 	function parseExpression(): Statement {
 		// Choose parse function with lowest precedence
-		return parseTernaryExpression();
+		return parseIfExpression();
 	}
 
-	function parseTernaryExpression(): Statement {
+	function parseIfExpression(): Statement {
 		const a = parseLogicalOrExpression();
 		if (is(TOKEN_TYPES.If)) {
 			// Ternary expression
 			++current; // consume if
 			const predicate = parseLogicalOrExpression();
-			expect(TOKEN_TYPES.Else, "Expected else token");
-			const b = parseLogicalOrExpression();
-			return new If(predicate, [a], [b]);
+
+			if (is(TOKEN_TYPES.Else)) {
+				// Ternary expression with else
+				++current; // consume else
+				const b = parseLogicalOrExpression();
+				return new If(predicate, [a], [b]);
+			} else {
+				// Select expression on iterable
+				return new SelectExpression(a, predicate);
+			}
 		}
 		return a;
 	}
@@ -477,7 +526,7 @@ export function parse(tokens: Token[]): Program {
 				return new StringLiteral(token.value);
 			case TOKEN_TYPES.BooleanLiteral:
 				++current;
-				return new BooleanLiteral(token.value === "true");
+				return new BooleanLiteral(token.value.toLowerCase() === "true");
 			case TOKEN_TYPES.Identifier:
 				++current;
 				return new Identifier(token.value);