Add Jinja module to package

Co-authored-by: John Mai <[email protected]>

Author: mattt

Package.swift

@@ -13,12 +13,13 @@ let package = Package(
     platforms: [.iOS(.v16), .macOS(.v13)],
     products: [
         .library(name: "Transformers", targets: ["Tokenizers", "Generation", "Models"]),
+        .library(name: "Jinja", targets: ["Jinja"]),
         .executable(name: "transformers", targets: ["TransformersCLI"]),
         .executable(name: "hub-cli", targets: ["HubCLI"]),
     ],
     dependencies: [
         .package(url: "https://github.com/apple/swift-argument-parser.git", .upToNextMinor(from: "1.4.0")),
-        .package(url: "https://github.com/johnmai-dev/Jinja", .upToNextMinor(from: "1.3.0")),
+        .package(url: "https://github.com/apple/swift-collections.git", from: "1.0.0"),
     ],
     targets: [
         .executableTarget(
@@ -29,13 +30,17 @@ let package = Package(
             ]
         ),
         .executableTarget(name: "HubCLI", dependencies: ["Hub", .product(name: "ArgumentParser", package: "swift-argument-parser")]),
-        .target(name: "Hub", resources: [.process("FallbackConfigs")], swiftSettings: swiftSettings),
-        .target(name: "Tokenizers", dependencies: ["Hub", .product(name: "Jinja", package: "Jinja")]),
+        .target(name: "Hub", dependencies: ["Jinja"], resources: [.process("FallbackConfigs")], swiftSettings: swiftSettings),
+        .target(name: "Jinja", dependencies: [
+            .product(name: "OrderedCollections", package: "swift-collections"),
+        ]),
+        .target(name: "Tokenizers", dependencies: ["Hub", "Jinja"]),
         .target(name: "TensorUtils"),
         .target(name: "Generation", dependencies: ["Tokenizers", "TensorUtils"]),
         .target(name: "Models", dependencies: ["Tokenizers", "Generation", "TensorUtils"]),
         .testTarget(name: "TokenizersTests", dependencies: ["Tokenizers", "Models", "Hub"], resources: [.process("Resources"), .process("Vocabs")]),
-        .testTarget(name: "HubTests", dependencies: ["Hub", .product(name: "Jinja", package: "Jinja")], swiftSettings: swiftSettings),
+        .testTarget(name: "HubTests", dependencies: ["Hub", "Jinja"], swiftSettings: swiftSettings),
         .testTarget(name: "TensorUtilsTests", dependencies: ["TensorUtils", "Models", "Hub"], resources: [.process("Resources")]),
+        .testTarget(name: "JinjaTests", dependencies: ["Jinja"]),
     ]
 )

Sources/Hub/Config.swift

@@ -5,6 +5,7 @@
 //  Created by Piotr Kowalczuk on 06.03.25.
 
 import Foundation
+import Jinja
 
 // MARK: - Configuration files with dynamic lookup
 
@@ -413,28 +414,32 @@ public struct Config: Hashable, Sendable,
         self.dictionary(or: or)
     }
 
-    public func toJinjaCompatible() -> Any? {
+    public func toJinjaCompatible() -> Jinja.Value {
         switch self.value {
         case let .array(val):
-            return val.map { $0.toJinjaCompatible() }
+            let converted: [Jinja.Value] = val.map { $0.toJinjaCompatible() }
+            return Jinja.Value.array(converted)
         case let .dictionary(val):
-            var result: [String: Any?] = [:]
+            var result: OrderedDictionary<String, Jinja.Value> = [:]
             for (key, config) in val {
                 result[key.string] = config.toJinjaCompatible()
             }
-            return result
+            return Jinja.Value.object(result)
         case let .boolean(val):
-            return val
+            return Jinja.Value.boolean(val)
         case let .floating(val):
-            return val
+            return Jinja.Value.double(Double(val))
         case let .integer(val):
-            return val
+            return Jinja.Value.int(val)
         case let .string(val):
-            return val.string
+            return Jinja.Value.string(val.string)
         case let .token(val):
-            return [String(val.0): val.1.string] as [String: String]
+            // Represent token as an object mapping id -> token string
+            var obj: OrderedDictionary<String, Jinja.Value> = [:]
+            obj[String(val.0)] = Jinja.Value.string(val.1.string)
+            return Jinja.Value.object(obj)
         case .null:
-            return nil
+            return Jinja.Value.null
         }
     }
 

Sources/Jinja/AST.swift

@@ -0,0 +1,187 @@
+import Foundation
+import OrderedCollections
+
+/// A node in the abstract syntax tree representing template content.
+public indirect enum Node: Hashable, Codable, Sendable {
+    /// Plain text content to be output directly.
+    case text(String)
+
+    /// Expression to be evaluated and output.
+    case expression(Expression)
+
+    /// Control flow statement to be executed.
+    case statement(Statement)
+
+    /// Comment content to be ignored during execution.
+    case comment(String)
+}
+
+/// An expression that can be evaluated to produce a value.
+public indirect enum Expression: Hashable, Codable, Sendable {
+    /// String literal value.
+    case string(String)
+
+    /// Floating-point number literal.
+    case number(Double)
+
+    /// Integer literal value.
+    case integer(Int)
+
+    /// Boolean literal value.
+    case boolean(Bool)
+
+    /// Null literal value.
+    case null
+
+    /// Array literal with ordered elements.
+    case array([Expression])
+
+    /// Tuple literal with ordered elements.
+    case tuple([Expression])
+
+    /// Object literal with key-value pairs.
+    case object(OrderedDictionary<String, Expression>)
+
+    /// Variable or function identifier reference.
+    case identifier(String)
+
+    /// Unary operators for expressions.
+    public enum UnaryOp: String, Hashable, CaseIterable, Codable, Sendable {
+        /// Logical negation operator.
+        case not = "not"
+        /// Numeric negation operator.
+        case minus = "-"
+        /// Numeric identity operator.
+        case plus = "+"
+        /// Unpacking operator.
+        case splat = "*"
+    }
+    /// Unary operation with operator and operand.
+    case unary(UnaryOp, Expression)
+
+    /// Binary operators for expressions.
+    public enum BinaryOp: String, Hashable, CaseIterable, Codable, Sendable {
+        // MARK: Arithmetic Operators
+
+        /// Addition operator (`+`)
+        case add = "+"
+
+        /// Subtraction operator (`-`)
+        case subtract = "-"
+
+        /// Multiplication operator (`*`)
+        case multiply = "*"
+
+        /// Division operator (`/`)
+        case divide = "/"
+
+        /// Floor division operator (`//`)
+        case floorDivide = "//"
+
+        /// Exponentiation operator (`**`)
+        case power = "**"
+
+        /// Modulo operator (`%`)
+        case modulo = "%"
+
+        // MARK: String Operators
+
+        /// String concatenation operator (`~`)
+        case concat = "~"
+
+        // MARK: Equality Comparison Operators
+
+        /// Equality operator (`==`)
+        case equal = "=="
+
+        /// Inequality operator (`!=`)
+        case notEqual = "!="
+
+        // MARK: Relational Comparison Operators
+
+        /// Less than operator (`<`)
+        case less = "<"
+
+        /// Less than or equal to operator (`<=`)
+        case lessEqual = "<="
+
+        /// Greater than operator (`>`)
+        case greater = ">"
+
+        /// Greater than or equal to operator (`>=`)
+        case greaterEqual = ">="
+
+        // MARK: Logical Operators
+
+        /// Logical AND operator (`and`)
+        case and = "and"
+
+        /// Logical OR operator (`or`)
+        case or = "or"
+
+        // MARK: Membership Test Operators
+
+        /// Membership test operator (`in`)
+        case `in` = "in"
+
+        /// Negated membership test operator (`not in`)
+        case notIn = "not in"
+    }
+    /// Binary operation with operator and operands.
+    case binary(BinaryOp, Expression, Expression)
+
+    /// Ternary conditional expression (`value if test else alternate`)
+    case ternary(Expression, test: Expression, alternate: Expression?)
+
+    /// Function call with arguments and keyword arguments.
+    case call(Expression, [Expression], [String: Expression])
+
+    /// Member access (object.property or object[key]).
+    case member(Expression, Expression, computed: Bool)
+
+    /// Array or string slicing operation (`foo[1:2:3]`)
+    case slice(Expression, start: Expression?, stop: Expression?, step: Expression?)
+
+    /// Filter application to transform a value (`foo | filter(2)`)
+    case filter(Expression, String, [Expression], [String: Expression])
+
+    /// `is` test (`foo is divisibleby(2)`)
+    case test(Expression, String, [Expression], negated: Bool)
+}
+
+/// A control flow statement that affects template execution.
+public enum Statement: Hashable, Codable, Sendable {
+    /// Block of nodes to execute sequentially.
+    case program([Node])
+
+    /// Variable assignment statement.
+    case set(target: Expression, value: Expression?, body: [Node])
+
+    /// Conditional statement with test, body, and optional alternate.
+    case `if`(Expression, [Node], [Node])
+
+    /// Loop variable specification for for-loops.
+    public enum LoopVar: Hashable, Codable, Sendable {
+        /// Single loop variable.
+        case single(String)
+        /// Multiple loop variables for unpacking.
+        case tuple([String])
+    }
+    /// Loop statement with variable, iterable, body, else block, and optional condition.
+    case `for`(LoopVar, Expression, [Node], [Node], test: Expression?)
+
+    /// Macro definition with name, parameters, default values, and body.
+    case macro(String, [String], OrderedDictionary<String, Expression>, [Node])
+
+    /// Exits a loop immediately.
+    case `break`
+
+    /// Skips the current iteration of a loop.
+    case `continue`
+
+    /// Calls a macro with a body.
+    case call(callable: Expression, callerArgs: [Expression]?, body: [Node])
+
+    /// Applies a filter to a block of content.
+    case filter(filterExpr: Expression, body: [Node])
+}

Sources/Jinja/Error.swift

@@ -0,0 +1,11 @@
+/// Errors that can occur during Jinja template processing.
+public enum JinjaError: Error, Sendable {
+    /// Error during tokenization of template source.
+    case lexer(String)
+    /// Error during parsing of tokens into AST.
+    case parser(String)
+    /// Error during template execution or evaluation.
+    case runtime(String)
+    /// Error due to invalid template syntax.
+    case syntax(String)
+}