Add substring() function to stdlib and improve logging (#21)

hardbyte · web-flow · commit d18ff9e030a1 · 2025-10-14T13:26:17.000+13:00
* Remove warning log
* Add substring function to stdlib, use stdlib in CLI
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+- Minor removed warning level logging in cel crate
 
 ## [0.5.2] - 2025-09-12
 
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.toml b/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name = "cel"
-version = "0.5.2"
+version = "0.5.3"
 edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
diff --git a/docs/getting-started/quick-start.md b/docs/getting-started/quick-start.md
@@ -261,8 +261,8 @@ assert result == 6  # → 6 (unsigned integers convert to regular int)
 result = evaluate('"hello world".size()')
 assert result == 11  # → 11 (string length via size() method)
 
-result = evaluate('"hello"[1]')
-assert result == "e"  # → "e" (zero-indexed string character access)
+# Note: String indexing like "hello"[1] is not supported in CEL
+# Use string methods instead: startsWith(), endsWith(), contains(), matches()
 
 result = evaluate('"test".startsWith("te")')
 assert result == True  # → True (rich string method support)
diff --git a/docs/how-to-guides/error-handling.md b/docs/how-to-guides/error-handling.md
@@ -17,15 +17,15 @@ try:
     evaluate("1 + + 2")  # Invalid syntax
     assert False, "Expected ValueError"
 except ValueError as e:
-    assert "Failed to compile expression" in str(e)
-    # → ValueError: Failed to compile expression (graceful failure)
+    assert "Failed to parse expression" in str(e)
+    # → ValueError: Failed to parse expression (graceful failure)
 
 try:
     evaluate("")  # Empty expression
     assert False, "Expected ValueError"
 except ValueError as e:
-    assert "Invalid syntax" in str(e) or "malformed" in str(e)
-    # → ValueError: Invalid syntax or malformed (safe error handling)
+    assert "Failed to parse expression" in str(e)
+    # → ValueError: Failed to parse expression (safe error handling)
 ```
 
 ### `RuntimeError` - Variable and Function Errors
@@ -99,14 +99,14 @@ try:
     evaluate("'unclosed quote", {})
     assert False, "Should have raised ValueError"
 except ValueError as e:
-    assert "Invalid syntax or malformed string" in str(e)
+    assert "Failed to parse expression" in str(e)
     # → ValueError: Malformed input handled safely (no crash)
 
 try:
     evaluate('"mixed quotes\'', {})
     assert False, "Should have raised ValueError"
 except ValueError as e:
-    assert "Invalid syntax or malformed string" in str(e)
+    assert "Failed to parse expression" in str(e)
     # → ValueError: Quote mismatch detected (process remains stable)
 ```
 
@@ -339,7 +339,7 @@ invalid_syntax = 'user.age >='  # Incomplete comparison
 success, result, errors = safe_user_expression_eval(invalid_syntax, context)
 assert success == False, "Invalid syntax should be rejected"
 assert len(errors) > 0, "Should report syntax errors"
-# → False, errors: ['Evaluation error: Failed to compile'] (malformed input caught)
+# → False, errors: ['Evaluation error: Failed to parse'] (malformed input caught)
 
 # Test 4: Empty expression
 success, result, errors = safe_user_expression_eval('', context)
diff --git a/docs/reference/cel-compliance.md b/docs/reference/cel-compliance.md
@@ -92,7 +92,7 @@ This implementation correctly follows the CEL specification where maps can have
 
 #### Other Operators
 - `?:` (ternary conditional) - Conditional expressions
-- `[]` (indexing) - Lists, maps, strings
+- `[]` (indexing) - Lists and maps only (string indexing not supported)
 - `.` (member access) - Object property access
 
 ### ✅ Built-in Functions
@@ -119,9 +119,12 @@ This implementation correctly follows the CEL specification where maps can have
 - **endsWith()**: `"hello".endsWith("lo")` → `True`
 - **matches()**: `"hello world".matches(".*world")` → `True`
 - **String concatenation**: `"hello" + " world"` → `"hello world"`
-- **String indexing**: `"hello"[1]` → `"e"`  
 - **String size**: `size("hello")` → `5`
 
+#### ❌ String Indexing Not Supported
+- **String indexing**: `"hello"[1]` is **NOT** supported (returns "No such key" error)
+- **Workaround**: Use `substring()` function (when available) or Python context functions
+
 ### ✅ Collection Macros
 - **all()**: `[1,2,3].all(x, x > 0)` → `True`
 - **exists()**: `[1,2,3].exists(x, x == 2)` → `True`
@@ -148,13 +151,39 @@ This section focuses on what you need to know to use CEL effectively in your app
 ### 🔧 Safe Patterns & Workarounds
 
 #### String Processing Workarounds
+
+**Using cel.stdlib (Recommended)**
+
+This library provides Python implementations of missing CEL functions:
+
+```python
+from cel import Context, evaluate
+from cel.stdlib import add_stdlib_to_context
+
+# Add all standard library functions at once
+context = Context()
+add_stdlib_to_context(context)
+
+# substring() is now available as a function (not a method)
+result = evaluate('substring("hello world", 0, 5)', context)  # → "hello"
+result = evaluate('substring("hello world", 6)', context)  # → "world"
+
+# Note: Use function syntax, not method syntax
+# ✅ substring("hello", 2, 4)  - correct
+# ❌ "hello".substring(2, 4)   - not supported
+```
+
+**Using Custom Python Functions**
+
+You can also add your own custom functions:
+
 ```python
 from cel import Context, evaluate
 
-# Since lowerAscii(), upperAscii(), indexOf() are missing:
+# Add custom functions for missing CEL features
 context = Context()
 context.add_function("lower", str.lower)
-context.add_function("upper", str.upper) 
+context.add_function("upper", str.upper)
 context.add_function("find", str.find)
 
 # Add variables to the context
diff --git a/docs/reference/cli-reference.md b/docs/reference/cli-reference.md
@@ -15,6 +15,22 @@ cel --version
 
 The `cel` command-line tool provides a convenient way to evaluate CEL expressions from the command line, in scripts, or interactively. It supports context loading, file processing, and various output formats.
 
+## Standard Library Functions
+
+The CLI automatically includes all [standard library functions](../reference/cel-compliance.md#using-celstdlib-recommended) from `cel.stdlib`. These functions are available without any additional setup:
+
+### Available Functions
+
+- **`substring(str, start, end?)`** - Extract substring from a string
+  ```bash
+  cel 'substring("hello world", 0, 5)'  # → hello
+  cel 'substring("hello world", 6)'     # → world
+  ```
+
+**Note**: Use function syntax `substring("text", 0, 5)`, not method syntax `"text".substring(0, 5)`.
+
+For programmatic use (Python API), import and use `cel.stdlib.add_stdlib_to_context()` to add these functions to your context.
+
 ## Options
 
 ### Global Options
diff --git a/docs/reference/python-api.md b/docs/reference/python-api.md
@@ -208,18 +208,18 @@ from cel import evaluate
 # Invalid syntax raises ValueError
 try:
     evaluate("1 + + 2")  # Invalid syntax
-    # → ValueError: Failed to compile expression: ...
+    # → ValueError: Failed to parse expression: ...
     assert False, "Should have raised ValueError"
 except ValueError as e:
-    assert "Failed to compile expression" in str(e)
+    assert "Failed to parse expression" in str(e)
 
 # Empty expression raises ValueError
 try:
     evaluate("")
-    # → ValueError: Invalid syntax or malformed expression
+    # → ValueError: Failed to parse expression
     assert False, "Should have raised ValueError"
 except ValueError as e:
-    assert "Invalid syntax" in str(e) or "malformed" in str(e)
+    assert "Failed to parse expression" in str(e)
 ```
 
 #### `RuntimeError` - Variable and Function Errors
diff --git a/python/cel/__init__.py b/python/cel/__init__.py
@@ -1,4 +1,4 @@
 # Import the Rust extension
 
-from . import cli
+from . import cli, stdlib
 from .cel import *
diff --git a/python/cel/cli.py b/python/cel/cli.py
@@ -39,6 +39,7 @@
 
 # Import directly from relative modules to avoid circular imports
 from .cel import Context, evaluate
+from .stdlib import add_stdlib_to_context
 
 # Initialize Rich console
 console = Console()
@@ -74,7 +75,7 @@ class CELLexer(RegexLexer):
             # Built-in functions
             (
                 r"\b(size|has|timestamp|duration|int|uint|double|string|bytes|"
-                r"startsWith|endsWith|contains|matches)\b(?=\()",
+                r"startsWith|endsWith|contains|matches|substring)\b(?=\()",
                 token.Name.Function,
             ),
             # String literals
@@ -192,7 +193,10 @@ def _update_cel_context(self):
         if self.context:
             self._cel_context = Context(self.context)
         else:
-            self._cel_context = None
+            self._cel_context = Context()
+
+        # Always add stdlib functions to the context
+        add_stdlib_to_context(self._cel_context)
 
     def evaluate(self, expression: str) -> Any:
         """Evaluate a CEL expression."""
@@ -241,6 +245,7 @@ def __init__(self, evaluator: CELEvaluator, history_limit: int = 10):
             "double",
             "string",
             "bytes",
+            "substring",  # stdlib function
         ]
 
         # Command dispatch dictionary for cleaner organization
diff --git a/python/cel/stdlib.py b/python/cel/stdlib.py
@@ -0,0 +1,61 @@
+"""
+Standard library functions for CEL that aren't available in cel-rust.
+
+This module provides Python implementations of CEL standard library functions
+that are missing from the upstream cel-rust implementation.
+"""
+
+
+def substring(s: str, start: int, end: int | None = None) -> str:
+    """
+    Extract a substring from a string.
+
+    This implements the CEL substring() function which is not yet available
+    in cel-rust upstream. See https://github.com/cel-rust/cel-rust/issues/200
+
+    Args:
+        s: The source string
+        start: Starting index (0-based, inclusive)
+        end: Optional ending index (0-based, exclusive). If not provided,
+             extracts to the end of the string.
+
+    Returns:
+        The extracted substring
+
+    Examples:
+        >>> substring("hello world", 0, 5)
+        'hello'
+        >>> substring("hello world", 6)
+        'world'
+        >>> substring("hello", 1, 4)
+        'ell'
+    """
+    if end is None:
+        return s[start:]
+    return s[start:end]
+
+
+# Dictionary mapping function names to their implementations
+# This makes it easy to add all stdlib functions to a Context at once
+STDLIB_FUNCTIONS = {
+    "substring": substring,
+}
+
+
+def add_stdlib_to_context(context):
+    """
+    Add all stdlib functions to a CEL Context.
+
+    Args:
+        context: A cel.Context object
+
+    Example:
+        >>> import cel
+        >>> from cel.stdlib import add_stdlib_to_context
+        >>> context = cel.Context()
+        >>> add_stdlib_to_context(context)
+        >>> cel.evaluate('substring("hello", 0, 2)', context)
+        'he'
+    """
+    for name, func in STDLIB_FUNCTIONS.items():
+        context.add_function(name, func)
diff --git a/src/lib.rs b/src/lib.rs
@@ -2,7 +2,7 @@ mod context;
 
 use ::cel::objects::{Key, TryIntoValue};
 use ::cel::{Context as CelContext, ExecutionError, Program, Value};
-use log::{debug, warn};
+use log::debug;
 use pyo3::exceptions::{PyRuntimeError, PyTypeError, PyValueError};
 use pyo3::prelude::*;
 use pyo3::BoundObject;
@@ -429,7 +429,7 @@ fn evaluate(src: String, evaluation_context: Option<&Bound<'_, PyAny>>) -> PyRes
                 "Failed to parse expression '{src}': Invalid syntax or malformed string"
             ))
         })?
-        .map_err(|e| PyValueError::new_err(format!("Failed to compile expression '{src}': {e}")))?;
+        .map_err(|e| PyValueError::new_err(format!("Failed to parse expression '{src}': {e}")))?;
 
     // Add variables and functions if we have a context
     if evaluation_context.is_some() {
@@ -515,8 +515,8 @@ fn evaluate(src: String, evaluation_context: Option<&Bound<'_, PyAny>>) -> PyRes
 
     match result {
         Err(error) => {
-            warn!("An error occurred during execution");
-            warn!("Execution error: {error:?}");
+            debug!("An error occurred during execution");
+            debug!("Execution error: {error:?}");
             Err(map_execution_error_to_python(&error))
         }
 
diff --git a/tests/test_basics.py b/tests/test_basics.py
@@ -80,7 +80,12 @@ def test_expressions_with_context(expression_context_result):
     assert result == expected_result
 
 
+@pytest.mark.xfail(
+    reason="String indexing not supported in cel-interpreter 0.11.x - see test_upstream_improvements.py",
+    strict=True,
+)
 def test_str_context_expression():
+    """Test string indexing - currently not supported by cel-interpreter."""
     result = cel.evaluate("word[1]", {"word": "hello"})
     assert result == "e"
 
diff --git a/tests/test_cli.py b/tests/test_cli.py
@@ -133,6 +133,31 @@ def test_create_evaluator_empty_context(self):
         evaluator = CELEvaluator()
         assert evaluator.context == {}
 
+    def test_stdlib_functions_available(self):
+        """Test that stdlib functions are automatically available in CLI evaluator."""
+        evaluator = CELEvaluator()
+
+        # Test substring function from stdlib
+        result = evaluator.evaluate('substring("hello world", 0, 5)')
+        assert result == "hello"
+
+        result = evaluator.evaluate('substring("hello world", 6)')
+        assert result == "world"
+
+        result = evaluator.evaluate('substring("test", 1, 3)')
+        assert result == "es"
+
+    def test_stdlib_functions_with_context(self):
+        """Test that stdlib functions work alongside context variables."""
+        evaluator = CELEvaluator({"text": "hello world"})
+
+        # Use stdlib function with context variable
+        result = evaluator.evaluate("substring(text, 0, 5)")
+        assert result == "hello"
+
+        result = evaluator.evaluate("substring(text, 6)")
+        assert result == "world"
+
     def test_create_evaluator_with_context(self):
         """Test creating evaluator with initial context."""
         context = {"x": 10, "y": 20}
diff --git a/tests/test_parser_errors.py b/tests/test_parser_errors.py
@@ -122,6 +122,6 @@ def test_cli_passes_through_parser_errors(self):
         with pytest.raises(ValueError, match="Failed to parse expression"):
             evaluator.evaluate('"unclosed quote')
 
-        # This gives a clean compile error
-        with pytest.raises(ValueError, match="Failed to compile expression"):
+        # This also gives a clean parse error
+        with pytest.raises(ValueError, match="Failed to parse expression"):
             evaluator.evaluate("(1 + 2")
diff --git a/tests/test_stdlib.py b/tests/test_stdlib.py

Original file line number	Diff line number	Diff line change
`@@ -7,6 +7,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0`
`7`	`7`
`8`	`8`	`## [Unreleased]`
`9`	`9`
	`10`	`+- Minor removed warning level logging in cel crate`
`10`	`11`
`11`	`12`	`## [0.5.2] - 2025-09-12`
`12`	`13`