Add support for utf-8 chars

Closes #24

Example command:

python src/zkregex_fuzzer/cli.py fuzz \
    --oracle combined \
    --target noir \
    --invalid-input-generator mixed --valid-input-generator mixed \
    --fuzzer grammar --grammar-custom-grammar controlled_utf8 \
    --process-num 9 --save INVALID_SEED COMPILE_ERROR RUN_ERROR FAILED SUBSTR_MISMATCH --inputs-num 10 --regex-num 10

Author: StefanosChaliasos

src/zkregex_fuzzer/chars.py

@@ -1,4 +1,5 @@
 import string
+from dataclasses import dataclass
 
 
 def create_range(start_char: str, end_char: str) -> set[str]:
@@ -8,13 +9,99 @@ def create_range(start_char: str, end_char: str) -> set[str]:
     return {chr(i) for i in range(ord(start_char), ord(end_char) + 1)}
 
 
+# TODO singleton
 LATIN_EXT_CHARS = create_range("¡", "ƿ")
 GREEK_CHARS = create_range("Ͱ", "Ͽ")
 CYRILLIC_CHARS = create_range("Ѐ", "ӿ")
-ASCII_CHARS = set(string.printable)
-ALL_CHARS = ASCII_CHARS.union(LATIN_EXT_CHARS).union(GREEK_CHARS).union(CYRILLIC_CHARS)
-SUPPORTED_CHARS = ASCII_CHARS
+ASCII = set(string.printable)
+CONTROLLED_UTF8_CHARS = (
+    ASCII.union(LATIN_EXT_CHARS).union(GREEK_CHARS).union(CYRILLIC_CHARS)
+)
+UNCONTROLLED_UTF8_CHARS = {
+    chr(codepoint)
+    for codepoint in range(0x110000)
+    if not (0xD800 <= codepoint <= 0xDFFF)
+}
+
 # All supported characters and escape all the regex characters that need to be escaped
 ESCAPE_CHARS = ["\\", "^", "$", ".", "|", "?", "*", "+", "()", "[]", "{", "}"]
 ESCAPED_CHARS = [f"\\{c}" for c in ESCAPE_CHARS]
-SUPPORTED_ESCAPE_CHARS = ASCII_CHARS.difference(ESCAPE_CHARS).union(ESCAPED_CHARS)
+
+
+@dataclass
+class SupportedChars:
+    all_chars: set[str]
+    non_escaped_chars: set[str]
+    including_escaped_chars: set[str]
+
+
+ASCII_CHARS = SupportedChars(
+    all_chars=ASCII,
+    non_escaped_chars=ASCII.difference(ESCAPE_CHARS),
+    including_escaped_chars=ASCII.difference(ESCAPE_CHARS).union(ESCAPED_CHARS),
+)
+
+CONTROLLED_UTF8_CHARS = SupportedChars(
+    all_chars=CONTROLLED_UTF8_CHARS,
+    non_escaped_chars=CONTROLLED_UTF8_CHARS.difference(ESCAPE_CHARS),
+    including_escaped_chars=CONTROLLED_UTF8_CHARS.difference(ESCAPE_CHARS).union(
+        ESCAPED_CHARS
+    ),
+)
+
+UNCONTROLLED_UTF8_CHARS = SupportedChars(
+    all_chars=UNCONTROLLED_UTF8_CHARS,
+    non_escaped_chars=UNCONTROLLED_UTF8_CHARS.difference(ESCAPE_CHARS),
+    including_escaped_chars=UNCONTROLLED_UTF8_CHARS.difference(ESCAPE_CHARS).union(
+        ESCAPED_CHARS
+    ),
+)
+
+
+class SupportedCharsManager:
+    """Singleton for supported characters."""
+
+    _instance = None
+
+    def __new__(cls, char_set="ascii"):
+        if cls._instance is None:
+            cls._instance = super(SupportedCharsManager, cls).__new__(cls)
+            cls._instance.chars = None  # Initialize the attribute
+            cls._instance._set_chars(char_set)
+
+        return cls._instance
+
+    def _set_chars(self, char_set):
+        """Set the character set based on the provided name."""
+        if char_set == "ascii":
+            self.chars = ASCII_CHARS
+        elif char_set == "controlled_utf8":
+            self.chars = CONTROLLED_UTF8_CHARS
+        elif char_set == "uncontrolled_utf8":
+            self.chars = UNCONTROLLED_UTF8_CHARS
+        else:
+            raise ValueError(f"Invalid character set: {char_set}")
+
+    def get_chars(self):
+        """Get the supported characters."""
+        return self.chars
+
+    @classmethod
+    def override(cls, char_set):
+        """
+        Override the character set of the singleton instance.
+        If the instance doesn't exist, it will be created.
+
+        Args:
+            char_set: The name of the character set to use
+
+        Returns:
+            The singleton instance
+        """
+        # Create the instance if it doesn't exist
+        if cls._instance is None:
+            return cls(char_set)
+
+        # Override the existing instance's character set
+        cls._instance._set_chars(char_set)
+        return cls._instance

src/zkregex_fuzzer/cli.py

@@ -8,6 +8,7 @@
 import uuid
 from pathlib import Path
 
+from zkregex_fuzzer.chars import SupportedCharsManager
 from zkregex_fuzzer.configs import (
     DEFAULT_HARNESS_TIMEOUT,
     DEFAULT_INPUT_GEN_TIMEOUT,
@@ -178,6 +179,12 @@ def fuzz_parser():
         default=DEFAULT_HARNESS_TIMEOUT,
         help="Timeout for harness execution (default: 300).",
     )
+    parser.add_argument(
+        "--char-set",
+        choices=["ascii", "controlled_utf8", "uncontrolled_utf8"],
+        default="ascii",
+        help="The character set to use for the fuzzer (default: ascii).",
+    )
     return parser
 
 
@@ -283,6 +290,7 @@ def do_fuzz(args):
         logging_file=logging_file,
         output_path=args.save_output,
         save_options=args.save,
+        char_set=args.char_set,
     )
 
     # Use the new reporting function to print configuration
@@ -384,6 +392,15 @@ def main():
 
     logger.setLevel(args.logger_level)
 
+    # set supported chars singleton
+    if args.grammar_custom_grammar == "basic":
+        args.char_set = "ascii"
+    elif args.grammar_custom_grammar == "controlled_utf8":
+        args.char_set = "controlled_utf8"
+    elif args.grammar_custom_grammar == "uncontrolled_utf8":
+        args.char_set = "uncontrolled_utf8"
+    SupportedCharsManager(args.char_set)
+
     if args.subcommand == "fuzz":
         do_fuzz(args)
     elif args.subcommand == "reproduce":

src/zkregex_fuzzer/configs.py

@@ -1,4 +1,9 @@
-from zkregex_fuzzer.grammar import BASIC_REGEX_GRAMMAR, OLD_GRAMMAR
+from zkregex_fuzzer.grammar import (
+    BASIC_REGEX_GRAMMAR,
+    CONTROLLED_UTF8_GRAMMAR,
+    OLD_GRAMMAR,
+    UNCONTROLLED_UTF8_GRAMMAR,
+)
 from zkregex_fuzzer.invinpgen import (
     ComplementBasedGenerator,
     MutationBasedGenerator,
@@ -36,6 +41,8 @@
 GRAMMARS = {
     "basic": BASIC_REGEX_GRAMMAR,
     "old": OLD_GRAMMAR,
+    "controlled_utf8": CONTROLLED_UTF8_GRAMMAR,
+    "uncontrolled_utf8": UNCONTROLLED_UTF8_GRAMMAR,
 }
 
 VALID_INPUT_GENERATORS = {

src/zkregex_fuzzer/dfa.py

@@ -13,23 +13,14 @@
 from automata.fa.gnfa import GNFA
 from automata.fa.nfa import NFA
 
-from zkregex_fuzzer.chars import SUPPORTED_CHARS
-
-
-def get_supported_symbols() -> set[str]:
-    """
-    Get the set of symbols that are supported by the regex engine.
-    """
-    # TODO make this configurable
-    # Symbols should include at least all ASCII characters
-    return SUPPORTED_CHARS
+from zkregex_fuzzer.chars import SupportedCharsManager
 
 
 def regex_to_nfa(regex: str) -> NFA:
     """
     Convert a regex to an NFA.
     """
-    symbols = get_supported_symbols()
+    symbols = SupportedCharsManager().get_chars().all_chars
     regex = unwrap_regex(regex)
 
     try:
@@ -370,7 +361,9 @@ def dfa_string_matching(
     # TODO make this configurable
     max_length = 500
     # Convert regex to NFA
-    nfa = NFA.from_regex(regex, input_symbols=get_supported_symbols())
+    nfa = NFA.from_regex(
+        regex, input_symbols=SupportedCharsManager().get_chars().all_chars
+    )
 
     # Start with the initial state and an empty string
     current_states = nfa._get_lambda_closures()[nfa.initial_state]

src/zkregex_fuzzer/fuzzer.py

@@ -52,6 +52,10 @@ def fuzz_with_grammar(
 
     if target_grammar == "basic":
         grammar = GRAMMARS[target_grammar]
+    elif target_grammar == "controlled_utf8":
+        grammar = GRAMMARS[target_grammar]
+    elif target_grammar == "uncontrolled_utf8":
+        grammar = GRAMMARS[target_grammar]
     elif target_grammar.endswith(".py"):
         try:
             # Get absolute path
@@ -277,7 +281,7 @@ def _process_results(regex, result):
                             pbar.update(1)
                 except concurrent.futures.TimeoutError:
                     logger.error(
-                        f"Timeout after {timeout_per_regex*len(params)}s processing regexes"
+                        f"Timeout after {timeout_per_regex * len(params)}s processing regexes"
                     )
                     # Cancel any remaining futures before exiting the context
                     for future in futures_to_regex:

src/zkregex_fuzzer/grammar.py

@@ -20,11 +20,14 @@
  - Add more grammars.
 """
 
+import copy
 import string
 from typing import List
 
 from fuzzingbook.Grammars import Expansion, Grammar
 
+from zkregex_fuzzer.chars import CONTROLLED_UTF8_CHARS, UNCONTROLLED_UTF8_CHARS
+
 
 def srange(characters: str) -> List[Expansion]:
     """Return a list of single-character expansions from the given string."""
@@ -136,6 +139,28 @@ def crange(start: str, end: str) -> List[Expansion]:
     "<ESCAPED>": [f"\\{c}" for c in "\\^$.|?*+()[]{}`-&"],
 }
 
+CONTROLLED_UTF8_GRAMMAR: Grammar = copy.deepcopy(BASIC_REGEX_GRAMMAR)
+CONTROLLED_UTF8_GRAMMAR["<CHAR>"] = [
+    "<LETTER>",
+    "<DIGIT>",
+    "<SYMBOL>",
+    "<ESCAPED>",
+    "<UTF8_CHAR>",
+]
+CONTROLLED_UTF8_GRAMMAR["<UTF8_CHAR>"] = list(CONTROLLED_UTF8_CHARS.non_escaped_chars)
+
+UNCONTROLLED_UTF8_GRAMMAR: Grammar = copy.deepcopy(BASIC_REGEX_GRAMMAR)
+UNCONTROLLED_UTF8_GRAMMAR["<CHAR>"] = [
+    "<LETTER>",
+    "<DIGIT>",
+    "<SYMBOL>",
+    "<ESCAPED>",
+    "<UTF8_CHAR>",
+]
+UNCONTROLLED_UTF8_GRAMMAR["<UTF8_CHAR>"] = list(
+    UNCONTROLLED_UTF8_CHARS.non_escaped_chars
+)
+
 OLD_GRAMMAR: Grammar = {
     # Entry point
     "<start>": ["<REGEX>"],

src/zkregex_fuzzer/invinpgen.py

@@ -11,7 +11,7 @@
 
 import exrex
 
-from zkregex_fuzzer.chars import SUPPORTED_CHARS
+from zkregex_fuzzer.chars import SupportedCharsManager
 from zkregex_fuzzer.dfa import regex_to_nfa
 from zkregex_fuzzer.logger import logger
 from zkregex_fuzzer.utils import check_if_string_is_valid, extract_parts, pretty_regex
@@ -170,7 +170,11 @@ def _mutate_input(self, valid_input: str) -> str:
                 if should_mutate:
                     # Note that we can still mutate to a valid character
                     invalid_input[i] = random.choice(
-                        list(SUPPORTED_CHARS.difference({invalid_input[i]}))
+                        list(
+                            SupportedCharsManager()
+                            .get_chars()
+                            .including_escaped_chars.difference({invalid_input[i]})
+                        )
                     )
                     if (
                         not check_if_string_is_valid(self.regex, "".join(invalid_input))

src/zkregex_fuzzer/report.py

@@ -30,6 +30,7 @@ class Configuration:
     logging_file: str | None
     output_path: str
     save_options: list[str]
+    char_set: str
 
 
 class Stats:
@@ -181,6 +182,7 @@ def get_fuzzing_configuration_string(configuration: Configuration):
 Logging file: {configuration.logging_file}
 Output path: {configuration.output_path}
 Save options: {configuration.save_options}
+Char set: {configuration.char_set}
 """
 
 
@@ -251,6 +253,7 @@ def print_fuzzing_configuration(configuration: Configuration):
             f"📥 Inputs num: {configuration.inputs_num}",
             f"🔍 Max non-terminals: {configuration.grammar_max_non_terminals}",
             f"🔍 Custom grammar: {configuration.grammar_custom_grammar}",
+            f"🔍 Char set: {configuration.char_set}",
             f"🌱 Seed: {configuration.seed}",
             f"🔄 Num process: {configuration.num_process}",
             f"🔍 Logging file: {os.path.relpath(configuration.logging_file, os.getcwd()) if configuration.logging_file else 'None'}",