Implement Logging of Terminal Output to a .log file. (#180)

* Implemented logging of terminal output to file.

* Set default config values for log_to_file, log_dir

* Change default log_dir to ./logs

[skip ci]

* Bring log-file up to date with Master.

* Re-implement logging of Terminal output to log file.

* Move log _writing_ statements to separate function.

* Add a test for logging.

* Added test to check if logs generated are same.

* Added model log file.

* Test on this branch

* Change Regex expression for windows.

* Print log on failure

* Print log windows.

* Edit regex expression for both unix and windows.

* Edit regex expression.

* Edit expected log.

* Try stripping whitespace.

* Move test_logging to a separate file.

* Print log for all OS's

* Make rules for passing looser.

* Fix a typo

* Split string before cleaning.

* Remove duplicate test_logging file.

* Move default log directory into media directory.

* Ignored Line Numbers during test.

Added some documentation for why word-for-word comparisons are not used.

* Allow changing of Max width and height of logs via config files.

* Implemented word-for-word tests for log file.

(Except for data that _will_ change between tests)

* Formatted as per black's wishes.

* Add cfg subcmd case for log_width and height.
Modified regex to only ignore numbers and paths.

* Make regex remove list indicators as well.

* Make regex remove timestamps.

* Added a comment about the monstrous regex.

Co-authored-by: Aathish Sivasubrahmanian <[email protected]>

Author: Aathish04

manim/default.cfg

@@ -48,6 +48,9 @@ output_file =
 # --leave_progress_bars
 leave_progress_bars = False
 
+# --log_to_file
+log_to_file = False
+
 # -c, --color
 background_color = BLACK
 
@@ -67,6 +70,9 @@ upto_animation_number = -1
 # --media_dir
 media_dir = ./media
 
+# --log_dir
+log_dir = %(media_dir)s/logs
+
 # # --video_dir
 # video_dir = %(MEDIA_DIR)s/videos
 
@@ -153,3 +159,5 @@ log_level =
 log_time = cyan dim
 log_message = 
 log_path = dim
+log_width = -1
+log_height = -1

manim/logger.py

@@ -21,28 +21,39 @@ def parse_theme(fp):
     config_parser.read(fp)
     theme = dict(config_parser["logger"])
     # replaces `_` by `.` as rich understands it
-    for key in theme:
-        temp = theme[key]
-        del theme[key]
-        key = key.replace("_", ".")
-        theme[key] = temp
+    theme = dict(
+        zip([key.replace("_", ".") for key in theme.keys()], list(theme.values()))
+    )
+
+    theme["log.width"] = None if theme["log.width"] == "-1" else int(theme["log.width"])
+
+    theme["log.height"] = (
+        None if theme["log.height"] == "-1" else int(theme["log.height"])
+    )
     try:
-        customTheme = Theme(theme)
+        customTheme = Theme(
+            {k: v for k, v in theme.items() if k not in ["log.width", "log.height"]}
+        )
     except (color.ColorParseError, errors.StyleSyntaxError):
         customTheme = None
         printf(
             "[logging.level.error]It seems your colour configuration couldn't be parsed. Loading the default color configuration...[/logging.level.error]"
         )
-    return customTheme
+    return customTheme, theme
 
 
 config_items = _run_config()
 config_parser, successfully_read_files = config_items[1], config_items[-1]
 try:
-    customTheme = parse_theme(successfully_read_files)
-    console = Console(theme=customTheme)
+    customTheme, themedict = parse_theme(successfully_read_files)
+    console = Console(
+        theme=customTheme,
+        record=True,
+        height=themedict["log.height"],
+        width=themedict["log.width"],
+    )
 except KeyError:
-    console = Console()
+    console = Console(record=True)
     printf(
         "[logging.level.warning]No cfg file found, creating one in "
         + successfully_read_files[0]

manim/scene/scene_file_writer.py

@@ -10,7 +10,7 @@
 
 from ..constants import FFMPEG_BIN, GIF_FILE_EXTENSION
 from ..config import file_writer_config
-from ..logger import logger
+from ..logger import logger, console
 from ..utils.config_ops import digest_config
 from ..utils.file_ops import guarantee_existence
 from ..utils.file_ops import add_extension_if_not_present
@@ -533,3 +533,13 @@ def print_file_ready_message(self, file_path):
         Prints the "File Ready" message to STDOUT.
         """
         logger.info("\nFile ready at {}\n".format(file_path))
+
+        if file_writer_config["log_to_file"]:
+            self.write_log()
+
+    def write_log(self):
+        log_file_path = os.path.join(
+            file_writer_config["log_dir"], f"{self.get_default_scene_name()}.log"
+        )
+        console.save_text(log_file_path)
+        logger.info("Log written to {}\n".format(log_file_path))

manim/utils/cfg_subcmds.py

@@ -93,15 +93,24 @@ def write(level=None, openfile=False):
                 console.print(RICH_COLOUR_INSTRUCTIONS)
                 default = replace_keys(default)
                 for key in default:
-                    console.print(f"Enter the style for {key}:", style=key, end="")
+                    desc = (
+                        "style" if key not in ["log.width", "log.height"] else "value"
+                    )
+                    style = key if key not in ["log.width", "log.height"] else None
+                    cond = (
+                        is_valid_style
+                        if key not in ["log.width", "log.height"]
+                        else lambda m: m.isdigit()
+                    )
+                    console.print(f"Enter the {desc} for {key}:", style=style, end="")
                     temp = input()
                     if temp:
-                        while not is_valid_style(temp):
+                        while not cond(temp):
                             console.print(
-                                "[red bold]Invalid style. Try again.[/red bold]"
+                                f"[red bold]Invalid {desc}. Try again.[/red bold]"
                             )
                             console.print(
-                                f"Enter the style for {key}:", style=key, end=""
+                                f"Enter the {desc} for {key}:", style=style, end=""
                             )
                             temp = input()
                         else:
@@ -162,7 +171,7 @@ def show():
     for category in current_config:
         console.print(f"{category}", style="bold green underline")
         for entry in current_config[category]:
-            if category == "logger":
+            if category == "logger" and entry not in ["log_width", "log_height"]:
                 console.print(f"{entry} :", end="")
                 console.print(
                     f" {current_config[category][entry]}",

manim/utils/config_utils.py

@@ -60,13 +60,14 @@ def _parse_file_writer_config(config_parser, args):
         "save_pngs",
         "save_as_gif",
         "write_all",
+        "log_to_file",
     ]:
         attr = getattr(args, boolean_opt)
         fw_config[boolean_opt] = (
             default.getboolean(boolean_opt) if attr is None else attr
         )
     # for str_opt in ['media_dir', 'video_dir', 'tex_dir', 'text_dir']:
-    for str_opt in ["media_dir"]:
+    for str_opt in ["media_dir", "log_dir"]:
         attr = getattr(args, str_opt)
         fw_config[str_opt] = os.path.relpath(default[str_opt]) if attr is None else attr
     dir_names = {"video_dir": "videos", "tex_dir": "Tex", "text_dir": "texts"}
@@ -255,6 +256,13 @@ def _parse_cli(arg_list, input=True):
         help="Save the video as gif",
     )
 
+    parser.add_argument(
+        "--log_to_file",
+        action="store_const",
+        const=True,
+        help="Log terminal output to file.",
+    )
+
     # The default value of the following is set in manim.cfg
     parser.add_argument(
         "-c", "--color", help="Background color",
@@ -265,6 +273,11 @@ def _parse_cli(arg_list, input=True):
     parser.add_argument(
         "--media_dir", help="directory to write media",
     )
+
+    parser.add_argument(
+        "--log_dir", help="directory to write log files to",
+    )
+
     # video_group = parser.add_mutually_exclusive_group()
     # video_group.add_argument(
     #     "--video_dir",
@@ -370,9 +383,14 @@ def _init_dirs(config):
         config["video_dir"],
         config["tex_dir"],
         config["text_dir"],
+        config["log_dir"],
     ]:
         if not os.path.exists(folder):
-            os.makedirs(folder)
+            # If log_to_file is False, ignore log_dir
+            if folder is config["log_dir"] and (not config["log_to_file"]):
+                pass
+            else:
+                os.makedirs(folder)
 
 
 def _from_command_line():

tests/test_cli/test_cfg_subcmd.py

@@ -20,7 +20,7 @@ def test_cfg_show(python_version):
     """Test if the `manim cfg show` command works as intended."""
     command = f"cd {this_folder} && {python_version} -m manim cfg show"
     out, err, exitcode = capture(command, use_shell=True)
-    assert exitcode == 0
+    assert exitcode == 0, err
     assert f"{os.path.sep}tests{os.path.sep}".encode("utf-8") in out, err
 
 

tests/test_cli/write_cfg_sbcmd_input.txt

@@ -24,3 +24,6 @@ False
 
 
 
+
+
+

tests/test_logging/expected.txt

@@ -0,0 +1,4 @@
+           INFO     Read configuration files:                                                                                                                                                                                                       config.py:  
+           INFO                                                                                                                                                                                                                         scene_file_writer.py:   
+                    File ready at                                                                                                                                                                                                                               
+                                                                                                                                                                                                                                                                

tests/test_logging/test_logging.py

@@ -0,0 +1,58 @@
+import subprocess
+import os
+import sys
+from shutil import rmtree
+import pytest
+import re
+
+
+def capture(command, instream=None):
+    proc = subprocess.Popen(
+        command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=instream
+    )
+    out, err = proc.communicate()
+    return out, err, proc.returncode
+
+
+def test_logging_to_file(python_version):
+    """Test logging Terminal output to a log file.
+    As some data will differ with each log (the timestamps, file paths, line nums etc)
+    a regex substitution has been employed to replace the strings that may change with
+    whitespace.
+    """
+    path_basic_scene = os.path.join("tests", "tests_data", "basic_scenes.py")
+    path_output = os.path.join("tests_cache", "media_temp")
+    command = [
+        python_version,
+        "-m",
+        "manim",
+        path_basic_scene,
+        "SquareToCircle",
+        "-l",
+        "--log_to_file",
+        "--log_dir",
+        os.path.join(path_output, "logs"),
+        "--media_dir",
+        path_output,
+    ]
+    out, err, exitcode = capture(command)
+    log_file_path = os.path.join(path_output, "logs", "SquareToCircle.log")
+    assert exitcode == 0, err
+    assert os.path.exists(log_file_path), err
+    if sys.platform.startswith("win32") or sys.platform.startswith("cygwin"):
+        enc = "Windows-1252"
+    else:
+        enc = "utf-8"
+    with open(log_file_path, encoding=enc) as logfile:
+        logs = logfile.read()
+    # The following regex pattern selects timestamps, file paths and all numbers..
+    pattern = r"(\[?\d+:?]?)|(\['[A-Z]?:?[\/\\].*cfg'])|([A-Z]?:?[\/\\].*mp4)"
+
+    logs = re.sub(pattern, lambda m: " " * len((m.group(0))), logs)
+    with open(
+        os.path.join(os.path.dirname(__file__), "expected.txt"), "r"
+    ) as expectedfile:
+        expected = re.sub(
+            pattern, lambda m: " " * len((m.group(0))), expectedfile.read()
+        )
+    assert logs == expected, logs

tests/tests_data/manim.cfg

@@ -4,3 +4,6 @@ write_to_movie = True
 # write_all = False
 save_last_frame = False
 # save_pngs = False
+
+[logger]
+log_width = 256