From 384b84a8f67538565f7605f57a3df0074eb65941 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Thu, 25 Jun 2020 16:46:37 +0200 Subject: [PATCH 01/47] added caching functionnality --- manim/scene/scene.py | 47 ++++++++++++++++++++------ manim/scene/scene_file_writer.py | 57 +++++++++++++++++++++----------- manim/utils/hashing.py | 44 ++++++++++++++++++++++++ 3 files changed, 118 insertions(+), 30 deletions(-) create mode 100644 manim/utils/hashing.py diff --git a/manim/scene/scene.py b/manim/scene/scene.py index ce24579c48..585415b513 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -15,6 +15,7 @@ from ..mobject.mobject import Mobject from ..scene.scene_file_writer import SceneFileWriter from ..utils.iterables import list_update +from ..utils.hashing import get_hash_from_play_call class Scene(Container): @@ -52,15 +53,15 @@ def construct(self): "start_at_animation_number": None, "end_at_animation_number": None, "leave_progress_bars": False, - } - + "enable_caching": True, + } def __init__(self, **kwargs): Container.__init__(self, **kwargs) self.camera = self.camera_class(**self.camera_config) self.file_writer = SceneFileWriter( self, **self.file_writer_config, ) - + self.play_hashes_list = [] self.mobjects = [] # TODO, remove need for foreground mobjects self.foreground_mobjects = [] @@ -381,6 +382,17 @@ def add_mobjects_among(self, values): )) return self + def add_mobjects_from_animations(self, animations): + + curr_mobjects = self.get_mobject_family_members() + for animation in animations: + # Anything animated that's not already in the + # scene gets added to the scene + mob = animation.mobject + if mob not in curr_mobjects: + self.add(mob) + curr_mobjects += mob.get_family() + def remove(self, *mobjects): """ Removes mobjects in the passed list of mobjects @@ -832,6 +844,24 @@ def update_skipping_status(self): self.skip_animations = True raise EndSceneEarlyException() + def handle_caching(func): + def wrapper(self, *args, **kwargs): + animations = self.compile_play_args_to_animation_list( + *args, **kwargs + ) + # We have to add all the mobjects, because we can have hash-collisions if not. + self.add_mobjects_from_animations(animations) + mobjects_on_scene = self.get_mobjects() + hash_play = get_hash_from_play_call(animations, mobjects_on_scene) + self.play_hashes_list.append(hash_play) + if self.enable_caching and self.file_writer.is_already_cached(hash_play): + logger.info(f'Animation {self.num_plays} : Using cached data (hash : {hash_play}') + self.skip_animations = True + else: + self.revert_to_original_skipping_status() + func(self, *args, **kwargs) + return wrapper + def handle_play_like_call(func): """ This method is used internally to wrap the @@ -873,16 +903,9 @@ def begin_animations(self, animations): List of involved animations. """ - curr_mobjects = self.get_mobject_family_members() for animation in animations: # Begin animation animation.begin() - # Anything animated that's not already in the - # scene gets added to the scene - mob = animation.mobject - if mob not in curr_mobjects: - self.add(mob) - curr_mobjects += mob.get_family() def progress_through_animations(self, animations): """ @@ -933,6 +956,7 @@ def finish_animations(self, animations): else: self.update_mobjects(0) + @handle_caching @handle_play_like_call def play(self, *args, **kwargs): """ @@ -1035,7 +1059,8 @@ def get_wait_time_progression(self, duration, stop_condition): "Waiting {}".format(self.num_plays) ) return time_progression - + + @handle_caching @handle_play_like_call def wait(self, duration=DEFAULT_WAIT_TIME, stop_condition=None): """ diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index a7b1b2f4b8..e359a38ae4 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -193,10 +193,17 @@ def get_next_partial_movie_path(self): str The path of the next partial movie. """ + # result = os.path.join( + # self.partial_movie_directory, + # "{:05}{}".format( + # self.scene.num_plays, + # self.movie_file_extension, + # ) + # ) result = os.path.join( self.partial_movie_directory, - "{:05}{}".format( - self.scene.num_plays, + "{}{}".format( + self.scene.play_hashes_list[self.scene.num_plays], self.movie_file_extension, ) ) @@ -439,6 +446,15 @@ def close_movie_pipe(self): self.temp_partial_movie_file_path, self.partial_movie_file_path, ) + logger.debug(f"Animation {self.scene.num_plays} : Partial movie file written in {self.partial_movie_file_path}") + + def cancel_animation(self): + pass + def is_already_cached(self, hash_play): + #SI le fichier caché existe, alors oui + #Sinon, nique ta mère et fais un rendu de ton fichier enculé + path = os.path.join(self.partial_movie_directory, "{}{}".format(hash_play, self.movie_file_extension)) + return os.path.exists(path) def combine_movie_files(self): """ @@ -453,35 +469,38 @@ def combine_movie_files(self): # cuts at all the places you might want. But for viewing # the scene as a whole, one of course wants to see it as a # single piece. - kwargs = { - "remove_non_integer_files": True, - "extension": self.movie_file_extension, - } - if self.scene.start_at_animation_number is not None: - kwargs["min_index"] = self.scene.start_at_animation_number - if self.scene.end_at_animation_number is not None: - kwargs["max_index"] = self.scene.end_at_animation_number - else: - kwargs["remove_indices_greater_than"] = self.scene.num_plays - 1 - partial_movie_files = get_sorted_integer_files( - self.partial_movie_directory, - **kwargs - ) + + # kwargs = { + # "remove_non_integer_files": False, #TODO remove this shit/or change it + # "extension": self.movie_file_extension, + # } + # if self.scene.start_at_animation_number is not None: + # kwargs["min_index"] = self.scene.start_at_animation_number #TODO remove this shit + # if self.scene.end_at_animation_number is not None: + # kwargs["max_index"] = self.scene.end_at_animation_number + # else: + # kwargs["remove_indices_greater_than"] = self.scene.num_plays - 1 + # partial_movie_files = get_sorted_integer_files( #TODO REMOVE THIS BAG OF SHIT + # self.partial_movie_directory, + # **kwargs + # ) + + partial_movie_files = [os.path.join(self.partial_movie_directory, "{}{}".format(hash_play, self.movie_file_extension)) for hash_play in self.scene.play_hashes_list] if len(partial_movie_files) == 0: logger.error("No animations in this scene") return - # Write a file partial_file_list.txt containing all - # partial movie files + # partial movie files. This is used for FFMPEG. file_list = os.path.join( self.partial_movie_directory, "partial_movie_file_list.txt" ) with open(file_list, 'w') as fp: + fp.write("# This file is used internally by FFMPEG.\n") for pf_path in partial_movie_files: if os.name == 'nt': pf_path = pf_path.replace('\\', '/') - fp.write("file \'file:{}\'\n".format(pf_path)) + fp.write("file \'file:{}'\n".format(pf_path)) movie_file_path = self.get_movie_file_path() commands = [ diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py new file mode 100644 index 0000000000..db07c94a74 --- /dev/null +++ b/manim/utils/hashing.py @@ -0,0 +1,44 @@ +import json +import zlib +import inspect +import numpy as np + + +class CustomEncoder(json.JSONEncoder): + def default(self, o): + ''' + Convert objects unrecognized by the default encoder + + Parameters + ---------- + obj : any + Arbitrary object to convert + + Returns + ------- + any + Python object that JSON encoder will recognize + + ''' + if callable(o): + return inspect.getsource(o) # Improvement to do ? + elif isinstance(o, np.ndarray): + return list(o) + elif hasattr(o, "__dict__"): + return getattr(o, '__dict__') + return json.JSONEncoder.default(self, o) + + +def get_json(obj): + return json.dumps(obj, cls=CustomEncoder,) + + +def get_hash_from_play_call(animations_list, current_mobjects_list): + animations_list_json = [get_json(x).encode() for x in sorted( + animations_list, key=lambda obj: str(obj))] + current_mobjects_list_json = [get_json(x).encode() for x in sorted( + current_mobjects_list, key=lambda obj: str(obj))] + hash_animations = zlib.crc32(repr(animations_list_json).encode()) + hash_current_mobjects = zlib.crc32( + repr(current_mobjects_list_json).encode()) + return "{}_{}".format(hash_animations, hash_current_mobjects) From deeade57f0719e7df2ce0ae15262e0cb0d47f4fe Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Thu, 25 Jun 2020 16:47:02 +0200 Subject: [PATCH 02/47] deleted comments --- manim/scene/scene_file_writer.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index e359a38ae4..c39e9efdd8 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -193,13 +193,6 @@ def get_next_partial_movie_path(self): str The path of the next partial movie. """ - # result = os.path.join( - # self.partial_movie_directory, - # "{:05}{}".format( - # self.scene.num_plays, - # self.movie_file_extension, - # ) - # ) result = os.path.join( self.partial_movie_directory, "{}{}".format( @@ -448,8 +441,6 @@ def close_movie_pipe(self): ) logger.debug(f"Animation {self.scene.num_plays} : Partial movie file written in {self.partial_movie_file_path}") - def cancel_animation(self): - pass def is_already_cached(self, hash_play): #SI le fichier caché existe, alors oui #Sinon, nique ta mère et fais un rendu de ton fichier enculé @@ -490,7 +481,7 @@ def combine_movie_files(self): logger.error("No animations in this scene") return # Write a file partial_file_list.txt containing all - # partial movie files. This is used for FFMPEG. + # partial movie files. This is used by FFMPEG. file_list = os.path.join( self.partial_movie_directory, "partial_movie_file_list.txt" From 04d8c45b5559dd329562f470591165e3dc98f95f Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Fri, 26 Jun 2020 17:44:19 +0200 Subject: [PATCH 03/47] fixed wait bug --- manim/scene/scene.py | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/manim/scene/scene.py b/manim/scene/scene.py index 585415b513..bcd18966ac 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -15,7 +15,7 @@ from ..mobject.mobject import Mobject from ..scene.scene_file_writer import SceneFileWriter from ..utils.iterables import list_update -from ..utils.hashing import get_hash_from_play_call +from ..utils.hashing import get_hash_from_play_call, get_hash_from_wait_call class Scene(Container): @@ -801,7 +801,6 @@ def compile_method(state): state["last_method"] = state["curr_method"] state["curr_method"] = None state["method_args"] = [] - for arg in args: if isinstance(arg, Animation): compile_method(state) @@ -844,24 +843,36 @@ def update_skipping_status(self): self.skip_animations = True raise EndSceneEarlyException() - def handle_caching(func): + def handle_caching_play(func): def wrapper(self, *args, **kwargs): animations = self.compile_play_args_to_animation_list( *args, **kwargs ) # We have to add all the mobjects, because we can have hash-collisions if not. self.add_mobjects_from_animations(animations) - mobjects_on_scene = self.get_mobjects() + mobjects_on_scene = self.get_mobjects() hash_play = get_hash_from_play_call(animations, mobjects_on_scene) self.play_hashes_list.append(hash_play) if self.enable_caching and self.file_writer.is_already_cached(hash_play): - logger.info(f'Animation {self.num_plays} : Using cached data (hash : {hash_play}') + logger.info(f'Animation {self.num_plays} : Using cached data (hash : {hash_play})') self.skip_animations = True else: self.revert_to_original_skipping_status() func(self, *args, **kwargs) return wrapper + def handle_caching_wait(func): + def wrapper(self, duration=DEFAULT_WAIT_TIME, stop_condition=None): + hash_wait = get_hash_from_wait_call(duration, stop_condition, self.get_mobjects()) + self.play_hashes_list .append(hash_wait) + if self.enable_caching and self.file_writer.is_already_cached(hash_wait): + logger.info(f'Wait {self.num_plays} : Using cached data (hash : {hash_wait})') + self.skip_animations = True + else : + self.revert_to_original_skipping_status() + func(self, duration, stop_condition) + return wrapper + def handle_play_like_call(func): """ This method is used internally to wrap the @@ -956,7 +967,7 @@ def finish_animations(self, animations): else: self.update_mobjects(0) - @handle_caching + @handle_caching_play @handle_play_like_call def play(self, *args, **kwargs): """ @@ -1060,7 +1071,7 @@ def get_wait_time_progression(self, duration, stop_condition): ) return time_progression - @handle_caching + @handle_caching_wait @handle_play_like_call def wait(self, duration=DEFAULT_WAIT_TIME, stop_condition=None): """ From 555b304ca3470105cb05721f2c77ffb9a9640d5d Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Fri, 26 Jun 2020 20:19:03 +0200 Subject: [PATCH 04/47] added docstrings --- manim/scene/scene.py | 25 ++++++++- manim/scene/scene_file_writer.py | 24 ++++++--- manim/utils/hashing.py | 93 ++++++++++++++++++++++++++------ 3 files changed, 117 insertions(+), 25 deletions(-) diff --git a/manim/scene/scene.py b/manim/scene/scene.py index bcd18966ac..471b6810fa 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -844,11 +844,22 @@ def update_skipping_status(self): raise EndSceneEarlyException() def handle_caching_play(func): + """ + This method is used internally to wrap the passed function into a function that will compute the hash of the play invokation, + and will act accordingly : either skip the animation because already cached, either nothing and let the play invokation be processed normally. + + Parameters + ---------- + *args : + Animation or mobject with mobject method and params + + **kwargs : + named parameters affecting what was passed in *args e.g run_time, lag_ratio etc. + """ def wrapper(self, *args, **kwargs): animations = self.compile_play_args_to_animation_list( *args, **kwargs ) - # We have to add all the mobjects, because we can have hash-collisions if not. self.add_mobjects_from_animations(animations) mobjects_on_scene = self.get_mobjects() hash_play = get_hash_from_play_call(animations, mobjects_on_scene) @@ -862,6 +873,18 @@ def wrapper(self, *args, **kwargs): return wrapper def handle_caching_wait(func): + """ + This method is used internally to wrap the passed function into a function that will compute the hash of the wait invokation, + and will act accordingly : either skip the animation because already cached or nothing and let the play invokation be processed normally + + Parameters + ---------- + *args : + Animation or mobject with mobject method and params + + **kwargs : + named parameters affecting what was passed in *args e.g run_time, lag_ratio etc. + """ def wrapper(self, duration=DEFAULT_WAIT_TIME, stop_condition=None): hash_wait = get_hash_from_wait_call(duration, stop_condition, self.get_mobjects()) self.play_hashes_list .append(hash_wait) diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index c39e9efdd8..a508ed9961 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -441,10 +441,20 @@ def close_movie_pipe(self): ) logger.debug(f"Animation {self.scene.num_plays} : Partial movie file written in {self.partial_movie_file_path}") - def is_already_cached(self, hash_play): - #SI le fichier caché existe, alors oui - #Sinon, nique ta mère et fais un rendu de ton fichier enculé - path = os.path.join(self.partial_movie_directory, "{}{}".format(hash_play, self.movie_file_extension)) + def is_already_cached(self, hash_invokation): + """Will check if a file named with `hash_play` exists. + + Parameters + ---------- + hash_play : :class:`str` + The hash corresponding to an invokation to either `scene.play` or `scene.wait`. + + Returns + ------- + `bool` + Wether the file exists. + """ + path = os.path.join(self.partial_movie_directory, "{}{}".format(hash_invokation, self.movie_file_extension)) return os.path.exists(path) def combine_movie_files(self): @@ -462,16 +472,16 @@ def combine_movie_files(self): # single piece. # kwargs = { - # "remove_non_integer_files": False, #TODO remove this shit/or change it + # "remove_non_integer_files": False, # "extension": self.movie_file_extension, # } # if self.scene.start_at_animation_number is not None: - # kwargs["min_index"] = self.scene.start_at_animation_number #TODO remove this shit + # kwargs["min_index"] = self.scene.start_at_animation_number # if self.scene.end_at_animation_number is not None: # kwargs["max_index"] = self.scene.end_at_animation_number # else: # kwargs["remove_indices_greater_than"] = self.scene.num_plays - 1 - # partial_movie_files = get_sorted_integer_files( #TODO REMOVE THIS BAG OF SHIT + # partial_movie_files = get_sorted_integer_files( # self.partial_movie_directory, # **kwargs # ) diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py index db07c94a74..57bc06966e 100644 --- a/manim/utils/hashing.py +++ b/manim/utils/hashing.py @@ -5,35 +5,66 @@ class CustomEncoder(json.JSONEncoder): - def default(self, o): - ''' - Convert objects unrecognized by the default encoder + def default(self, obj): + """ + This method is used to encode objects contained in some dict in an object that JSON encoder will recognize. + If obj is a function, then it will return the code of this function. + if obj is a np.darray, it converts it into a list. + if obj is an object with __dict__ attribute, it returns its __dict__. + Else, will let the JSONEncoder do the stuff, and throw an error if the type is not suitable for JSONEncoder. Parameters ---------- - obj : any - Arbitrary object to convert + obj : `any` + Arbitrary object to convert Returns ------- - any - Python object that JSON encoder will recognize + `any` + Python object that JSON encoder will recognize - ''' - if callable(o): - return inspect.getsource(o) # Improvement to do ? - elif isinstance(o, np.ndarray): - return list(o) - elif hasattr(o, "__dict__"): - return getattr(o, '__dict__') - return json.JSONEncoder.default(self, o) + """ + if callable(obj): + return inspect.getsource(obj) # Improvement to do ? + elif isinstance(obj, np.ndarray): + return list(obj) + elif hasattr(obj, "__dict__"): + return getattr(obj, '__dict__') + return json.JSONEncoder.default(self, obj) -def get_json(obj): - return json.dumps(obj, cls=CustomEncoder,) +def get_json(dict_config): + """Flatten a dictionnary of objects by tranforming these objects into their __dict__ or another type defined in the CustomEncoder class + + Paramaters + ---------- + dict_config : :class:`dict` + The dict to flatten + + Returns + ------- + `str` + the dict flattened + """ + return json.dumps(dict_config, cls=CustomEncoder) def get_hash_from_play_call(animations_list, current_mobjects_list): + """Take the list of animations and a list of mobjects and output their hash. Is meant to be used for `scene.play` function. + + Parameters + ----------- + animations_list : :class:`list` + The list of animations + + current_mobjects_list : :class:`list` + The list of mobjects. + + Returns + ------- + `str` + concatenation of the hash of animations_list and current_mobjects_list separated by '_'. + """ animations_list_json = [get_json(x).encode() for x in sorted( animations_list, key=lambda obj: str(obj))] current_mobjects_list_json = [get_json(x).encode() for x in sorted( @@ -42,3 +73,31 @@ def get_hash_from_play_call(animations_list, current_mobjects_list): hash_current_mobjects = zlib.crc32( repr(current_mobjects_list_json).encode()) return "{}_{}".format(hash_animations, hash_current_mobjects) + + +def get_hash_from_wait_call(wait_time, stop_condition_function, current_mobjects_list): + """Take a wait time, a boolean function as stop_condition and a list of mobjects output their hash. Is meant to be used for `scene.wait` function. + + Parameters + ----------- + wait_time : :class:`int` + The time to wait + + stop_condition_function : :class:`func` + Boolean function used as a stop_condition in `wait`. + + Returns + ------- + `str` + concatenation of the hash of animations_list and current_mobjects_list separated by '_'. + """ + current_mobjects_list_json = [get_json(x).encode() for x in sorted( + current_mobjects_list, key=lambda obj: str(obj))] + hash_current_mobjects = zlib.crc32( + repr(current_mobjects_list_json).encode()) + if stop_condition_function != None: + hash_function = zlib.crc32(inspect.getsource( + stop_condition_function).encode()) + return "{}{}_{}".format(wait_time, hash_function, hash_current_mobjects) + else: + return "{}_{}".format(wait_time, hash_current_mobjects) From 1e6e7aaa17c899958ea7b7cae500cf83e556b4f0 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Sat, 27 Jun 2020 16:26:32 +0200 Subject: [PATCH 05/47] fixed minor typo in doc --- manim/scene/scene_file_writer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index a508ed9961..fb5e3f2bc5 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -442,11 +442,11 @@ def close_movie_pipe(self): logger.debug(f"Animation {self.scene.num_plays} : Partial movie file written in {self.partial_movie_file_path}") def is_already_cached(self, hash_invokation): - """Will check if a file named with `hash_play` exists. + """Will check if a file named with `hash_invokation` exists. Parameters ---------- - hash_play : :class:`str` + hash_invokation : :class:`str` The hash corresponding to an invokation to either `scene.play` or `scene.wait`. Returns From b5ac0e26e9a1c62ce0488c5fa8d93037f0792ea9 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Sat, 27 Jun 2020 17:41:52 +0200 Subject: [PATCH 06/47] added cache cleaner --- manim/scene/scene_file_writer.py | 57 ++++++++++++++++++++++---------- manim/utils/file_ops.py | 14 ++++++-- 2 files changed, 51 insertions(+), 20 deletions(-) diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index fb5e3f2bc5..87480d7427 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -18,6 +18,7 @@ from ..utils.file_ops import guarantee_existence from ..utils.file_ops import add_extension_if_not_present from ..utils.file_ops import get_sorted_integer_files +from ..utils.file_ops import modify_atime from ..utils.sounds import get_full_sound_file_path @@ -68,8 +69,8 @@ def init_output_directories(self): """ module_directory = self.output_directory or self.get_default_module_directory() scene_name = self.file_name or self.get_default_scene_name() - #print("1") - #print(dirs.MEDIA_DIR) + # print("1") + # print(dirs.MEDIA_DIR) if self.save_last_frame or self.save_pngs: if dirs.MEDIA_DIR != "": image_dir = guarantee_existence(os.path.join( @@ -146,7 +147,7 @@ def get_resolution_directory(self): that immediately contains the video file will be 480p15. The file structure should look something like: - + MEDIA_DIR |--Tex |--texts @@ -172,7 +173,7 @@ def get_image_file_path(self): written to. It is usually named "images", but can be changed by changing "image_file_path". - + Returns ------- str @@ -232,16 +233,16 @@ def add_audio_segment(self, new_segment, """ This method adds an audio segment from an AudioSegment type object and suitable parameters. - + Parameters ---------- new_segment : AudioSegment The audio segment to add - + time : int, float, optional the timestamp at which the sound should be added. - + gain_to_background : optional The gain of the segment from the background. """ @@ -276,7 +277,7 @@ def add_sound(self, sound_file, time=None, gain=None, **kwargs): ---------- sound_file : str The path to the sound file. - + time : float or int, optional The timestamp at which the audio should be added. @@ -380,6 +381,8 @@ def finish(self): if hasattr(self, "writing_process"): self.writing_process.terminate() self.combine_movie_files() + # TODO add an option to disable clea_cache. Possible when #98 is merged ! + self.clean_cache() if self.save_last_frame: self.scene.update_frame(ignore_skipping=True) self.save_final_image(self.scene.get_image()) @@ -391,7 +394,8 @@ def open_movie_pipe(self): buffer. """ file_path = self.get_next_partial_movie_path() - temp_file_path = os.path.splitext(file_path)[0] + '_temp' + self.movie_file_extension + temp_file_path = os.path.splitext( + file_path)[0] + '_temp' + self.movie_file_extension self.partial_movie_file_path = file_path self.temp_partial_movie_file_path = temp_file_path @@ -439,9 +443,10 @@ def close_movie_pipe(self): self.temp_partial_movie_file_path, self.partial_movie_file_path, ) - logger.debug(f"Animation {self.scene.num_plays} : Partial movie file written in {self.partial_movie_file_path}") - - def is_already_cached(self, hash_invokation): + logger.debug( + f"Animation {self.scene.num_plays} : Partial movie file written in {self.partial_movie_file_path}") + + def is_already_cached(self, hash_invokation): """Will check if a file named with `hash_invokation` exists. Parameters @@ -454,7 +459,8 @@ def is_already_cached(self, hash_invokation): `bool` Wether the file exists. """ - path = os.path.join(self.partial_movie_directory, "{}{}".format(hash_invokation, self.movie_file_extension)) + path = os.path.join(self.partial_movie_directory, "{}{}".format( + hash_invokation, self.movie_file_extension)) return os.path.exists(path) def combine_movie_files(self): @@ -476,17 +482,18 @@ def combine_movie_files(self): # "extension": self.movie_file_extension, # } # if self.scene.start_at_animation_number is not None: - # kwargs["min_index"] = self.scene.start_at_animation_number + # kwargs["min_index"] = self.scene.start_at_animation_number # if self.scene.end_at_animation_number is not None: # kwargs["max_index"] = self.scene.end_at_animation_number # else: # kwargs["remove_indices_greater_than"] = self.scene.num_plays - 1 - # partial_movie_files = get_sorted_integer_files( + # partial_movie_files = get_sorted_integer_files( # self.partial_movie_directory, # **kwargs # ) - partial_movie_files = [os.path.join(self.partial_movie_directory, "{}{}".format(hash_play, self.movie_file_extension)) for hash_play in self.scene.play_hashes_list] + partial_movie_files = [os.path.join(self.partial_movie_directory, "{}{}".format( + hash_play, self.movie_file_extension)) for hash_play in self.scene.play_hashes_list] if len(partial_movie_files) == 0: logger.error("No animations in this scene") return @@ -515,8 +522,8 @@ def combine_movie_files(self): if self.write_to_movie: commands += [ - '-c', 'copy', - movie_file_path + '-c', 'copy', + movie_file_path ] if self.save_as_gif: @@ -561,6 +568,20 @@ def combine_movie_files(self): os.remove(sound_file_path) self.print_file_ready_message(movie_file_path) + for file_path in partial_movie_files: + # We have to modify the accessed time so if we have to clean the cache we remove the one used the longest. + modify_atime(file_path) + + def clean_cache(self): + """Will clean the cache by removing the partial_move used by manim the longest ago.""" + cached_partial_movies = [os.path.join(self.partial_movie_directory, file_name) for file_name in os.listdir( + self.partial_movie_directory) if file_name != "partial_movie_file_list.txt"] + max_file_cached = 7 # TODO put this in CONFIG when #98 is merged + if len(cached_partial_movies) > max_file_cached: + oldest_file_path = min(cached_partial_movies, key=os.path.getatime) + logger.info( + f"Partial movie directory is full. (> {max_file_cached} files). Manim removed the file used by manim the longest ago. ({os.path.basename(oldest_file_path)}).") + os.remove(oldest_file_path) def print_file_ready_message(self, file_path): """ diff --git a/manim/utils/file_ops.py b/manim/utils/file_ops.py index 181cfb426b..e79a9a5cbe 100644 --- a/manim/utils/file_ops.py +++ b/manim/utils/file_ops.py @@ -1,6 +1,6 @@ import os import numpy as np - +import time def add_extension_if_not_present(file_name, extension): # This could conceivably be smarter about handling existing differing extensions @@ -56,4 +56,14 @@ def get_sorted_integer_files(directory, elif remove_non_integer_files: os.remove(full_path) indexed_files.sort(key=lambda p: p[0]) - return list(map(lambda p: os.path.join(directory, p[1]), indexed_files)) \ No newline at end of file + return list(map(lambda p: os.path.join(directory, p[1]), indexed_files)) + +def modify_atime(file_path): + """Will manually change the accessed time (called `atime`) of the file, as on a lot of OS the accessed time refresh is disabled by default. + + Paramaters + ---------- + file_path : :class:`str` + The path of the file. + """ + os.utime(file_path, times=(time.time(),os.path.getmtime(file_path))) \ No newline at end of file From 39b39d89ba68df301781b8104522ff0c81265df8 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Mon, 6 Jul 2020 22:39:39 +0200 Subject: [PATCH 07/47] added disable_caching option --- manim/config.py | 14 ++++++++++---- manim/scene/scene.py | 8 ++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/manim/config.py b/manim/config.py index a866eb89dd..757e45136b 100644 --- a/manim/config.py +++ b/manim/config.py @@ -110,7 +110,7 @@ def _parse_file_writer_config(config_parser, args): # in batches, depending on their type: booleans and strings for boolean_opt in ['preview', 'show_file_in_finder', 'quiet', 'sound', 'leave_progress_bars', 'write_to_movie', 'save_last_frame', - 'save_pngs', 'save_as_gif', 'write_all']: + 'save_pngs', 'save_as_gif', 'write_all', 'disable_caching']: attr = getattr(args, boolean_opt) config[boolean_opt] = (default.getboolean(boolean_opt) if attr is None else attr) @@ -129,7 +129,7 @@ def _parse_file_writer_config(config_parser, args): # both CLI and the cfg file, so read the config dict directly if config['save_last_frame']: config['write_to_movie'] = False - + # Handle the -t (--transparent) flag. This flag determines which # section to use from the .cfg file. section = config_parser['transparent'] if args.transparent else default @@ -158,7 +158,8 @@ def _parse_file_writer_config(config_parser, args): for opt in ['write_to_movie', 'save_last_frame', 'save_pngs', 'save_as_gif', 'write_all']: config[opt] = config_parser['dry_run'].getboolean(opt) - + if not config['write_to_movie'] : + config['disable_caching'] = True # Read in the streaming section -- all values are strings config['streaming'] = {opt: config_parser['streaming'][opt] for opt in ['live_stream_name', 'twitch_stream_key', @@ -262,6 +263,12 @@ def _parse_cli(arg_list, input=True): const=True, help="Save the video as gif", ) + parser.add_argument( + "--disable_caching", + action="store_const", + const=True, + help="Disable caching (will create a partial movie file anyway", + ) # The default value of the following is set in manim.cfg parser.add_argument( @@ -306,7 +313,6 @@ def _parse_cli(arg_list, input=True): action="store_true", help="Do a dry run (render scenes but generate no output files)", ) - # The following overrides PNG_MODE, MOVIE_FILE_EXTENSION, and # BACKGROUND_OPACITY parser.add_argument( diff --git a/manim/scene/scene.py b/manim/scene/scene.py index 60293b5877..7140332942 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -860,9 +860,9 @@ def wrapper(self, *args, **kwargs): mobjects_on_scene = self.get_mobjects() hash_play = get_hash_from_play_call(animations, mobjects_on_scene) self.play_hashes_list.append(hash_play) - if self.enable_caching and self.file_writer.is_already_cached(hash_play): + if not file_writer_config['disable_caching'] and self.file_writer.is_already_cached(hash_play): logger.info(f'Animation {self.num_plays} : Using cached data (hash : {hash_play})') - self.skip_animations = True + file_writer_config['skip_animations'] = True else: self.revert_to_original_skipping_status() func(self, *args, **kwargs) @@ -884,9 +884,9 @@ def handle_caching_wait(func): def wrapper(self, duration=DEFAULT_WAIT_TIME, stop_condition=None): hash_wait = get_hash_from_wait_call(duration, stop_condition, self.get_mobjects()) self.play_hashes_list .append(hash_wait) - if self.enable_caching and self.file_writer.is_already_cached(hash_wait): + if not file_writer_config['disable_caching'] and self.file_writer.is_already_cached(hash_wait): logger.info(f'Wait {self.num_plays} : Using cached data (hash : {hash_wait})') - self.skip_animations = True + file_writer_config['skip_animations'] = True else : self.revert_to_original_skipping_status() func(self, duration, stop_condition) From ae3d3d40e5943aa91a5569e163890397f837f0d3 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Fri, 10 Jul 2020 15:25:51 +0200 Subject: [PATCH 08/47] supported camera_config, and hashing functions --- manim/scene/scene.py | 5 ++- manim/utils/hashing.py | 88 ++++++++++++++++++++++++++++++++---------- 2 files changed, 70 insertions(+), 23 deletions(-) diff --git a/manim/scene/scene.py b/manim/scene/scene.py index 7140332942..1f75f8281a 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -2,6 +2,7 @@ import random import warnings import platform +import copy from tqdm import tqdm as ProgressDisplay import numpy as np @@ -858,7 +859,7 @@ def wrapper(self, *args, **kwargs): ) self.add_mobjects_from_animations(animations) mobjects_on_scene = self.get_mobjects() - hash_play = get_hash_from_play_call(animations, mobjects_on_scene) + hash_play = get_hash_from_play_call(self.__dict__['camera'], animations, mobjects_on_scene) self.play_hashes_list.append(hash_play) if not file_writer_config['disable_caching'] and self.file_writer.is_already_cached(hash_play): logger.info(f'Animation {self.num_plays} : Using cached data (hash : {hash_play})') @@ -882,7 +883,7 @@ def handle_caching_wait(func): named parameters affecting what was passed in *args e.g run_time, lag_ratio etc. """ def wrapper(self, duration=DEFAULT_WAIT_TIME, stop_condition=None): - hash_wait = get_hash_from_wait_call(duration, stop_condition, self.get_mobjects()) + hash_wait = get_hash_from_wait_call(self.__dict__['camera'], duration, stop_condition, self.get_mobjects()) self.play_hashes_list .append(hash_wait) if not file_writer_config['disable_caching'] and self.file_writer.is_already_cached(hash_wait): logger.info(f'Wait {self.num_plays} : Using cached data (hash : {hash_wait})') diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py index 57bc06966e..e791788f0c 100644 --- a/manim/utils/hashing.py +++ b/manim/utils/hashing.py @@ -1,14 +1,19 @@ import json import zlib import inspect +import copy +import dis import numpy as np +from types import ModuleType + +from ..logger import logger class CustomEncoder(json.JSONEncoder): def default(self, obj): """ - This method is used to encode objects contained in some dict in an object that JSON encoder will recognize. - If obj is a function, then it will return the code of this function. + This method is used to serialize objects to JSON format. + If obj is a function, then it will return a dict with two keys : 'code', for the code source, and 'nonlocals' for all nonlocalsvalues. (including nonlocals functions, that will be serialized as this is recursive.) if obj is a np.darray, it converts it into a list. if obj is an object with __dict__ attribute, it returns its __dict__. Else, will let the JSONEncoder do the stuff, and throw an error if the type is not suitable for JSONEncoder. @@ -24,17 +29,31 @@ def default(self, obj): Python object that JSON encoder will recognize """ - if callable(obj): - return inspect.getsource(obj) # Improvement to do ? + if inspect.isfunction(obj) and not isinstance(obj, ModuleType): + r = inspect.getclosurevars(obj) + x = {**copy.copy(r.globals), **copy.copy(r.nonlocals)} + for i in list(x): + # NOTE : All module types objects are removed, because otherwise it throws ValueError: Circular reference detected if not. TODO + if isinstance(x[i], ModuleType): + del x[i] + return {'code': inspect.getsource(obj), + 'nonlocals': x} elif isinstance(obj, np.ndarray): return list(obj) elif hasattr(obj, "__dict__"): return getattr(obj, '__dict__') - return json.JSONEncoder.default(self, obj) + elif isinstance(obj, np.uint8): + return int(obj) + try: + return json.JSONEncoder.default(self, obj) + except TypeError: + # This is used when the user enters an unknown type in CONFIG. Rather than throwing an error, we transform + # it into a string "Unsupported type for hashing" so it won't affect the hash. + return "Unsupported type for hashing" -def get_json(dict_config): - """Flatten a dictionnary of objects by tranforming these objects into their __dict__ or another type defined in the CustomEncoder class +def get_json(object): + """Recursively serialize object to JSON. Use CustomEncoder class above. Paramaters ---------- @@ -44,16 +63,40 @@ def get_json(dict_config): Returns ------- `str` - the dict flattened + the object flattened + """ + return json.dumps(object, cls=CustomEncoder) + + +def get_camera_dict_for_hashing(camera_object): + """Remove some keys from cameraobject.__dict__ that are useless for the caching functionnality and very heavy. + + Parameters + ---------- + object_camera : :class:``~.Camera` + The camera object used in the scene + + Returns + ------- + `dict` + Camera.__dict__ but cleaned. """ - return json.dumps(dict_config, cls=CustomEncoder) + camera_object_dict = copy.deepcopy(camera_object.__dict__) + # We have to clean a little bit camera_dict, as pixel_array and background are two very big numpy array. + # They are not essential to caching process. + for to_clean in ['background', 'pixel_array']: + camera_object_dict.pop(to_clean, None) + return camera_object_dict -def get_hash_from_play_call(animations_list, current_mobjects_list): +def get_hash_from_play_call(camera_object, animations_list, current_mobjects_list): """Take the list of animations and a list of mobjects and output their hash. Is meant to be used for `scene.play` function. Parameters ----------- + object_camera : :class:``~.Camera` + The camera object used in the scene + animations_list : :class:`list` The list of animations @@ -63,24 +106,26 @@ def get_hash_from_play_call(animations_list, current_mobjects_list): Returns ------- `str` - concatenation of the hash of animations_list and current_mobjects_list separated by '_'. + concatenation of the hash of object_camera, animations_list and current_mobjects_list separated by '_'. """ - animations_list_json = [get_json(x).encode() for x in sorted( + camera_json = get_json(get_camera_dict_for_hashing(camera_object)) + animations_list_json = [get_json(x) for x in sorted( animations_list, key=lambda obj: str(obj))] - current_mobjects_list_json = [get_json(x).encode() for x in sorted( + current_mobjects_list_json = [get_json(x) for x in sorted( current_mobjects_list, key=lambda obj: str(obj))] + hash_camera = zlib.crc32(repr(camera_json).encode()) hash_animations = zlib.crc32(repr(animations_list_json).encode()) hash_current_mobjects = zlib.crc32( repr(current_mobjects_list_json).encode()) - return "{}_{}".format(hash_animations, hash_current_mobjects) + return "{}_{}_{}".format(hash_camera, hash_animations, hash_current_mobjects) -def get_hash_from_wait_call(wait_time, stop_condition_function, current_mobjects_list): +def get_hash_from_wait_call(camera_object, wait_time, stop_condition_function, current_mobjects_list): """Take a wait time, a boolean function as stop_condition and a list of mobjects output their hash. Is meant to be used for `scene.wait` function. Parameters ----------- - wait_time : :class:`int` + wait_time : :class:`float` The time to wait stop_condition_function : :class:`func` @@ -91,13 +136,14 @@ def get_hash_from_wait_call(wait_time, stop_condition_function, current_mobjects `str` concatenation of the hash of animations_list and current_mobjects_list separated by '_'. """ - current_mobjects_list_json = [get_json(x).encode() for x in sorted( + camera_json = get_json(get_camera_dict_for_hashing(camera_object)) + current_mobjects_list_json = [get_json(x) for x in sorted( current_mobjects_list, key=lambda obj: str(obj))] hash_current_mobjects = zlib.crc32( repr(current_mobjects_list_json).encode()) + hash_camera = zlib.crc32(repr(camera_json).encode()) if stop_condition_function != None: - hash_function = zlib.crc32(inspect.getsource( - stop_condition_function).encode()) - return "{}{}_{}".format(wait_time, hash_function, hash_current_mobjects) + hash_function = zlib.crc32(get_json(stop_condition_function).encode()) + return "{}_{}{}_{}".format(hash_camera, str(wait_time).replace('.', '-'), hash_function, hash_current_mobjects) else: - return "{}_{}".format(wait_time, hash_current_mobjects) + return "{}_{}_{}".format(hash_camera, str(wait_time).replace('.', '-'), hash_current_mobjects) From 648825206a7a0bf0c70196f4a95d35e2b6db345b Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Wed, 22 Jul 2020 14:50:34 +0200 Subject: [PATCH 09/47] use now digital naming when disable_caching --- manim/scene/scene.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/manim/scene/scene.py b/manim/scene/scene.py index 1f75f8281a..2a2262c2c9 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -858,13 +858,16 @@ def wrapper(self, *args, **kwargs): *args, **kwargs ) self.add_mobjects_from_animations(animations) - mobjects_on_scene = self.get_mobjects() - hash_play = get_hash_from_play_call(self.__dict__['camera'], animations, mobjects_on_scene) - self.play_hashes_list.append(hash_play) - if not file_writer_config['disable_caching'] and self.file_writer.is_already_cached(hash_play): - logger.info(f'Animation {self.num_plays} : Using cached data (hash : {hash_play})') - file_writer_config['skip_animations'] = True + if not file_writer_config['disable_caching']: + mobjects_on_scene = self.get_mobjects() + hash_play = get_hash_from_play_call(self.__dict__['camera'], animations, mobjects_on_scene) + self.play_hashes_list.append(hash_play) + if self.file_writer.is_already_cached(hash_play): + logger.info(f'Animation {self.num_plays} : Using cached data (hash : {hash_play})') + file_writer_config['skip_animations'] = True else: + hash_play = "uncached_{:05}".format(self.num_plays) + self.play_hashes_list.append(hash_play) self.revert_to_original_skipping_status() func(self, *args, **kwargs) return wrapper From d02309b21db3f621a79d1dd16bb7000b9088daec Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Wed, 22 Jul 2020 15:30:44 +0200 Subject: [PATCH 10/47] added an option to flush the cache. --- manim/config.py | 10 ++++++++-- manim/scene/scene_file_writer.py | 13 ++++++++++++- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/manim/config.py b/manim/config.py index 757e45136b..4f48e4f6a1 100644 --- a/manim/config.py +++ b/manim/config.py @@ -110,7 +110,7 @@ def _parse_file_writer_config(config_parser, args): # in batches, depending on their type: booleans and strings for boolean_opt in ['preview', 'show_file_in_finder', 'quiet', 'sound', 'leave_progress_bars', 'write_to_movie', 'save_last_frame', - 'save_pngs', 'save_as_gif', 'write_all', 'disable_caching']: + 'save_pngs', 'save_as_gif', 'write_all', 'disable_caching', 'flush_cache']: attr = getattr(args, boolean_opt) config[boolean_opt] = (default.getboolean(boolean_opt) if attr is None else attr) @@ -267,7 +267,13 @@ def _parse_cli(arg_list, input=True): "--disable_caching", action="store_const", const=True, - help="Disable caching (will create a partial movie file anyway", + help="Disable caching (will create a partial movie file anyway)", + ) + parser.add_argument( + "--flush_cache", + action="store_const", + const=True, + help="Remove all partial-movie-files cached.", ) # The default value of the following is set in manim.cfg diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index a8d556a9d3..07f2d1c923 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -362,7 +362,10 @@ def finish(self): if hasattr(self, "writing_process"): self.writing_process.terminate() self.combine_movie_files() - self.clean_cache() + if file_writer_config['flush_cache']: + self.flush_cache_directory() + else : + self.clean_cache() if file_writer_config['save_last_frame']: self.scene.update_frame(ignore_skipping=True) self.save_final_image(self.scene.get_image()) @@ -563,6 +566,14 @@ def clean_cache(self): f"Partial movie directory is full. (> {max_file_cached} files). Manim removed the file used by manim the longest ago. ({os.path.basename(oldest_file_path)}).") os.remove(oldest_file_path) + def flush_cache_directory(self): + """Delete all the partial movie files cached""" + cached_partial_movies = [os.path.join(self.partial_movie_directory, file_name) for file_name in os.listdir( + self.partial_movie_directory) if file_name != "partial_movie_file_list.txt"] + for f in cached_partial_movies: + os.remove(f) + logger.info(f"Cache flushed. {len(cached_partial_movies)} file(s) deleted in {self.partial_movie_directory}.") + def print_file_ready_message(self, file_path): """ Prints the "File Ready" message to STDOUT. From f1da2ffe119ce3bd2c8504e1914a36b9834a82af Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Wed, 22 Jul 2020 15:31:58 +0200 Subject: [PATCH 11/47] fixed shameful typo --- manim/scene/scene.py | 8 ++++---- manim/scene/scene_file_writer.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/manim/scene/scene.py b/manim/scene/scene.py index 2a2262c2c9..31b7497b30 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -842,8 +842,8 @@ def update_skipping_status(self): def handle_caching_play(func): """ - This method is used internally to wrap the passed function into a function that will compute the hash of the play invokation, - and will act accordingly : either skip the animation because already cached, either nothing and let the play invokation be processed normally. + This method is used internally to wrap the passed function into a function that will compute the hash of the play invocation, + and will act accordingly : either skip the animation because already cached, either nothing and let the play invocation be processed normally. Parameters ---------- @@ -874,8 +874,8 @@ def wrapper(self, *args, **kwargs): def handle_caching_wait(func): """ - This method is used internally to wrap the passed function into a function that will compute the hash of the wait invokation, - and will act accordingly : either skip the animation because already cached or nothing and let the play invokation be processed normally + This method is used internally to wrap the passed function into a function that will compute the hash of the wait invocation, + and will act accordingly : either skip the animation because already cached or nothing and let the play invocation be processed normally Parameters ---------- diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index 07f2d1c923..9b644fd2ab 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -435,7 +435,7 @@ def is_already_cached(self, hash_invokation): Parameters ---------- hash_invokation : :class:`str` - The hash corresponding to an invokation to either `scene.play` or `scene.wait`. + The hash corresponding to an invocation to either `scene.play` or `scene.wait`. Returns ------- From 83b03eb1bedb07424f631b71eed3e9aa8b18173e Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Wed, 22 Jul 2020 15:43:36 +0200 Subject: [PATCH 12/47] added max_cached_files --- manim/config.py | 1 + manim/default.cfg | 3 +++ manim/scene/scene_file_writer.py | 6 +++--- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/manim/config.py b/manim/config.py index 4f48e4f6a1..96545e2843 100644 --- a/manim/config.py +++ b/manim/config.py @@ -171,6 +171,7 @@ def _parse_file_writer_config(config_parser, args): # For internal use (no CLI flag) config['skip_animations'] = any([config['save_last_frame'], config['from_animation_number']]) + config['max_file_cached'] = default.getint('max_file_cached') return config diff --git a/manim/default.cfg b/manim/default.cfg index c2cf102950..a0aeaa75bb 100644 --- a/manim/default.cfg +++ b/manim/default.cfg @@ -88,6 +88,9 @@ frame_rate = 60 pixel_height = 1440 pixel_width = 2560 +max_file_cached = 7 +flush_cache = False + # These override the previous by using -t, --transparent [transparent] png_mode = RGBA diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index 9b644fd2ab..1d0f11487c 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -559,11 +559,11 @@ def clean_cache(self): """Will clean the cache by removing the partial_move used by manim the longest ago.""" cached_partial_movies = [os.path.join(self.partial_movie_directory, file_name) for file_name in os.listdir( self.partial_movie_directory) if file_name != "partial_movie_file_list.txt"] - max_file_cached = 7 # TODO put this in CONFIG when #98 is merged - if len(cached_partial_movies) > max_file_cached: + if len(cached_partial_movies) > file_writer_config['max_file_cached']: oldest_file_path = min(cached_partial_movies, key=os.path.getatime) logger.info( - f"Partial movie directory is full. (> {max_file_cached} files). Manim removed the file used by manim the longest ago. ({os.path.basename(oldest_file_path)}).") + f"Partial movie directory is full. (> {file_writer_config['max_file_cached']} files). Manim removed the file used by manim the longest ago. ({os.path.basename(oldest_file_path)})." + + "You can change this behaviour by changing max_file_cached in config.") os.remove(oldest_file_path) def flush_cache_directory(self): From c5a0b1f8cf668817e140f3a861ec70f6de71248e Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Wed, 22 Jul 2020 16:00:36 +0200 Subject: [PATCH 13/47] fixed merge issues --- manim/config.py | 317 ------------------------------------ manim/default.cfg | 1 + manim/utils/config_utils.py | 8 +- 3 files changed, 8 insertions(+), 318 deletions(-) diff --git a/manim/config.py b/manim/config.py index 36688613e0..5ab17cabda 100644 --- a/manim/config.py +++ b/manim/config.py @@ -76,323 +76,6 @@ def _parse_config(config_parser, args): f"Custom TeX template {tex_fn} not found or not readable. " "Falling back to the default template." ) - filename = None - config['tex_template_file'] = filename - config['tex_template'] = (TexTemplateFromFile(filename=filename) - if filename is not None - else TexTemplate()) - - return config - - -def _parse_file_writer_config(config_parser, args): - """Parse config files and CLI arguments into a single dictionary.""" - # By default, use the CLI section of the digested .cfg files - default = config_parser['CLI'] - - # This will be the final config dict exposed to the user - config = {} - - # Handle input files and scenes. Note these cannot be set from - # the .cfg files, only from CLI arguments - config['input_file'] = args.file - config['scene_names'] = (args.scene_names - if args.scene_names is not None else []) - config['output_file'] = args.output_file - - # Handle all options that are directly overridden by CLI - # arguments. Note ConfigParser options are all strings and each - # needs to be converted to the appropriate type. Thus, we do this - # in batches, depending on their type: booleans and strings - for boolean_opt in ['preview', 'show_file_in_finder', 'quiet', 'sound', - 'leave_progress_bars', 'write_to_movie', 'save_last_frame', - 'save_pngs', 'save_as_gif', 'write_all', 'disable_caching', 'flush_cache']: - attr = getattr(args, boolean_opt) - config[boolean_opt] = (default.getboolean(boolean_opt) - if attr is None else attr) - # for str_opt in ['media_dir', 'video_dir', 'tex_dir', 'text_dir']: - for str_opt in ['media_dir']: - attr = getattr(args, str_opt) - config[str_opt] = (default[str_opt] if attr is None else attr) - dir_names = {'video_dir': 'videos', - 'tex_dir': 'Tex', - 'text_dir': 'texts'} - for name in dir_names: - config[name] = os.path.join(config['media_dir'], dir_names[name]) - - # Handle the -s (--save_last_frame) flag: invalidate the -w flag - # At this point the save_last_frame option has already been set by - # both CLI and the cfg file, so read the config dict directly - if config['save_last_frame']: - config['write_to_movie'] = False - - # Handle the -t (--transparent) flag. This flag determines which - # section to use from the .cfg file. - section = config_parser['transparent'] if args.transparent else default - for opt in ['png_mode', 'movie_file_extension', 'background_opacity']: - config[opt] = section[opt] - - # Handle the -n flag. Read first from the cfg and then override with CLI. - # These two are integers -- use getint() - for opt in ['from_animation_number', 'upto_animation_number']: - config[opt] = default.getint(opt) - if config['upto_animation_number'] == -1: - config['upto_animation_number'] = float('inf') - nflag = args.from_animation_number - if nflag is not None: - if ',' in nflag: - start, end = nflag.split(',') - config['from_animation_number'] = int(start) - config['upto_animation_number'] = int(end) - else: - config['from_animation_number'] = int(nflag) - - # Handle the --dry_run flag. This flag determines which section - # to use from the .cfg file. All options involved are boolean. - # Note this overrides the flags -w, -s, -a, -g, and -i. - if args.dry_run: - for opt in ['write_to_movie', 'save_last_frame', 'save_pngs', - 'save_as_gif', 'write_all']: - config[opt] = config_parser['dry_run'].getboolean(opt) - if not config['write_to_movie'] : - config['disable_caching'] = True - # Read in the streaming section -- all values are strings - config['streaming'] = {opt: config_parser['streaming'][opt] - for opt in ['live_stream_name', 'twitch_stream_key', - 'streaming_protocol', 'streaming_ip', - 'streaming_protocol', 'streaming_client', - 'streaming_port', 'streaming_port', - 'streaming_console_banner']} - - # For internal use (no CLI flag) - config['skip_animations'] = any([config['save_last_frame'], - config['from_animation_number']]) - config['max_file_cached'] = default.getint('max_file_cached') - - return config - - -def _parse_cli(arg_list, input=True): - parser = argparse.ArgumentParser( - description='Animation engine for explanatory math videos', - epilog='Made with ❤ by the manim community devs' - ) - if input: - parser.add_argument( - "file", - help="path to file holding the python code for the scene", - ) - parser.add_argument( - "scene_names", - nargs="*", - help="Name of the Scene class you want to see", - default=[''], - ) - parser.add_argument( - "-o", "--output_file", - help="Specify the name of the output file, if" - "it should be different from the scene class name", - default='', - ) - - # The following use (action='store_const', const=True) instead of - # the built-in (action='store_true'). This is because the latter - # will default to False if not specified, while the former sets no - # default value. Since we want to set the default value in - # manim.cfg rather than here, we use the former. - parser.add_argument( - "-p", "--preview", - action="store_const", - const=True, - help="Automatically open the saved file once its done", - ) - parser.add_argument( - "-f", "--show_file_in_finder", - action="store_const", - const=True, - help="Show the output file in finder", - ) - parser.add_argument( - "-q", "--quiet", - action="store_const", - const=True, - help="Quiet mode", - ) - parser.add_argument( - "--sound", - action="store_const", - const=True, - help="Play a success/failure sound", - ) - parser.add_argument( - "--leave_progress_bars", - action="store_const", - const=True, - help="Leave progress bars displayed in terminal", - ) - parser.add_argument( - "-a", "--write_all", - action="store_const", - const=True, - help="Write all the scenes from a file", - ) - parser.add_argument( - "-w", "--write_to_movie", - action="store_const", - const=True, - help="Render the scene as a movie file", - ) - parser.add_argument( - "-s", "--save_last_frame", - action="store_const", - const=True, - help="Save the last frame (and do not save movie)", - ) - parser.add_argument( - "-g", "--save_pngs", - action="store_const", - const=True, - help="Save each frame as a png", - ) - parser.add_argument( - "-i", "--save_as_gif", - action="store_const", - const=True, - help="Save the video as gif", - ) - parser.add_argument( - "--disable_caching", - action="store_const", - const=True, - help="Disable caching (will create a partial movie file anyway)", - ) - parser.add_argument( - "--flush_cache", - action="store_const", - const=True, - help="Remove all partial-movie-files cached.", - ) - - # The default value of the following is set in manim.cfg - parser.add_argument( - "-c", "--color", - help="Background color", - ) - parser.add_argument( - "--background_opacity", - help="Background opacity", - ) - parser.add_argument( - "--media_dir", - help="directory to write media", - ) - # video_group = parser.add_mutually_exclusive_group() - # video_group.add_argument( - # "--video_dir", - # help="directory to write file tree for video", - # ) - # parser.add_argument( - # "--tex_dir", - # help="directory to write tex", - # ) - # parser.add_argument( - # "--text_dir", - # help="directory to write text", - # ) - parser.add_argument( - "--tex_template", - help="Specify a custom TeX template file", - ) - - # All of the following use (action="store_true"). This means that - # they are by default False. In contrast to the previous ones that - # used (action="store_const", const=True), the following do not - # correspond to a single configuration option. Rather, they - # override several options at the same time. - - # The following overrides -w, -a, -g, and -i - parser.add_argument( - "--dry_run", - action="store_true", - help="Do a dry run (render scenes but generate no output files)", - ) - # The following overrides PNG_MODE, MOVIE_FILE_EXTENSION, and - # BACKGROUND_OPACITY - parser.add_argument( - "-t", "--transparent", - action="store_true", - help="Render to a movie file with an alpha channel", - ) - - # The following are mutually exclusive and each overrides - # FRAME_RATE, PIXEL_HEIGHT, and PIXEL_WIDTH, - parser.add_argument( - "-l", "--low_quality", - action="store_true", - help="Render at low quality (for fastest rendering)", - ) - parser.add_argument( - "-m", "--medium_quality", - action="store_true", - help="Render at medium quality (for much faster rendering)", - ) - parser.add_argument( - "-e", "--high_quality", - action="store_true", - help="Render at high quality (for slightly faster rendering)", - ) - parser.add_argument( - "-k", "--fourk_quality", - action="store_true", - help="Render at 4K quality (slower rendering)", - ) - - # This overrides any of the above - parser.add_argument( - "-r", "--resolution", - help="Resolution, passed as \"height,width\"", - ) - - # This sets FROM_ANIMATION_NUMBER and UPTO_ANIMATION_NUMBER - parser.add_argument( - "-n", "--from_animation_number", - help="Start rendering not from the first animation, but" - "from another, specified by its index. If you pass" - "in two comma separated values, e.g. \"3,6\", it will end" - "the rendering at the second value", - ) - - # Specify the manim.cfg file - parser.add_argument( - "--config_file", - help="Specify the configuration file", - ) - - return parser.parse_args(arg_list) - - -def _init_dirs(config): - # Make sure all folders exist - for folder in [config["media_dir"], config["video_dir"], - config["tex_dir"], config["text_dir"]]: - if not os.path.exists(folder): - os.makedirs(folder) - - -def _from_command_line(): - """Determine if manim was called from the command line.""" - # Manim can be called from the command line in three different - # ways. The first two involve using the manim or manimcm commands - prog = os.path.split(sys.argv[0])[-1] - from_cli_command = prog in ['manim', 'manimcm'] - - # The third way involves using `python -m manim ...`. In this - # case, the CLI arguments passed to manim do not include 'manim', - # 'manimcm', or even 'python'. However, the -m flag will always - # be the first argument. - from_python_m = sys.argv[0] == '-m' - - return from_cli_command or from_python_m tex_fn = None config["tex_template_file"] = tex_fn config["tex_template"] = ( diff --git a/manim/default.cfg b/manim/default.cfg index 03dc460999..b73f7dc658 100644 --- a/manim/default.cfg +++ b/manim/default.cfg @@ -90,6 +90,7 @@ pixel_width = 2560 max_file_cached = 7 flush_cache = False +disable_caching = False # These override the previous by using -t, --transparent [transparent] diff --git a/manim/utils/config_utils.py b/manim/utils/config_utils.py index 798b0b8a91..6ef972a361 100644 --- a/manim/utils/config_utils.py +++ b/manim/utils/config_utils.py @@ -50,7 +50,7 @@ def _parse_file_writer_config(config_parser, args): "write_all", "disable_caching", "flush_cache"]: - ]: + attr = getattr(args, boolean_opt) fw_config[boolean_opt] = ( default.getboolean(boolean_opt) if attr is None else attr @@ -220,6 +220,12 @@ def _parse_cli(arg_list, input=True): const=True, help="Save the video as gif", ) + parser.add_argument( + "--disable_caching", + action="store_const", + const=True, + help="Disable caching (will generate partial-movie-files anyway).", + ) parser.add_argument( "--flush_cache", action="store_const", From 3fb8fe23f18811b48974350df40353962d86e911 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Wed, 22 Jul 2020 16:04:13 +0200 Subject: [PATCH 14/47] added digital naming when disable_caching --- manim/scene/scene.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/manim/scene/scene.py b/manim/scene/scene.py index 03cf4c5e53..ac440077b4 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -886,12 +886,15 @@ def handle_caching_wait(func): named parameters affecting what was passed in *args e.g run_time, lag_ratio etc. """ def wrapper(self, duration=DEFAULT_WAIT_TIME, stop_condition=None): - hash_wait = get_hash_from_wait_call(self.__dict__['camera'], duration, stop_condition, self.get_mobjects()) - self.play_hashes_list .append(hash_wait) - if not file_writer_config['disable_caching'] and self.file_writer.is_already_cached(hash_wait): - logger.info(f'Wait {self.num_plays} : Using cached data (hash : {hash_wait})') - file_writer_config['skip_animations'] = True + if not file_writer_config['disable_caching']: + hash_wait = get_hash_from_wait_call(self.__dict__['camera'], duration, stop_condition, self.get_mobjects()) + self.play_hashes_list .append(hash_wait) + if self.file_writer.is_already_cached(hash_wait): + logger.info(f'Wait {self.num_plays} : Using cached data (hash : {hash_wait})') + file_writer_config['skip_animations'] = True else : + hash_play = "uncached_{:05}".format(self.num_plays) + self.play_hashes_list.append(hash_play) self.revert_to_original_skipping_status() func(self, duration, stop_condition) return wrapper From 9ff6d41e54b1442f820581d4199db86728d9658a Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Wed, 22 Jul 2020 22:33:09 +0200 Subject: [PATCH 15/47] foxed skip_animations dlag that #98 broke --- manim/scene/scene.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/manim/scene/scene.py b/manim/scene/scene.py index ac440077b4..cd5044ebca 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -853,7 +853,8 @@ def handle_caching_play(func): **kwargs : named parameters affecting what was passed in *args e.g run_time, lag_ratio etc. """ - def wrapper(self, *args, **kwargs): + def wrapper(self, *args, **kwargs): + self.revert_to_original_skipping_status() animations = self.compile_play_args_to_animation_list( *args, **kwargs ) @@ -886,15 +887,16 @@ def handle_caching_wait(func): named parameters affecting what was passed in *args e.g run_time, lag_ratio etc. """ def wrapper(self, duration=DEFAULT_WAIT_TIME, stop_condition=None): + self.revert_to_original_skipping_status() if not file_writer_config['disable_caching']: hash_wait = get_hash_from_wait_call(self.__dict__['camera'], duration, stop_condition, self.get_mobjects()) - self.play_hashes_list .append(hash_wait) + self.play_hashes_list.append(hash_wait) if self.file_writer.is_already_cached(hash_wait): logger.info(f'Wait {self.num_plays} : Using cached data (hash : {hash_wait})') file_writer_config['skip_animations'] = True else : - hash_play = "uncached_{:05}".format(self.num_plays) - self.play_hashes_list.append(hash_play) + hash_wait = "uncached_{:05}".format(self.num_plays) + self.play_hashes_list.append(hash_wait) self.revert_to_original_skipping_status() func(self, duration, stop_condition) return wrapper @@ -1171,8 +1173,8 @@ def force_skipping(self): Scene The Scene, with skipping turned on. """ - self.original_skipping_status = self.SKIP_ANIMATIONS - self.SKIP_ANIMATIONS = True + self.original_skipping_status = file_writer_config['skip_animations'] + file_writer_config['skip_animations'] = True return self def revert_to_original_skipping_status(self): @@ -1187,7 +1189,7 @@ def revert_to_original_skipping_status(self): The Scene, with the original skipping status. """ if hasattr(self, "original_skipping_status"): - self.SKIP_ANIMATIONS = self.original_skipping_status + file_writer_config['skip_animations'] = self.original_skipping_status return self def add_frames(self, *frames): @@ -1222,7 +1224,7 @@ def add_sound(self, sound_file, time_offset=0, gain=None, **kwargs): gain : """ - if self.SKIP_ANIMATIONS: + if file_writer_config['skip_animations']: return time = self.get_time() + time_offset self.file_writer.add_sound(sound_file, time, gain, **kwargs) From 64d82a357758e8aab81553bd70737cd2457892d8 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Wed, 22 Jul 2020 22:39:09 +0200 Subject: [PATCH 16/47] removed cairo context from hash --- manim/utils/hashing.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py index e791788f0c..bda6585ddf 100644 --- a/manim/utils/hashing.py +++ b/manim/utils/hashing.py @@ -81,10 +81,11 @@ def get_camera_dict_for_hashing(camera_object): `dict` Camera.__dict__ but cleaned. """ - camera_object_dict = copy.deepcopy(camera_object.__dict__) + camera_object_dict = copy.copy(camera_object.__dict__) # We have to clean a little bit camera_dict, as pixel_array and background are two very big numpy array. # They are not essential to caching process. - for to_clean in ['background', 'pixel_array']: + # We also have to remove pixel_array_to_cairo_context as it conntains uses memory adress (set randomly). See l.516 get_cached_cairo_context in camera.py + for to_clean in ['background', 'pixel_array', 'pixel_array_to_cairo_context']: camera_object_dict.pop(to_clean, None) return camera_object_dict From ebfcbc14f6f923f35f210776976170ce834619a8 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Wed, 22 Jul 2020 22:41:34 +0200 Subject: [PATCH 17/47] removed deprecated code --- manim/scene/scene_file_writer.py | 15 --------------- manim/utils/file_ops.py | 30 ------------------------------ 2 files changed, 45 deletions(-) diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index 1d0f11487c..a5baf907b5 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -14,7 +14,6 @@ from ..utils.config_ops import digest_config from ..utils.file_ops import guarantee_existence from ..utils.file_ops import add_extension_if_not_present -from ..utils.file_ops import get_sorted_integer_files from ..utils.file_ops import modify_atime from ..utils.sounds import get_full_sound_file_path @@ -460,20 +459,6 @@ def combine_movie_files(self): # the scene as a whole, one of course wants to see it as a # single piece. - # kwargs = { - # "remove_non_integer_files": False, - # "extension": self.movie_file_extension, - # } - # if self.scene.start_at_animation_number is not None: - # kwargs["min_index"] = self.scene.start_at_animation_number - # if self.scene.end_at_animation_number is not None: - # kwargs["max_index"] = self.scene.end_at_animation_number - # else: - # kwargs["remove_indices_greater_than"] = self.scene.num_plays - 1 - # partial_movie_files = get_sorted_integer_files( - # self.partial_movie_directory, - # **kwargs - # ) partial_movie_files = [os.path.join(self.partial_movie_directory, "{}{}".format( hash_play, file_writer_config['movie_file_extension'])) for hash_play in self.scene.play_hashes_list] # A OPTIMISER ! Là on recuperer deux fois la list des partial movies files, alors qu'on pourrait utiliser genre get_next_partial_movie_path jsp diff --git a/manim/utils/file_ops.py b/manim/utils/file_ops.py index e79a9a5cbe..170f7be83e 100644 --- a/manim/utils/file_ops.py +++ b/manim/utils/file_ops.py @@ -28,36 +28,6 @@ def seek_full_path_from_defaults(file_name, default_dir, extensions): raise IOError("File {} not Found".format(file_name)) -def get_sorted_integer_files(directory, - min_index=0, - max_index=np.inf, - remove_non_integer_files=False, - remove_indices_greater_than=None, - extension=None, - ): - indexed_files = [] - for file in os.listdir(directory): - if '.' in file: - index_str = file[:file.index('.')] - else: - index_str = file - - full_path = os.path.join(directory, file) - if index_str.isdigit(): - index = int(index_str) - if remove_indices_greater_than is not None: - if index > remove_indices_greater_than: - os.remove(full_path) - continue - if extension is not None and not file.endswith(extension): - continue - if index >= min_index and index < max_index: - indexed_files.append((index, file)) - elif remove_non_integer_files: - os.remove(full_path) - indexed_files.sort(key=lambda p: p[0]) - return list(map(lambda p: os.path.join(directory, p[1]), indexed_files)) - def modify_atime(file_path): """Will manually change the accessed time (called `atime`) of the file, as on a lot of OS the accessed time refresh is disabled by default. From 7f98b1c9e5f174a224bb3044c02d45f5b0086bc2 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Wed, 22 Jul 2020 22:51:33 +0200 Subject: [PATCH 18/47] fixed tests by setting write_to_movie to False --- manim/scene/scene_file_writer.py | 7 ++++--- tests/testing_utils.py | 2 ++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index a5baf907b5..cf91bbbc45 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -536,9 +536,10 @@ def combine_movie_files(self): os.remove(sound_file_path) self.print_file_ready_message(movie_file_path) - for file_path in partial_movie_files: - # We have to modify the accessed time so if we have to clean the cache we remove the one used the longest. - modify_atime(file_path) + if file_writer_config["write_to_movie"] : + for file_path in partial_movie_files: + # We have to modify the accessed time so if we have to clean the cache we remove the one used the longest. + modify_atime(file_path) def clean_cache(self): """Will clean the cache by removing the partial_move used by manim the longest ago.""" diff --git a/tests/testing_utils.py b/tests/testing_utils.py index f8651de969..8cdeeb021b 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -44,6 +44,8 @@ def __init__(self, scene_object, module_tested, caching_needed=False): self.path_tests_medias_cache, scene_object.__name__, 'Tex') file_writer_config['skip_animations'] = True + file_writer_config['disable_caching'] = True + file_writer_config["write_to_movie"] = False config['pixel_height'] = 480 config['pixel_width'] = 854 config['frame_rate'] = 15 From d10ee72ba507d19dc7f9159959873877f6b46853 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Thu, 23 Jul 2020 10:55:45 +0200 Subject: [PATCH 19/47] removed useless code --- manim/scene/scene.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/manim/scene/scene.py b/manim/scene/scene.py index cd5044ebca..28897ae8f7 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -869,7 +869,6 @@ def wrapper(self, *args, **kwargs): else: hash_play = "uncached_{:05}".format(self.num_plays) self.play_hashes_list.append(hash_play) - self.revert_to_original_skipping_status() func(self, *args, **kwargs) return wrapper @@ -897,7 +896,6 @@ def wrapper(self, duration=DEFAULT_WAIT_TIME, stop_condition=None): else : hash_wait = "uncached_{:05}".format(self.num_plays) self.play_hashes_list.append(hash_wait) - self.revert_to_original_skipping_status() func(self, duration, stop_condition) return wrapper From b27a416286e94cf331313444463467ac6c1a536a Mon Sep 17 00:00:00 2001 From: Aathish Date: Tue, 28 Jul 2020 16:57:03 +0530 Subject: [PATCH 20/47] Revert "Use file_writer_config instead of config to get text_dir for Text() (#220)" This reverts commit 7f2876991d64c436ae9bd4c72f3929d169c4501a. --- manim/__main__.py | 54 +++---- manim/config.py | 5 +- manim/utils/cfg_subcmds.py | 187 ----------------------- manim/utils/cfgwriter.py | 116 ++++++++++++++ manim/utils/config_utils.py | 125 +++++---------- manim/utils/file_ops.py | 19 +-- setup.py | 1 + tests/test_cli/manim.cfg | 1 - tests/test_cli/test_cfg_subcmd.py | 52 ------- tests/test_cli/write_cfg_sbcmd_input.txt | 26 ---- 10 files changed, 173 insertions(+), 413 deletions(-) delete mode 100644 manim/utils/cfg_subcmds.py create mode 100644 manim/utils/cfgwriter.py delete mode 100644 tests/test_cli/test_cfg_subcmd.py delete mode 100644 tests/test_cli/write_cfg_sbcmd_input.txt diff --git a/manim/__main__.py b/manim/__main__.py index a860105af0..fed9b7827a 100644 --- a/manim/__main__.py +++ b/manim/__main__.py @@ -8,8 +8,7 @@ import importlib.util import types -from .config import file_writer_config,args -from .utils import cfg_subcmds +from .config import file_writer_config from .scene.scene import Scene from .utils.sounds import play_error_sound from .utils.sounds import play_finish_sound @@ -128,7 +127,7 @@ def get_module(file_name): logger.info("Enter the animation's code & end with an EOF (CTRL+D on Linux/Unix, CTRL+Z on Windows):") code = sys.stdin.read() if not code.startswith("from manim import"): - logger.warning("Didn't find an import statement for Manim. Importing automatically...") + logger.warn("Didn't find an import statement for Manim. Importing automatically...") code="from manim import *\n"+code logger.info("Rendering animation from typed code...") try: @@ -151,38 +150,23 @@ def get_module(file_name): def main(): - if hasattr(args,"subcommands"): - if "cfg" in args.subcommands: - if args.cfg_subcommand is not None: - subcommand=args.cfg_subcommand - if subcommand == "write": - cfg_subcmds.write(args.level,args.open) - elif subcommand == "show": - cfg_subcmds.show() - elif subcommand == "export": - cfg_subcmds.export(args.dir) - else: - logger.error("No argument provided; Exiting...") - - - else: - module = get_module(file_writer_config["input_file"]) - all_scene_classes = get_scene_classes_from_module(module) - scene_classes_to_render = get_scenes_to_render(all_scene_classes) - sound_on = file_writer_config["sound"] - for SceneClass in scene_classes_to_render: - try: - # By invoking, this renders the full scene - scene = SceneClass() - open_file_if_needed(scene.file_writer) - if sound_on: - play_finish_sound() - except Exception: - print("\n\n") - traceback.print_exc() - print("\n\n") - if sound_on: - play_error_sound() + module = get_module(file_writer_config["input_file"]) + all_scene_classes = get_scene_classes_from_module(module) + scene_classes_to_render = get_scenes_to_render(all_scene_classes) + sound_on = file_writer_config["sound"] + for SceneClass in scene_classes_to_render: + try: + # By invoking, this renders the full scene + scene = SceneClass() + open_file_if_needed(scene.file_writer) + if sound_on: + play_finish_sound() + except Exception: + print("\n\n") + traceback.print_exc() + print("\n\n") + if sound_on: + play_error_sound() if __name__ == "__main__": diff --git a/manim/config.py b/manim/config.py index 9ba3e55aca..5ab17cabda 100644 --- a/manim/config.py +++ b/manim/config.py @@ -89,8 +89,7 @@ def _parse_config(config_parser, args): args, config_parser, file_writer_config, successfully_read_files = _run_config() if _from_command_line(): - logger.info(f"Read configuration files: {[os.path.abspath(cfgfile) for cfgfile in successfully_read_files]}") - if not(hasattr(args,"subcommands")): - _init_dirs(file_writer_config) + logger.info(f"Read configuration files: {os.path.abspath(successfully_read_files[-1])}") + _init_dirs(file_writer_config) config = _parse_config(config_parser, args) camera_config = config diff --git a/manim/utils/cfg_subcmds.py b/manim/utils/cfg_subcmds.py deleted file mode 100644 index 08a0446ca5..0000000000 --- a/manim/utils/cfg_subcmds.py +++ /dev/null @@ -1,187 +0,0 @@ -""" -cfg_subcmd.py ------------- - -General Config File Managing Utilities. -The functions below can be called via the `manim cfg` subcommand. - -""" -import os -import configparser - -from .config_utils import _run_config, _paths_config_file, finalized_configs_dict -from .file_ops import guarantee_existence, open_file - -from rich.console import Console -from rich.progress import track -from rich.style import Style -from rich.errors import StyleSyntaxError - -__all__ = ["write","show","export"] - -RICH_COLOUR_INSTRUCTIONS = """[red]The default colour is used by the input statement. -If left empty, the default colour will be used.[/red] -[magenta] For a full list of styles, visit[/magenta] [green]https://rich.readthedocs.io/en/latest/style.html[/green]""" - -console = Console() - -def is_valid_style(style): - """Checks whether the entered color is a valid color according to rich - Parameters - ---------- - style : :class:`str` - The style to check whether it is valid. - Returns - ------- - Boolean - Returns whether it is valid style or not according to rich. - """ - try: - Style.parse(style) - return True - except StyleSyntaxError: - return False - - -def replace_keys(default): - """Replaces _ to . and viceversa in a dictionary for rich - Parameters - ---------- - default : :class:`dict` - The dictionary to check and replace - Returns - ------- - :class:`dict` - The dictionary which is modified by replcaing _ with . and viceversa - """ - for key in default: - if "_" in key: - temp = default[key] - del default[key] - key = key.replace("_", ".") - default[key] = temp - else: - temp = default[key] - del default[key] - key = key.replace(".", "_") - default[key] = temp - return default - - -def write(level=None, openfile=False): - config = _run_config()[1] - config_paths = _paths_config_file() + [os.path.abspath("manim.cfg")] - console.print("[yellow bold]Manim Configuration File Writer[/yellow bold]", justify="center") - - USER_CONFIG_MSG = f"""A configuration file at [yellow]{config_paths[1]}[/yellow] has been created with your required changes. -This will be used when running the manim command. If you want to override this config, -you will have to create a manim.cfg in the local directory, where you want those changes to be overridden.""" - - CWD_CONFIG_MSG = f"""A configuration file at [yellow]{config_paths[2]}[/yellow] has been created. -To save your theme please save that file and place it in your current working directory, from where you run the manim command.""" - - if not openfile: - action = "save this as" - - for category in config: - console.print(f"{category}",style="bold green underline") - default = config[category] - if category == "logger": - console.print(RICH_COLOUR_INSTRUCTIONS) - default = replace_keys(default) - for key in default: - console.print(f"Enter the style for {key}:", style=key, end="") - temp = input() - if temp: - while not is_valid_style(temp): - console.print("[red bold]Invalid style. Try again.[/red bold]") - console.print(f"Enter the style for {key}:", style=key, end="") - temp = input() - else: - default[key] = temp - default = replace_keys(default) - - else: - for key in default: - if default[key] in ["True","False"]: - console.print( - f"Enter value for {key} (defaults to {default[key]}):", end="") - temp = input() - if temp: - while not temp.lower().capitalize() in ["True","False"]: - console.print( - "[red bold]Invalid value. Try again.[/red bold]") - console.print( - f"Enter the style for {key}:", style=key, end="") - temp = input() - else: - default[key] = temp - config[category] = dict(default) - else: - action = "open" - - if level is None: - console.print( - f"Do you want to {action} the default config for this User?(y/n)[[n]]", - style="dim purple", - end="", - ) - action_to_userpath = input() - else: - action_to_userpath = "" - - if action_to_userpath.lower() == "y" or level=="user": - cfg_file_path = os.path.join( - guarantee_existence( - os.path.dirname(config_paths[1]) - ),"manim.cfg") - console.print(USER_CONFIG_MSG) - else: - cfg_file_path = os.path.join( - guarantee_existence( - os.path.dirname(config_paths[2]) - ),"manim.cfg") - console.print(CWD_CONFIG_MSG) - with open(cfg_file_path, "w") as fp: - config.write(fp) - if openfile: - open_file(cfg_file_path) - -def show(): - current_config = finalized_configs_dict() - for category in current_config: - console.print(f"{category}",style="bold green underline") - for entry in current_config[category]: - if category=="logger": - console.print(f"{entry} :",end="") - console.print( - f" {current_config[category][entry]}", - style=current_config[category][entry] - ) - else: - console.print(f"{entry} : {current_config[category][entry]}") - console.print("\n") - -def export(path): - config = _run_config()[1] - if os.path.abspath(path) == os.path.abspath(os.getcwd()): - console.print( - """You are reading the config from the same directory you are exporting to. -This means that the exported config will overwrite the config for this directory. -Are you sure you want to continue?[y/n]""", - style="red bold", end="" - ) - proceed = True if input().lower()=="y" else False - else: - proceed = True - if proceed: - if not os.path.isdir(path): - console.print(f"Creating folder: {path}.",style="red bold") - os.mkdir(path) - with open(os.path.join(path,"manim.cfg"),"w") as outpath: - config.write(outpath) - from_path = os.path.join(os.getcwd(),'manim.cfg') - to_path = os.path.join(path,'manim.cfg') - console.print(f"Exported final Config at {from_path} to {to_path}.") - else: - console.print("Could NOT write config.", style="red bold") diff --git a/manim/utils/cfgwriter.py b/manim/utils/cfgwriter.py new file mode 100644 index 0000000000..3b74dbeb21 --- /dev/null +++ b/manim/utils/cfgwriter.py @@ -0,0 +1,116 @@ +""" +cfgwriter.py +------------ + +Inputs the configuration files while checking it is valid. Can be executed by `manim-cfg` command. + +""" +import os +import configparser + +from .config_utils import _run_config, _paths_config_file + +from rich.console import Console +from rich.progress import track +from rich.style import Style +from rich.errors import StyleSyntaxError + +__all__ = ["main"] + +INVALID_STYLE_MSG = "[red bold]Your Style is not valid. Try again.[/red bold]" +INTRO_INSTRUCTIONS = """[red]The default colour is used by the input statement. +If left empty, the default colour will be used.[/red] +[magenta] For a full list of styles, visit[/magenta] https://rich.readthedocs.io/en/latest/style.html""" +TITLE_TEXT = "[yellow bold]Manim Configuration File Writer[/yellow bold]" + + +def is_valid_style(style): + """Checks whether the entered color is a valid color according to rich + Parameters + ---------- + style : :class:`str` + The style to check whether it is valid. + Returns + ------- + Boolean + Returns whether it is valid style or not according to rich. + """ + try: + Style.parse(style) + return True + except StyleSyntaxError: + return False + + +def replace_keys(default): + """Replaces _ to . and viceversa in a dictionary for rich + Parameters + ---------- + default : :class:`dict` + The dictionary to check and replace + Returns + ------- + :class:`dict` + The dictionary which is modified by replcaing _ with . and viceversa + """ + for key in default: + if "_" in key: + temp = default[key] + del default[key] + key = key.replace("_", ".") + default[key] = temp + else: + temp = default[key] + del default[key] + key = key.replace(".", "_") + default[key] = temp + return default + + +def main(): + config = _run_config()[1] + console = Console() + default = config["logger"] + console.print(TITLE_TEXT, justify="center") + console.print(INTRO_INSTRUCTIONS) + default = replace_keys(default) + for key in default: + console.print("Enter the Style for %s" % key + ":", style=key, end="") + temp = input() + if temp: + while not is_valid_style(temp): + console.print(INVALID_STYLE_MSG) + console.print("Enter the Style for %s" % key + ":", style=key, end="") + temp = input() + else: + default[key] = temp + default = replace_keys(default) + config["logger"] = default + console.print( + "Do you want to save this as the default for this User?(y/n)[[n]]", + style="dim purple", + end="", + ) + save_to_userpath = input() + config_paths = _paths_config_file() + [os.path.abspath("manim.cfg")] + if save_to_userpath.lower() == "y": + if not os.path.exists(os.path.abspath(os.path.join(config_paths[1], ".."))): + os.makedirs(os.path.abspath(os.path.join(config_paths[1], ".."))) + with open(config_paths[1], "w") as fp: + config.write(fp) + console.print( + f"""A configuration file called [yellow]{config_paths[1]}[/yellow] has been created with your required changes. +This will be used when running the manim command. If you want to override this config, +you will have to create a manim.cfg in the local directory, where you want those changes to be overridden.""" + ) + else: + with open(config_paths[2], "w") as fp: + config.write(fp) + console.print( + f"""A configuration file called [yellow]{config_paths[2]}[/yellow] has been created. +To save your theme please save that file and place it in your current working directory, from where you run the manim command.""" + ) + + +if __name__ == "__main__": + main() diff --git a/manim/utils/config_utils.py b/manim/utils/config_utils.py index 2f9d989c84..56af80f9e4 100644 --- a/manim/utils/config_utils.py +++ b/manim/utils/config_utils.py @@ -16,9 +16,9 @@ from .. import constants from .tex import TexTemplate, TexTemplateFromFile -__all__ = ["_run_config", "_paths_config_file", "_from_command_line", "finalized_configs_dict"] +__all__ = ["_run_config", "_paths_config_file", "_from_command_line"] + -min_argvs = 3 if "-m" in sys.argv[0] else 2 def _parse_file_writer_config(config_parser, args): """Parse config files and CLI arguments into a single dictionary.""" # By default, use the CLI section of the digested .cfg files @@ -28,13 +28,10 @@ def _parse_file_writer_config(config_parser, args): fw_config = {} # Handle input files and scenes. Note these cannot be set from - # the .cfg files, only from CLI arguments. - # If a subcommand is given, manim will not render a video and - # thus these specific input/output files are not needed. - if not(hasattr(args,"subcommands")): - fw_config["input_file"] = args.file - fw_config["scene_names"] = args.scene_names if args.scene_names is not None else [] - fw_config["output_file"] = args.output_file + # the .cfg files, only from CLI arguments + fw_config["input_file"] = args.file + fw_config["scene_names"] = args.scene_names if args.scene_names is not None else [] + fw_config["output_file"] = args.output_file # Handle all options that are directly overridden by CLI # arguments. Note ConfigParser options are all strings and each @@ -137,49 +134,22 @@ def _parse_cli(arg_list, input=True): epilog="Made with <3 by the manim community devs", ) if input: - # If the only command is `manim`, we want both subcommands like `cfg` - # and mandatory positional arguments like `file` to show up in the help section. - if len(sys.argv) == min_argvs-1 or _subcommands_exist(): - subparsers = parser.add_subparsers(dest="subcommands") - cfg_related = subparsers.add_parser('cfg') - cfg_subparsers = cfg_related.add_subparsers(dest="cfg_subcommand") - - cfg_write_parser = cfg_subparsers.add_parser('write') - cfg_write_parser.add_argument( - "--level", - choices=["user", "cwd"], - default=None, - help="Specify if this config is for user or just the working directory." - ) - cfg_write_parser.add_argument( - "--open", - action="store_const", - const=True, - default = False - ) - cfg_subparsers.add_parser('show') - - cfg_export_parser = cfg_subparsers.add_parser("export") - cfg_export_parser.add_argument("--dir",default=os.getcwd()) - - if (len(sys.argv) == min_argvs-1 or - not _subcommands_exist(ignore = ["--help","-h"])): - parser.add_argument( - "file", help="path to file holding the python code for the scene", - ) - parser.add_argument( - "scene_names", - nargs="*", - help="Name of the Scene class you want to see", - default=[""], - ) - parser.add_argument( - "-o", - "--output_file", - help="Specify the name of the output file, if " - "it should be different from the scene class name", - default="", - ) + parser.add_argument( + "file", help="path to file holding the python code for the scene", + ) + parser.add_argument( + "scene_names", + nargs="*", + help="Name of the Scene class you want to see", + default=[""], + ) + parser.add_argument( + "-o", + "--output_file", + help="Specify the name of the output file, if " + "it should be different from the scene class name", + default="", + ) # The following use (action='store_const', const=True) instead of # the built-in (action='store_true'). This is because the latter @@ -357,14 +327,8 @@ def _parse_cli(arg_list, input=True): parser.add_argument( "--config_file", help="Specify the configuration file", ) - parsed=parser.parse_args(arg_list) - if hasattr(parsed,"subcommands"): - setattr(parsed, "cfg_subcommand", - cfg_related.parse_args( - sys.argv[min_argvs -(0 if min_argvs == 2 else 1):] - ).cfg_subcommand) - return parsed + return parser.parse_args(arg_list) def _init_dirs(config): @@ -378,6 +342,7 @@ def _init_dirs(config): if not os.path.exists(folder): os.makedirs(folder) + def _from_command_line(): """Determine if manim was called from the command line.""" # Manim can be called from the command line in three different @@ -393,11 +358,6 @@ def _from_command_line(): return from_cli_command or from_python_m -def _from_dunder_main(): - dunder_main_path = os.path.join( - os.path.dirname(os.path.dirname(__file__)), - "__main__.py") - return sys.argv[0]==dunder_main_path def _paths_config_file(): library_wide = os.path.abspath( @@ -425,24 +385,19 @@ def _paths_config_file(): def _run_config(): # Config files to be parsed, in ascending priority config_files = _paths_config_file() - if _from_command_line() or _from_dunder_main(): + if _from_command_line(): args = _parse_cli(sys.argv[1:]) - if not hasattr(args,"subcommands"): - if args.config_file is not None: - if os.path.exists(args.config_file): - config_files.append(args.config_file) - else: - raise FileNotFoundError(f"Config file {args.config_file} doesn't exist") + if args.config_file is not None: + if os.path.exists(args.config_file): + config_files.append(args.config_file) else: - script_directory_file_config = os.path.join( - os.path.dirname(args.file), "manim.cfg" - ) - if os.path.exists(script_directory_file_config): - config_files.append(script_directory_file_config) + raise FileNotFoundError(f"Config file {args.config_file} doesn't exist") else: - working_directory_file_config = os.path.join(os.getcwd(),"manim.cfg") - if os.path.exists(working_directory_file_config): - config_files.append(working_directory_file_config) + script_directory_file_config = os.path.join( + os.path.dirname(args.file), "manim.cfg" + ) + if os.path.exists(script_directory_file_config): + config_files.append(script_directory_file_config) else: # In this case, we still need an empty args object. @@ -456,15 +411,3 @@ def _run_config(): # this is for internal use when writing output files file_writer_config = _parse_file_writer_config(config_parser, args) return args, config_parser, file_writer_config, successfully_read_files - -def finalized_configs_dict(): - config=_run_config()[1] - return {section: dict(config[section]) for section in config.sections()} - -def _subcommands_exist(ignore = []): - NON_ANIM_UTILS = ["cfg","--help","-h"] - NON_ANIM_UTILS = [util for util in NON_ANIM_UTILS if util not in ignore] - - not_only_manim = len(sys.argv) > min_argvs-1 - sub_command_exists = any(a == item for a in sys.argv for item in NON_ANIM_UTILS) - return not_only_manim and sub_command_exists diff --git a/manim/utils/file_ops.py b/manim/utils/file_ops.py index 1dad6e6ebf..170f7be83e 100644 --- a/manim/utils/file_ops.py +++ b/manim/utils/file_ops.py @@ -1,6 +1,4 @@ import os -import subprocess as sp -import platform import numpy as np import time @@ -38,19 +36,4 @@ def modify_atime(file_path): file_path : :class:`str` The path of the file. """ - os.utime(file_path, times=(time.time(),os.path.getmtime(file_path))) - -def open_file(file_path): - current_os = platform.system() - if current_os == "Windows": - os.startfile(file_path) - else: - commands = [] - if current_os == "Linux": - commands.append("xdg-open") - elif current_os.startswith("CYGWIN"): - commands.append("cygstart") - else: # Assume macOS - commands.append("open") - commands.append(file_path) - sp.Popen(commands) + os.utime(file_path, times=(time.time(),os.path.getmtime(file_path))) \ No newline at end of file diff --git a/setup.py b/setup.py index af84f44ad7..2d081bb07e 100755 --- a/setup.py +++ b/setup.py @@ -11,6 +11,7 @@ "console_scripts": [ "manim=manim.__main__:main", "manimcm=manim.__main__:main", + "manim-cfg=manim.utils.cfgwriter:main", ] }, install_requires=[ diff --git a/tests/test_cli/manim.cfg b/tests/test_cli/manim.cfg index 1838b30c24..b71cfe2352 100644 --- a/tests/test_cli/manim.cfg +++ b/tests/test_cli/manim.cfg @@ -1,7 +1,6 @@ [CLI] movie_file_extension = .mp4 write_to_movie = True -sound = True # write_all = False save_last_frame = True # save_pngs = False diff --git a/tests/test_cli/test_cfg_subcmd.py b/tests/test_cli/test_cfg_subcmd.py deleted file mode 100644 index 2d88f4b0dc..0000000000 --- a/tests/test_cli/test_cfg_subcmd.py +++ /dev/null @@ -1,52 +0,0 @@ -import pytest -import os -import shutil - -from test_cli import capture - -def test_cfg_help(python_version): - """Test if Manim successfully adds configparsers when a subcommand is invoked.""" - os.chdir(os.path.dirname(__file__)) - command = [python_version, "-m", "manim", "cfg", "--help"] - out, err, exitcode = capture(command) - assert exitcode == 0, f"The cfg subcommand is not working as intended." - -def test_cfg_show(python_version): - """Test if the `manim cfg show` command works as intended.""" - command = [python_version, "-m", "manim", "cfg", "show"] - out, err, exitcode = capture(command) - assert exitcode == 0 - assert f"{os.path.sep}tests{os.path.sep}".encode("utf-8") in out, err - -def test_cfg_export(python_version): - """Test if the `manim cfg export` command works as intended.""" - command = [python_version, "-m", "manim", "cfg", "export", "--dir", "temp"] - out, err, exitcode = capture(command) - assert exitcode == 0 - assert os.path.exists(os.path.join("temp","manim.cfg")) - with open(os.path.join("temp","manim.cfg"),"r") as writtencfg: - assert "sound = True" in writtencfg.read(), err - shutil.rmtree("temp") - -def test_cfg_write(python_version): - """Simulate using the command `manim cfg write`""" - cfgfilepath = os.path.join(os.path.dirname(__file__), "manim.cfg") - command = [python_version, "-m", "manim","cfg","write","--level","cwd"] - - """As the number of config values that `manim cfg write` can modify increases, so - must the number of newlines and/or values written in write_cfg_sbcmd_input increase.""" - with open(cfgfilepath) as cfgfile: - original = cfgfile.read() - - out, err, exitcode = capture( - command, - open(os.path.join(os.path.dirname(__file__), "write_cfg_sbcmd_input.txt")) - ) - assert exitcode == 0, err - - with open(cfgfilepath,"r") as cfgfile: - assert "sound = False" in cfgfile.read() - - with open(cfgfilepath,"w") as cfgfile: - cfgfile.write(original) - os.chdir(os.path.abspath(os.path.join(__file__, os.pardir, os.pardir, os.pardir))) \ No newline at end of file diff --git a/tests/test_cli/write_cfg_sbcmd_input.txt b/tests/test_cli/write_cfg_sbcmd_input.txt deleted file mode 100644 index 110779f7a8..0000000000 --- a/tests/test_cli/write_cfg_sbcmd_input.txt +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - -False - - - - - - - - - - - - - - - - - From d52d9f364d59d98c22a1e174d1579723bea2e396 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Tue, 28 Jul 2020 14:00:56 +0200 Subject: [PATCH 21/47] Apply suggestions from code review Thanks @pgsuper ! Co-authored-by: Pg Biel <9021226+PgBiel@users.noreply.github.com> --- manim/default.cfg | 2 +- manim/scene/scene_file_writer.py | 10 ++++----- manim/utils/config_utils.py | 4 ++-- manim/utils/hashing.py | 38 ++++++++++++++++---------------- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/manim/default.cfg b/manim/default.cfg index b73f7dc658..4f0eecc57f 100644 --- a/manim/default.cfg +++ b/manim/default.cfg @@ -88,7 +88,7 @@ frame_rate = 60 pixel_height = 1440 pixel_width = 2560 -max_file_cached = 7 +max_files_cached = 7 flush_cache = False disable_caching = False diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index cf91bbbc45..0cfe050e87 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -363,7 +363,7 @@ def finish(self): self.combine_movie_files() if file_writer_config['flush_cache']: self.flush_cache_directory() - else : + else: self.clean_cache() if file_writer_config['save_last_frame']: self.scene.update_frame(ignore_skipping=True) @@ -438,8 +438,8 @@ def is_already_cached(self, hash_invokation): Returns ------- - `bool` - Wether the file exists. + :class:`bool` + Whether the file exists. """ path = os.path.join(self.partial_movie_directory, "{}{}".format( hash_invokation, self.movie_file_extension)) @@ -548,12 +548,12 @@ def clean_cache(self): if len(cached_partial_movies) > file_writer_config['max_file_cached']: oldest_file_path = min(cached_partial_movies, key=os.path.getatime) logger.info( - f"Partial movie directory is full. (> {file_writer_config['max_file_cached']} files). Manim removed the file used by manim the longest ago. ({os.path.basename(oldest_file_path)})." + f"The partial movie directory is full (> {file_writer_config['max_file_cached']} files). Therefore, manim has removed the file used by it the longest ago ({os.path.basename(oldest_file_path)})." + "You can change this behaviour by changing max_file_cached in config.") os.remove(oldest_file_path) def flush_cache_directory(self): - """Delete all the partial movie files cached""" + """Delete all the cached partial movie files""" cached_partial_movies = [os.path.join(self.partial_movie_directory, file_name) for file_name in os.listdir( self.partial_movie_directory) if file_name != "partial_movie_file_list.txt"] for f in cached_partial_movies: diff --git a/manim/utils/config_utils.py b/manim/utils/config_utils.py index 56af80f9e4..881e1a2498 100644 --- a/manim/utils/config_utils.py +++ b/manim/utils/config_utils.py @@ -102,7 +102,7 @@ def _parse_file_writer_config(config_parser, args): "write_all", ]: fw_config[opt] = config_parser["dry_run"].getboolean(opt) - if not fw_config['write_to_movie'] : + if not fw_config['write_to_movie']: fw_config['disable_caching'] = True # Read in the streaming section -- all values are strings fw_config["streaming"] = { @@ -230,7 +230,7 @@ def _parse_cli(arg_list, input=True): "--flush_cache", action="store_const", const=True, - help="Remove all partial-movie-files cached.", + help="Remove all cached partial-movie-files.", ) # The default value of the following is set in manim.cfg parser.add_argument( diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py index bda6585ddf..e7a3dc80bf 100644 --- a/manim/utils/hashing.py +++ b/manim/utils/hashing.py @@ -48,12 +48,12 @@ def default(self, obj): return json.JSONEncoder.default(self, obj) except TypeError: # This is used when the user enters an unknown type in CONFIG. Rather than throwing an error, we transform - # it into a string "Unsupported type for hashing" so it won't affect the hash. + # it into a string "Unsupported type for hashing" so that it won't affect the hash. return "Unsupported type for hashing" def get_json(object): - """Recursively serialize object to JSON. Use CustomEncoder class above. + """Recursively serialize `object` to JSON using the :class:`CustomEncoder` class. Paramaters ---------- @@ -62,52 +62,52 @@ def get_json(object): Returns ------- - `str` - the object flattened + :class:`str` + The flattened object """ return json.dumps(object, cls=CustomEncoder) def get_camera_dict_for_hashing(camera_object): - """Remove some keys from cameraobject.__dict__ that are useless for the caching functionnality and very heavy. + """Remove some keys from `camera_object.__dict__` that are very heavy and useless for the caching functionality. Parameters ---------- - object_camera : :class:``~.Camera` + camera_object : :class:`~.Camera` The camera object used in the scene Returns ------- - `dict` - Camera.__dict__ but cleaned. + :class:`dict` + `Camera.__dict__` but cleaned. """ camera_object_dict = copy.copy(camera_object.__dict__) - # We have to clean a little bit camera_dict, as pixel_array and background are two very big numpy array. + # We have to clean a little bit of camera_dict, as pixel_array and background are two very big numpy arrays. # They are not essential to caching process. - # We also have to remove pixel_array_to_cairo_context as it conntains uses memory adress (set randomly). See l.516 get_cached_cairo_context in camera.py + # We also have to remove pixel_array_to_cairo_context as it contains used memory adress (set randomly). See l.516 get_cached_cairo_context in camera.py for to_clean in ['background', 'pixel_array', 'pixel_array_to_cairo_context']: camera_object_dict.pop(to_clean, None) return camera_object_dict def get_hash_from_play_call(camera_object, animations_list, current_mobjects_list): - """Take the list of animations and a list of mobjects and output their hash. Is meant to be used for `scene.play` function. + """Take the list of animations and a list of mobjects and output their hashes. This is meant to be used for `scene.play` function. Parameters ----------- - object_camera : :class:``~.Camera` - The camera object used in the scene + camera_object : :class:`~.Camera` + The camera object used in the scene. animations_list : :class:`list` - The list of animations + The list of animations. current_mobjects_list : :class:`list` The list of mobjects. Returns ------- - `str` - concatenation of the hash of object_camera, animations_list and current_mobjects_list separated by '_'. + :class:`str` + A string concatenation of the respective hashes of `camera_object`, `animations_list` and `current_mobjects_list`, separated by `_`. """ camera_json = get_json(get_camera_dict_for_hashing(camera_object)) animations_list_json = [get_json(x) for x in sorted( @@ -122,7 +122,7 @@ def get_hash_from_play_call(camera_object, animations_list, current_mobjects_lis def get_hash_from_wait_call(camera_object, wait_time, stop_condition_function, current_mobjects_list): - """Take a wait time, a boolean function as stop_condition and a list of mobjects output their hash. Is meant to be used for `scene.wait` function. + """Take a wait time, a boolean function as a stop condition and a list of mobjects, and then output their individual hashes. This is meant to be used for `scene.wait` function. Parameters ----------- @@ -134,8 +134,8 @@ def get_hash_from_wait_call(camera_object, wait_time, stop_condition_function, c Returns ------- - `str` - concatenation of the hash of animations_list and current_mobjects_list separated by '_'. + :class:`str` + A concatenation of the respective hashes of `animations_list and `current_mobjects_list`, separated by `_`. """ camera_json = get_json(get_camera_dict_for_hashing(camera_object)) current_mobjects_list_json = [get_json(x) for x in sorted( From cdfbfa988e044e11f943f7f3e9d3572543aec6b9 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Tue, 28 Jul 2020 14:27:20 +0200 Subject: [PATCH 22/47] fixed max_files_cached typo --- manim/scene/scene_file_writer.py | 6 +++--- manim/utils/config_utils.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index 0cfe050e87..d132867b87 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -545,11 +545,11 @@ def clean_cache(self): """Will clean the cache by removing the partial_move used by manim the longest ago.""" cached_partial_movies = [os.path.join(self.partial_movie_directory, file_name) for file_name in os.listdir( self.partial_movie_directory) if file_name != "partial_movie_file_list.txt"] - if len(cached_partial_movies) > file_writer_config['max_file_cached']: + if len(cached_partial_movies) > file_writer_config['max_files_cached']: oldest_file_path = min(cached_partial_movies, key=os.path.getatime) logger.info( - f"The partial movie directory is full (> {file_writer_config['max_file_cached']} files). Therefore, manim has removed the file used by it the longest ago ({os.path.basename(oldest_file_path)})." - + "You can change this behaviour by changing max_file_cached in config.") + f"The partial movie directory is full (> {file_writer_config['max_files_cached']} files). Therefore, manim has removed the file used by it the longest ago ({os.path.basename(oldest_file_path)})." + + "You can change this behaviour by changing max_files_cached in config.") os.remove(oldest_file_path) def flush_cache_directory(self): diff --git a/manim/utils/config_utils.py b/manim/utils/config_utils.py index 881e1a2498..483025d970 100644 --- a/manim/utils/config_utils.py +++ b/manim/utils/config_utils.py @@ -124,7 +124,7 @@ def _parse_file_writer_config(config_parser, args): fw_config["skip_animations"] = any( [fw_config["save_last_frame"], fw_config["from_animation_number"]] ) - fw_config['max_file_cached'] = default.getint('max_file_cached') + fw_config['max_files_cached'] = default.getint('max_files_cached') return fw_config From 8499535ffde6d72cc430920f043e4fe5a8adec30 Mon Sep 17 00:00:00 2001 From: Aathish Date: Sun, 26 Jul 2020 18:29:50 +0530 Subject: [PATCH 23/47] Use file_writer_config instead of config to get text_dir for Text() (#220) * Use file_writer_config instead if config for Text() Fixes a bug where a keyerror would be thrown when rendering a Text() mobject. * Make testing_utils.py use file_writer_config. Thank you @huguesdevimeux ! Co-authored-by: Aathish Sivasubrahmanian --- manim/mobject/svg/text_mobject.py | 7 +++---- tests/testing_utils.py | 2 +- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/manim/mobject/svg/text_mobject.py b/manim/mobject/svg/text_mobject.py index 74751d92e0..386f9433c4 100644 --- a/manim/mobject/svg/text_mobject.py +++ b/manim/mobject/svg/text_mobject.py @@ -5,7 +5,7 @@ import cairo from ...constants import * -from ...config import config +from ...config import config,file_writer_config from ...container.container import Container from ...logger import logger from ...mobject.geometry import Dot, Rectangle @@ -96,7 +96,7 @@ def __init__(self, text, **config): def get_space_width(self): size = self.size * 10 - dir_name = config['text_dir'] + dir_name = file_writer_config['text_dir'] file_name = os.path.join(dir_name, "space") + '.svg' surface = cairo.SVGSurface(file_name, 600, 400) @@ -291,8 +291,7 @@ def text2svg(self): if self.font == '': if NOT_SETTING_FONT_MSG != '': logger.warning(NOT_SETTING_FONT_MSG) - - dir_name = config['text_dir'] + dir_name = file_writer_config['text_dir'] hash_name = self.text2hash() file_name = os.path.join(dir_name, hash_name)+'.svg' if os.path.exists(file_name): diff --git a/tests/testing_utils.py b/tests/testing_utils.py index 8cdeeb021b..fcdc9d8bcc 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -38,7 +38,7 @@ def __init__(self, scene_object, module_tested, caching_needed=False): self.path_tests_data = os.path.join('tests', 'tests_data', module_tested) if caching_needed: - config['text_dir'] = os.path.join( + file_writer_config['text_dir'] = os.path.join( self.path_tests_medias_cache, scene_object.__name__, 'Text') file_writer_config['tex_dir'] = os.path.join( self.path_tests_medias_cache, scene_object.__name__, 'Tex') From 0ace9a604a05ba0d5817aaed50ee140d23b3f635 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Thu, 23 Jul 2020 10:55:45 +0200 Subject: [PATCH 24/47] removed useless code --- manim/scene/scene.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/manim/scene/scene.py b/manim/scene/scene.py index cd5044ebca..28897ae8f7 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -869,7 +869,6 @@ def wrapper(self, *args, **kwargs): else: hash_play = "uncached_{:05}".format(self.num_plays) self.play_hashes_list.append(hash_play) - self.revert_to_original_skipping_status() func(self, *args, **kwargs) return wrapper @@ -897,7 +896,6 @@ def wrapper(self, duration=DEFAULT_WAIT_TIME, stop_condition=None): else : hash_wait = "uncached_{:05}".format(self.num_plays) self.play_hashes_list.append(hash_wait) - self.revert_to_original_skipping_status() func(self, duration, stop_condition) return wrapper From b1a378e2d2a8f628231ee0fdd128401cffbd5350 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Tue, 28 Jul 2020 14:34:58 +0200 Subject: [PATCH 25/47] used self.camera instead of __dict__ --- manim/scene/scene.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/manim/scene/scene.py b/manim/scene/scene.py index 28897ae8f7..d0a5fd09ad 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -861,7 +861,7 @@ def wrapper(self, *args, **kwargs): self.add_mobjects_from_animations(animations) if not file_writer_config['disable_caching']: mobjects_on_scene = self.get_mobjects() - hash_play = get_hash_from_play_call(self.__dict__['camera'], animations, mobjects_on_scene) + hash_play = get_hash_from_play_call(self.camera, animations, mobjects_on_scene) self.play_hashes_list.append(hash_play) if self.file_writer.is_already_cached(hash_play): logger.info(f'Animation {self.num_plays} : Using cached data (hash : {hash_play})') @@ -888,7 +888,7 @@ def handle_caching_wait(func): def wrapper(self, duration=DEFAULT_WAIT_TIME, stop_condition=None): self.revert_to_original_skipping_status() if not file_writer_config['disable_caching']: - hash_wait = get_hash_from_wait_call(self.__dict__['camera'], duration, stop_condition, self.get_mobjects()) + hash_wait = get_hash_from_wait_call(self.camera, duration, stop_condition, self.get_mobjects()) self.play_hashes_list.append(hash_wait) if self.file_writer.is_already_cached(hash_wait): logger.info(f'Wait {self.num_plays} : Using cached data (hash : {hash_wait})') From 2c95a7ee99578d7841eb60405ab055cf541a972e Mon Sep 17 00:00:00 2001 From: Aathish Date: Tue, 28 Jul 2020 18:05:27 +0530 Subject: [PATCH 26/47] Apply suggestions from code review Thanks @pgsuper ! Co-authored-by: Pg Biel <9021226+PgBiel@users.noreply.github.com> --- manim/default.cfg | 2 +- manim/scene/scene_file_writer.py | 10 ++++----- manim/utils/config_utils.py | 4 ++-- manim/utils/hashing.py | 38 ++++++++++++++++---------------- 4 files changed, 27 insertions(+), 27 deletions(-) diff --git a/manim/default.cfg b/manim/default.cfg index b73f7dc658..4f0eecc57f 100644 --- a/manim/default.cfg +++ b/manim/default.cfg @@ -88,7 +88,7 @@ frame_rate = 60 pixel_height = 1440 pixel_width = 2560 -max_file_cached = 7 +max_files_cached = 7 flush_cache = False disable_caching = False diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index cf91bbbc45..0cfe050e87 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -363,7 +363,7 @@ def finish(self): self.combine_movie_files() if file_writer_config['flush_cache']: self.flush_cache_directory() - else : + else: self.clean_cache() if file_writer_config['save_last_frame']: self.scene.update_frame(ignore_skipping=True) @@ -438,8 +438,8 @@ def is_already_cached(self, hash_invokation): Returns ------- - `bool` - Wether the file exists. + :class:`bool` + Whether the file exists. """ path = os.path.join(self.partial_movie_directory, "{}{}".format( hash_invokation, self.movie_file_extension)) @@ -548,12 +548,12 @@ def clean_cache(self): if len(cached_partial_movies) > file_writer_config['max_file_cached']: oldest_file_path = min(cached_partial_movies, key=os.path.getatime) logger.info( - f"Partial movie directory is full. (> {file_writer_config['max_file_cached']} files). Manim removed the file used by manim the longest ago. ({os.path.basename(oldest_file_path)})." + f"The partial movie directory is full (> {file_writer_config['max_file_cached']} files). Therefore, manim has removed the file used by it the longest ago ({os.path.basename(oldest_file_path)})." + "You can change this behaviour by changing max_file_cached in config.") os.remove(oldest_file_path) def flush_cache_directory(self): - """Delete all the partial movie files cached""" + """Delete all the cached partial movie files""" cached_partial_movies = [os.path.join(self.partial_movie_directory, file_name) for file_name in os.listdir( self.partial_movie_directory) if file_name != "partial_movie_file_list.txt"] for f in cached_partial_movies: diff --git a/manim/utils/config_utils.py b/manim/utils/config_utils.py index 6ef972a361..b6d8f7dc3b 100644 --- a/manim/utils/config_utils.py +++ b/manim/utils/config_utils.py @@ -102,7 +102,7 @@ def _parse_file_writer_config(config_parser, args): "write_all", ]: fw_config[opt] = config_parser["dry_run"].getboolean(opt) - if not fw_config['write_to_movie'] : + if not fw_config['write_to_movie']: fw_config['disable_caching'] = True # Read in the streaming section -- all values are strings fw_config["streaming"] = { @@ -230,7 +230,7 @@ def _parse_cli(arg_list, input=True): "--flush_cache", action="store_const", const=True, - help="Remove all partial-movie-files cached.", + help="Remove all cached partial-movie-files.", ) # The default value of the following is set in manim.cfg parser.add_argument( diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py index bda6585ddf..e7a3dc80bf 100644 --- a/manim/utils/hashing.py +++ b/manim/utils/hashing.py @@ -48,12 +48,12 @@ def default(self, obj): return json.JSONEncoder.default(self, obj) except TypeError: # This is used when the user enters an unknown type in CONFIG. Rather than throwing an error, we transform - # it into a string "Unsupported type for hashing" so it won't affect the hash. + # it into a string "Unsupported type for hashing" so that it won't affect the hash. return "Unsupported type for hashing" def get_json(object): - """Recursively serialize object to JSON. Use CustomEncoder class above. + """Recursively serialize `object` to JSON using the :class:`CustomEncoder` class. Paramaters ---------- @@ -62,52 +62,52 @@ def get_json(object): Returns ------- - `str` - the object flattened + :class:`str` + The flattened object """ return json.dumps(object, cls=CustomEncoder) def get_camera_dict_for_hashing(camera_object): - """Remove some keys from cameraobject.__dict__ that are useless for the caching functionnality and very heavy. + """Remove some keys from `camera_object.__dict__` that are very heavy and useless for the caching functionality. Parameters ---------- - object_camera : :class:``~.Camera` + camera_object : :class:`~.Camera` The camera object used in the scene Returns ------- - `dict` - Camera.__dict__ but cleaned. + :class:`dict` + `Camera.__dict__` but cleaned. """ camera_object_dict = copy.copy(camera_object.__dict__) - # We have to clean a little bit camera_dict, as pixel_array and background are two very big numpy array. + # We have to clean a little bit of camera_dict, as pixel_array and background are two very big numpy arrays. # They are not essential to caching process. - # We also have to remove pixel_array_to_cairo_context as it conntains uses memory adress (set randomly). See l.516 get_cached_cairo_context in camera.py + # We also have to remove pixel_array_to_cairo_context as it contains used memory adress (set randomly). See l.516 get_cached_cairo_context in camera.py for to_clean in ['background', 'pixel_array', 'pixel_array_to_cairo_context']: camera_object_dict.pop(to_clean, None) return camera_object_dict def get_hash_from_play_call(camera_object, animations_list, current_mobjects_list): - """Take the list of animations and a list of mobjects and output their hash. Is meant to be used for `scene.play` function. + """Take the list of animations and a list of mobjects and output their hashes. This is meant to be used for `scene.play` function. Parameters ----------- - object_camera : :class:``~.Camera` - The camera object used in the scene + camera_object : :class:`~.Camera` + The camera object used in the scene. animations_list : :class:`list` - The list of animations + The list of animations. current_mobjects_list : :class:`list` The list of mobjects. Returns ------- - `str` - concatenation of the hash of object_camera, animations_list and current_mobjects_list separated by '_'. + :class:`str` + A string concatenation of the respective hashes of `camera_object`, `animations_list` and `current_mobjects_list`, separated by `_`. """ camera_json = get_json(get_camera_dict_for_hashing(camera_object)) animations_list_json = [get_json(x) for x in sorted( @@ -122,7 +122,7 @@ def get_hash_from_play_call(camera_object, animations_list, current_mobjects_lis def get_hash_from_wait_call(camera_object, wait_time, stop_condition_function, current_mobjects_list): - """Take a wait time, a boolean function as stop_condition and a list of mobjects output their hash. Is meant to be used for `scene.wait` function. + """Take a wait time, a boolean function as a stop condition and a list of mobjects, and then output their individual hashes. This is meant to be used for `scene.wait` function. Parameters ----------- @@ -134,8 +134,8 @@ def get_hash_from_wait_call(camera_object, wait_time, stop_condition_function, c Returns ------- - `str` - concatenation of the hash of animations_list and current_mobjects_list separated by '_'. + :class:`str` + A concatenation of the respective hashes of `animations_list and `current_mobjects_list`, separated by `_`. """ camera_json = get_json(get_camera_dict_for_hashing(camera_object)) current_mobjects_list_json = [get_json(x) for x in sorted( From 4844847672fcfa8b0a2c44fdf49bad0e83ea0b06 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Tue, 28 Jul 2020 14:37:24 +0200 Subject: [PATCH 27/47] more pythonic --- manim/utils/hashing.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py index e7a3dc80bf..8edc58a31f 100644 --- a/manim/utils/hashing.py +++ b/manim/utils/hashing.py @@ -114,10 +114,10 @@ def get_hash_from_play_call(camera_object, animations_list, current_mobjects_lis animations_list, key=lambda obj: str(obj))] current_mobjects_list_json = [get_json(x) for x in sorted( current_mobjects_list, key=lambda obj: str(obj))] - hash_camera = zlib.crc32(repr(camera_json).encode()) - hash_animations = zlib.crc32(repr(animations_list_json).encode()) - hash_current_mobjects = zlib.crc32( - repr(current_mobjects_list_json).encode()) + hash_camera, hash_animations, hash_current_mobjects = [ + zlib.crc32(repr(json_val).encode()) + for json_val in [camera_json, animations_list_json, current_mobjects_list_json] + ] return "{}_{}_{}".format(hash_camera, hash_animations, hash_current_mobjects) From 54274911ede6c49d34ad2845a0599955eb2691ed Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Wed, 29 Jul 2020 19:32:39 +0200 Subject: [PATCH 28/47] fixed bug related to dict keys When a dict containing keys not of the right type it caused an error. thanks to @Aatish04. --- manim/utils/hashing.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py index 8edc58a31f..0d5c9c1d40 100644 --- a/manim/utils/hashing.py +++ b/manim/utils/hashing.py @@ -41,7 +41,14 @@ def default(self, obj): elif isinstance(obj, np.ndarray): return list(obj) elif hasattr(obj, "__dict__"): - return getattr(obj, '__dict__') + temp = getattr(obj, '__dict__') + # As dict keys must be of the type (str, int, float, bool), we have to clean them. + # To do that, if one is not of the good type we turn it into its hash using the same + # method as all the objects here. + def key_to_hash(key): + if not isinstance(key, (str, int, float, bool)) and key is not None : + return zlib.crc32(json.dumps(key, cls=self).encode()) + return {key_to_hash(k) : i for k,i in temp.items()} elif isinstance(obj, np.uint8): return int(obj) try: @@ -109,14 +116,14 @@ def get_hash_from_play_call(camera_object, animations_list, current_mobjects_lis :class:`str` A string concatenation of the respective hashes of `camera_object`, `animations_list` and `current_mobjects_list`, separated by `_`. """ - camera_json = get_json(get_camera_dict_for_hashing(camera_object)) + # camera_json = get_json(get_camera_dict_for_hashing(camera_object)) animations_list_json = [get_json(x) for x in sorted( animations_list, key=lambda obj: str(obj))] current_mobjects_list_json = [get_json(x) for x in sorted( current_mobjects_list, key=lambda obj: str(obj))] hash_camera, hash_animations, hash_current_mobjects = [ zlib.crc32(repr(json_val).encode()) - for json_val in [camera_json, animations_list_json, current_mobjects_list_json] + for json_val in [['to_delete'], animations_list_json, current_mobjects_list_json] ] return "{}_{}_{}".format(hash_camera, hash_animations, hash_current_mobjects) From f2e924b7897243c92212132774705db7fc21d282 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Wed, 29 Jul 2020 20:15:29 +0200 Subject: [PATCH 29/47] fixed minor bufg of the last commit --- manim/utils/hashing.py | 1 + 1 file changed, 1 insertion(+) diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py index 0d5c9c1d40..308e7367e4 100644 --- a/manim/utils/hashing.py +++ b/manim/utils/hashing.py @@ -48,6 +48,7 @@ def default(self, obj): def key_to_hash(key): if not isinstance(key, (str, int, float, bool)) and key is not None : return zlib.crc32(json.dumps(key, cls=self).encode()) + return key return {key_to_hash(k) : i for k,i in temp.items()} elif isinstance(obj, np.uint8): return int(obj) From 901934d03c5ba9378c5925c6279a57cd09db1b95 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Wed, 29 Jul 2020 20:29:23 +0200 Subject: [PATCH 30/47] Now clean_cache can remove multiple files. --- manim/scene/scene_file_writer.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index d132867b87..636bfa1879 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -542,15 +542,18 @@ def combine_movie_files(self): modify_atime(file_path) def clean_cache(self): - """Will clean the cache by removing the partial_move used by manim the longest ago.""" + """Will clean the cache by removing the partial_movie_files used by manim the longest ago.""" cached_partial_movies = [os.path.join(self.partial_movie_directory, file_name) for file_name in os.listdir( self.partial_movie_directory) if file_name != "partial_movie_file_list.txt"] if len(cached_partial_movies) > file_writer_config['max_files_cached']: - oldest_file_path = min(cached_partial_movies, key=os.path.getatime) + number_files_to_delete = len(cached_partial_movies) - file_writer_config['max_files_cached'] + oldest_files_to_delete = sorted([partial_movie_file for partial_movie_file in cached_partial_movies], key=os.path.getatime)[:number_files_to_delete] + # oldest_file_path = min(cached_partial_movies, key=os.path.getatime) + for file_to_delete in oldest_files_to_delete: + os.remove(file_to_delete) logger.info( - f"The partial movie directory is full (> {file_writer_config['max_files_cached']} files). Therefore, manim has removed the file used by it the longest ago ({os.path.basename(oldest_file_path)})." + f"The partial movie directory is full (> {file_writer_config['max_files_cached']} files). Therefore, manim has removed {number_files_to_delete} file(s) used by it the longest ago." + "You can change this behaviour by changing max_files_cached in config.") - os.remove(oldest_file_path) def flush_cache_directory(self): """Delete all the cached partial movie files""" From a03da90b4c0031216ba0b2ec741f67020f1688b1 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Thu, 30 Jul 2020 17:58:45 +0200 Subject: [PATCH 31/47] fixed bug when rendering multiple scenes. skp_animations was not reset to a default value. Appeared in #98 --- manim/scene/scene.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/manim/scene/scene.py b/manim/scene/scene.py index d0a5fd09ad..702aea7042 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -75,6 +75,9 @@ def __init__(self, **kwargs): except EndSceneEarlyException: pass self.tear_down() + # We have to reset these settings in case of multiple renders. + file_writer_config['skip_animations'] = False + self.original_skipping_status = file_writer_config['skip_animations'] self.file_writer.finish() self.print_end_message() From ec0bdafe681a26fc715ef29bf05c4b52d68c6515 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Thu, 30 Jul 2020 22:47:11 +0200 Subject: [PATCH 32/47] fixed bug related to nested dict with wrong keys See previous commit. --- manim/utils/hashing.py | 44 ++++++++++++++++++++++++++++++------------ 1 file changed, 32 insertions(+), 12 deletions(-) diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py index 308e7367e4..b6cbccd981 100644 --- a/manim/utils/hashing.py +++ b/manim/utils/hashing.py @@ -42,14 +42,7 @@ def default(self, obj): return list(obj) elif hasattr(obj, "__dict__"): temp = getattr(obj, '__dict__') - # As dict keys must be of the type (str, int, float, bool), we have to clean them. - # To do that, if one is not of the good type we turn it into its hash using the same - # method as all the objects here. - def key_to_hash(key): - if not isinstance(key, (str, int, float, bool)) and key is not None : - return zlib.crc32(json.dumps(key, cls=self).encode()) - return key - return {key_to_hash(k) : i for k,i in temp.items()} + return self._encode_dict(temp) elif isinstance(obj, np.uint8): return int(obj) try: @@ -59,8 +52,35 @@ def key_to_hash(key): # it into a string "Unsupported type for hashing" so that it won't affect the hash. return "Unsupported type for hashing" + def _encode_dict(self, obj): + """Clean dicts to be serialized : As dict keys must be of the type (str, int, float, bool), we have to change them when they are not of the right type. + To do that, if one is not of the good type we turn it into its hash using the same + method as all the objects here. -def get_json(object): + Parameters + ---------- + obj : :class:`any` + The obj to be cleaned. + + Returns + ------- + `any` + The object cleaned following the processus above. + """ + def key_to_hash(key): + if not isinstance(key, (str, int, float, bool)) and key is not None: + # print('called') + return zlib.crc32(json.dumps(key, cls=CustomEncoder).encode()) + return key + if isinstance(obj, dict): + return {key_to_hash(k): self._encode_dict(v) for k, v in obj.items()} + return obj + + def encode(self, obj): + return super().encode(self._encode_dict(obj)) + + +def get_json(obj): """Recursively serialize `object` to JSON using the :class:`CustomEncoder` class. Paramaters @@ -73,7 +93,7 @@ def get_json(object): :class:`str` The flattened object """ - return json.dumps(object, cls=CustomEncoder) + return json.dumps(obj, cls=CustomEncoder) def get_camera_dict_for_hashing(camera_object): @@ -117,14 +137,14 @@ def get_hash_from_play_call(camera_object, animations_list, current_mobjects_lis :class:`str` A string concatenation of the respective hashes of `camera_object`, `animations_list` and `current_mobjects_list`, separated by `_`. """ - # camera_json = get_json(get_camera_dict_for_hashing(camera_object)) + camera_json = get_json(get_camera_dict_for_hashing(camera_object)) animations_list_json = [get_json(x) for x in sorted( animations_list, key=lambda obj: str(obj))] current_mobjects_list_json = [get_json(x) for x in sorted( current_mobjects_list, key=lambda obj: str(obj))] hash_camera, hash_animations, hash_current_mobjects = [ zlib.crc32(repr(json_val).encode()) - for json_val in [['to_delete'], animations_list_json, current_mobjects_list_json] + for json_val in [camera_json, animations_list_json, current_mobjects_list_json] ] return "{}_{}_{}".format(hash_camera, hash_animations, hash_current_mobjects) From 6fd8c4b5246c1ea33da10c1e7cd03b9cfc5c122a Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Fri, 31 Jul 2020 10:56:22 +0200 Subject: [PATCH 33/47] added infinity support for max_file_cached and changed default --- manim/default.cfg | 3 ++- manim/utils/config_utils.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/manim/default.cfg b/manim/default.cfg index 4f0eecc57f..595f83c598 100644 --- a/manim/default.cfg +++ b/manim/default.cfg @@ -88,7 +88,8 @@ frame_rate = 60 pixel_height = 1440 pixel_width = 2560 -max_files_cached = 7 +# Use -1 to set max_files_cached to infinity. +max_files_cached = 12 flush_cache = False disable_caching = False diff --git a/manim/utils/config_utils.py b/manim/utils/config_utils.py index 483025d970..70c8c70b25 100644 --- a/manim/utils/config_utils.py +++ b/manim/utils/config_utils.py @@ -125,6 +125,8 @@ def _parse_file_writer_config(config_parser, args): [fw_config["save_last_frame"], fw_config["from_animation_number"]] ) fw_config['max_files_cached'] = default.getint('max_files_cached') + if fw_config['max_files_cached'] == -1: + fw_config['max_files_cached'] = float('inf') return fw_config From 31001563afdd8bd0be00e1fbcddcdfcfbe31d9e1 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Sat, 1 Aug 2020 14:38:09 +0200 Subject: [PATCH 34/47] changed default value of max_files_cached to 100 --- manim/default.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manim/default.cfg b/manim/default.cfg index 595f83c598..bf928f134a 100644 --- a/manim/default.cfg +++ b/manim/default.cfg @@ -89,7 +89,7 @@ pixel_height = 1440 pixel_width = 2560 # Use -1 to set max_files_cached to infinity. -max_files_cached = 12 +max_files_cached = 100 flush_cache = False disable_caching = False From 66631e883e93f92805cbd4a1f25a71b14f9a2995 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Sat, 1 Aug 2020 14:56:56 +0200 Subject: [PATCH 35/47] deleted comments --- manim/scene/scene_file_writer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index 636bfa1879..3b9309d589 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -461,7 +461,7 @@ def combine_movie_files(self): partial_movie_files = [os.path.join(self.partial_movie_directory, "{}{}".format( - hash_play, file_writer_config['movie_file_extension'])) for hash_play in self.scene.play_hashes_list] # A OPTIMISER ! Là on recuperer deux fois la list des partial movies files, alors qu'on pourrait utiliser genre get_next_partial_movie_path jsp + hash_play, file_writer_config['movie_file_extension'])) for hash_play in self.scene.play_hashes_list] if len(partial_movie_files) == 0: logger.error("No animations in this scene") return From 18ccc27d514bc9f1e6a144a6085c73c5d4af607f Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Sat, 1 Aug 2020 15:14:28 +0200 Subject: [PATCH 36/47] fixed docstrings --- manim/scene/scene.py | 14 ++++---------- manim/utils/hashing.py | 2 +- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/manim/scene/scene.py b/manim/scene/scene.py index 702aea7042..a49200c2a8 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -850,11 +850,8 @@ def handle_caching_play(func): Parameters ---------- - *args : - Animation or mobject with mobject method and params - - **kwargs : - named parameters affecting what was passed in *args e.g run_time, lag_ratio etc. + func : :class:`Callable[[...], None]` + The play like function that has to be written to the video file stream. Take the same parameters as `scene.play`. """ def wrapper(self, *args, **kwargs): self.revert_to_original_skipping_status() @@ -882,11 +879,8 @@ def handle_caching_wait(func): Parameters ---------- - *args : - Animation or mobject with mobject method and params - - **kwargs : - named parameters affecting what was passed in *args e.g run_time, lag_ratio etc. + func : :class:`Callable[[...], None]` + The wait like function that has to be written to the video file stream. Take the same parameters as `scene.wait`. """ def wrapper(self, duration=DEFAULT_WAIT_TIME, stop_condition=None): self.revert_to_original_skipping_status() diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py index b6cbccd981..de29a62e0d 100644 --- a/manim/utils/hashing.py +++ b/manim/utils/hashing.py @@ -157,7 +157,7 @@ def get_hash_from_wait_call(camera_object, wait_time, stop_condition_function, c wait_time : :class:`float` The time to wait - stop_condition_function : :class:`func` + stop_condition_function : Callable[[...], bool] Boolean function used as a stop_condition in `wait`. Returns From ccad56bb2439609e999838261395e0954fc12851 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Sun, 2 Aug 2020 12:47:56 +0200 Subject: [PATCH 37/47] fixed format (maybe) --- manim/scene/scene.py | 65 +++++++++++++++------------ manim/scene/scene_file_writer.py | 77 ++++++++++++++++++++------------ manim/utils/config_utils.py | 19 ++++---- manim/utils/file_ops.py | 5 ++- manim/utils/hashing.py | 42 ++++++++++------- tests/testing_utils.py | 20 +++++---- 6 files changed, 136 insertions(+), 92 deletions(-) diff --git a/manim/scene/scene.py b/manim/scene/scene.py index 67c0a227f8..589a0156d2 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -2,7 +2,7 @@ import random import warnings import platform -import copy +import copy from tqdm import tqdm as ProgressDisplay import numpy as np @@ -56,9 +56,7 @@ def construct(self): def __init__(self, **kwargs): Container.__init__(self, **kwargs) self.camera = self.camera_class(**camera_config) - self.file_writer = SceneFileWriter( - self, **file_writer_config, - ) + self.file_writer = SceneFileWriter(self, **file_writer_config,) self.play_hashes_list = [] self.mobjects = [] # TODO, remove need for foreground mobjects @@ -77,8 +75,8 @@ def __init__(self, **kwargs): pass self.tear_down() # We have to reset these settings in case of multiple renders. - file_writer_config['skip_animations'] = False - self.original_skipping_status = file_writer_config['skip_animations'] + file_writer_config["skip_animations"] = False + self.original_skipping_status = file_writer_config["skip_animations"] self.file_writer.finish() self.print_end_message() @@ -378,7 +376,7 @@ def add_mobjects_among(self, values): self.add(*filter(lambda m: isinstance(m, Mobject), values)) return self - def add_mobjects_from_animations(self, animations): + def add_mobjects_from_animations(self, animations): curr_mobjects = self.get_mobject_family_members() for animation in animations: @@ -388,7 +386,7 @@ def add_mobjects_from_animations(self, animations): if mob not in curr_mobjects: self.add(mob) curr_mobjects += mob.get_family() - + def remove(self, *mobjects): """ Removes mobjects in the passed list of mobjects @@ -802,6 +800,7 @@ def compile_method(state): state["last_method"] = state["curr_method"] state["curr_method"] = None state["method_args"] = [] + for arg in args: if isinstance(arg, Animation): compile_method(state) @@ -846,7 +845,7 @@ def update_skipping_status(self): file_writer_config["skip_animations"] = True raise EndSceneEarlyException() - def handle_caching_play(func): + def handle_caching_play(func): """ This method is used internally to wrap the passed function into a function that will compute the hash of the play invocation, and will act accordingly : either skip the animation because already cached, either nothing and let the play invocation be processed normally. @@ -856,26 +855,30 @@ def handle_caching_play(func): func : :class:`Callable[[...], None]` The play like function that has to be written to the video file stream. Take the same parameters as `scene.play`. """ + def wrapper(self, *args, **kwargs): self.revert_to_original_skipping_status() - animations = self.compile_play_args_to_animation_list( - *args, **kwargs - ) + animations = self.compile_play_args_to_animation_list(*args, **kwargs) self.add_mobjects_from_animations(animations) - if not file_writer_config['disable_caching']: + if not file_writer_config["disable_caching"]: mobjects_on_scene = self.get_mobjects() - hash_play = get_hash_from_play_call(self.camera, animations, mobjects_on_scene) + hash_play = get_hash_from_play_call( + self.camera, animations, mobjects_on_scene + ) self.play_hashes_list.append(hash_play) if self.file_writer.is_already_cached(hash_play): - logger.info(f'Animation {self.num_plays} : Using cached data (hash : {hash_play})') - file_writer_config['skip_animations'] = True - else: + logger.info( + f"Animation {self.num_plays} : Using cached data (hash : {hash_play})" + ) + file_writer_config["skip_animations"] = True + else: hash_play = "uncached_{:05}".format(self.num_plays) self.play_hashes_list.append(hash_play) func(self, *args, **kwargs) + return wrapper - def handle_caching_wait(func): + def handle_caching_wait(func): """ This method is used internally to wrap the passed function into a function that will compute the hash of the wait invocation, and will act accordingly : either skip the animation because already cached or nothing and let the play invocation be processed normally @@ -885,18 +888,24 @@ def handle_caching_wait(func): func : :class:`Callable[[...], None]` The wait like function that has to be written to the video file stream. Take the same parameters as `scene.wait`. """ + def wrapper(self, duration=DEFAULT_WAIT_TIME, stop_condition=None): self.revert_to_original_skipping_status() - if not file_writer_config['disable_caching']: - hash_wait = get_hash_from_wait_call(self.camera, duration, stop_condition, self.get_mobjects()) + if not file_writer_config["disable_caching"]: + hash_wait = get_hash_from_wait_call( + self.camera, duration, stop_condition, self.get_mobjects() + ) self.play_hashes_list.append(hash_wait) if self.file_writer.is_already_cached(hash_wait): - logger.info(f'Wait {self.num_plays} : Using cached data (hash : {hash_wait})') - file_writer_config['skip_animations'] = True - else : + logger.info( + f"Wait {self.num_plays} : Using cached data (hash : {hash_wait})" + ) + file_writer_config["skip_animations"] = True + else: hash_wait = "uncached_{:05}".format(self.num_plays) self.play_hashes_list.append(hash_wait) func(self, duration, stop_condition) + return wrapper def handle_play_like_call(func): @@ -1092,7 +1101,7 @@ def get_wait_time_progression(self, duration, stop_condition): time_progression = self.get_time_progression(duration) time_progression.set_description("Waiting {}".format(self.num_plays)) return time_progression - + @handle_caching_wait @handle_play_like_call def wait(self, duration=DEFAULT_WAIT_TIME, stop_condition=None): @@ -1167,8 +1176,8 @@ def force_skipping(self): Scene The Scene, with skipping turned on. """ - self.original_skipping_status = file_writer_config['skip_animations'] - file_writer_config['skip_animations'] = True + self.original_skipping_status = file_writer_config["skip_animations"] + file_writer_config["skip_animations"] = True return self def revert_to_original_skipping_status(self): @@ -1183,7 +1192,7 @@ def revert_to_original_skipping_status(self): The Scene, with the original skipping status. """ if hasattr(self, "original_skipping_status"): - file_writer_config['skip_animations'] = self.original_skipping_status + file_writer_config["skip_animations"] = self.original_skipping_status return self def add_frames(self, *frames): @@ -1218,7 +1227,7 @@ def add_sound(self, sound_file, time_offset=0, gain=None, **kwargs): gain : """ - if file_writer_config['skip_animations']: + if file_writer_config["skip_animations"]: return time = self.get_time() + time_offset self.file_writer.add_sound(sound_file, time, gain, **kwargs) diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index 55752d68ff..8b3b9b35e4 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -172,8 +172,8 @@ def get_next_partial_movie_path(self): self.partial_movie_directory, "{}{}".format( self.scene.play_hashes_list[self.scene.num_plays], - file_writer_config['movie_file_extension'], - ) + file_writer_config["movie_file_extension"], + ), ) return result @@ -352,11 +352,11 @@ def finish(self): if hasattr(self, "writing_process"): self.writing_process.terminate() self.combine_movie_files() - if file_writer_config['flush_cache']: + if file_writer_config["flush_cache"]: self.flush_cache_directory() else: self.clean_cache() - if file_writer_config['save_last_frame']: + if file_writer_config["save_last_frame"]: self.scene.update_frame(ignore_skipping=True) self.save_final_image(self.scene.get_image()) @@ -427,7 +427,8 @@ def close_movie_pipe(self): self.temp_partial_movie_file_path, self.partial_movie_file_path, ) logger.debug( - f"Animation {self.scene.num_plays} : Partial movie file written in {self.partial_movie_file_path}") + f"Animation {self.scene.num_plays} : Partial movie file written in {self.partial_movie_file_path}" + ) def is_already_cached(self, hash_invokation): """Will check if a file named with `hash_invokation` exists. @@ -442,8 +443,10 @@ def is_already_cached(self, hash_invokation): :class:`bool` Whether the file exists. """ - path = os.path.join(self.partial_movie_directory, "{}{}".format( - hash_invokation, self.movie_file_extension)) + path = os.path.join( + self.partial_movie_directory, + "{}{}".format(hash_invokation, self.movie_file_extension), + ) return os.path.exists(path) def combine_movie_files(self): @@ -459,8 +462,13 @@ def combine_movie_files(self): # cuts at all the places you might want. But for viewing # the scene as a whole, one of course wants to see it as a # single piece. - partial_movie_files = [os.path.join(self.partial_movie_directory, "{}{}".format( - hash_play, file_writer_config['movie_file_extension'])) for hash_play in self.scene.play_hashes_list] + partial_movie_files = [ + os.path.join( + self.partial_movie_directory, + "{}{}".format(hash_play, file_writer_config["movie_file_extension"]), + ) + for hash_play in self.scene.play_hashes_list + ] if len(partial_movie_files) == 0: logger.error("No animations in this scene") return @@ -469,12 +477,12 @@ def combine_movie_files(self): file_list = os.path.join( self.partial_movie_directory, "partial_movie_file_list.txt" ) - with open(file_list, 'w') as fp: + with open(file_list, "w") as fp: fp.write("# This file is used internally by FFMPEG.\n") for pf_path in partial_movie_files: - if os.name == 'nt': - pf_path = pf_path.replace('\\', '/') - fp.write("file \'file:{}'\n".format(pf_path)) + if os.name == "nt": + pf_path = pf_path.replace("\\", "/") + fp.write("file 'file:{}'\n".format(pf_path)) movie_file_path = self.get_movie_file_path() commands = [ FFMPEG_BIN, @@ -490,10 +498,7 @@ def combine_movie_files(self): ] if self.write_to_movie: - commands += [ - '-c', 'copy', - movie_file_path - ] + commands += ["-c", "copy", movie_file_path] if self.save_as_gif: commands += [self.gif_file_path] @@ -542,32 +547,46 @@ def combine_movie_files(self): os.remove(sound_file_path) self.print_file_ready_message(movie_file_path) - if file_writer_config["write_to_movie"] : + if file_writer_config["write_to_movie"]: for file_path in partial_movie_files: # We have to modify the accessed time so if we have to clean the cache we remove the one used the longest. modify_atime(file_path) def clean_cache(self): """Will clean the cache by removing the partial_movie_files used by manim the longest ago.""" - cached_partial_movies = [os.path.join(self.partial_movie_directory, file_name) for file_name in os.listdir( - self.partial_movie_directory) if file_name != "partial_movie_file_list.txt"] - if len(cached_partial_movies) > file_writer_config['max_files_cached']: - number_files_to_delete = len(cached_partial_movies) - file_writer_config['max_files_cached'] - oldest_files_to_delete = sorted([partial_movie_file for partial_movie_file in cached_partial_movies], key=os.path.getatime)[:number_files_to_delete] + cached_partial_movies = [ + os.path.join(self.partial_movie_directory, file_name) + for file_name in os.listdir(self.partial_movie_directory) + if file_name != "partial_movie_file_list.txt" + ] + if len(cached_partial_movies) > file_writer_config["max_files_cached"]: + number_files_to_delete = ( + len(cached_partial_movies) - file_writer_config["max_files_cached"] + ) + oldest_files_to_delete = sorted( + [partial_movie_file for partial_movie_file in cached_partial_movies], + key=os.path.getatime, + )[:number_files_to_delete] # oldest_file_path = min(cached_partial_movies, key=os.path.getatime) for file_to_delete in oldest_files_to_delete: os.remove(file_to_delete) logger.info( f"The partial movie directory is full (> {file_writer_config['max_files_cached']} files). Therefore, manim has removed {number_files_to_delete} file(s) used by it the longest ago." - + "You can change this behaviour by changing max_files_cached in config.") + + "You can change this behaviour by changing max_files_cached in config." + ) - def flush_cache_directory(self): + def flush_cache_directory(self): """Delete all the cached partial movie files""" - cached_partial_movies = [os.path.join(self.partial_movie_directory, file_name) for file_name in os.listdir( - self.partial_movie_directory) if file_name != "partial_movie_file_list.txt"] - for f in cached_partial_movies: + cached_partial_movies = [ + os.path.join(self.partial_movie_directory, file_name) + for file_name in os.listdir(self.partial_movie_directory) + if file_name != "partial_movie_file_list.txt" + ] + for f in cached_partial_movies: os.remove(f) - logger.info(f"Cache flushed. {len(cached_partial_movies)} file(s) deleted in {self.partial_movie_directory}.") + logger.info( + f"Cache flushed. {len(cached_partial_movies)} file(s) deleted in {self.partial_movie_directory}." + ) def print_file_ready_message(self, file_path): """ diff --git a/manim/utils/config_utils.py b/manim/utils/config_utils.py index 70c8c70b25..731af25cb0 100644 --- a/manim/utils/config_utils.py +++ b/manim/utils/config_utils.py @@ -48,8 +48,9 @@ def _parse_file_writer_config(config_parser, args): "save_pngs", "save_as_gif", "write_all", - "disable_caching", - "flush_cache"]: + "disable_caching", + "flush_cache", + ]: attr = getattr(args, boolean_opt) fw_config[boolean_opt] = ( @@ -102,8 +103,8 @@ def _parse_file_writer_config(config_parser, args): "write_all", ]: fw_config[opt] = config_parser["dry_run"].getboolean(opt) - if not fw_config['write_to_movie']: - fw_config['disable_caching'] = True + if not fw_config["write_to_movie"]: + fw_config["disable_caching"] = True # Read in the streaming section -- all values are strings fw_config["streaming"] = { opt: config_parser["streaming"][opt] @@ -124,9 +125,9 @@ def _parse_file_writer_config(config_parser, args): fw_config["skip_animations"] = any( [fw_config["save_last_frame"], fw_config["from_animation_number"]] ) - fw_config['max_files_cached'] = default.getint('max_files_cached') - if fw_config['max_files_cached'] == -1: - fw_config['max_files_cached'] = float('inf') + fw_config["max_files_cached"] = default.getint("max_files_cached") + if fw_config["max_files_cached"] == -1: + fw_config["max_files_cached"] = float("inf") return fw_config @@ -226,13 +227,13 @@ def _parse_cli(arg_list, input=True): "--disable_caching", action="store_const", const=True, - help="Disable caching (will generate partial-movie-files anyway).", + help="Disable caching (will generate partial-movie-files anyway).", ) parser.add_argument( "--flush_cache", action="store_const", const=True, - help="Remove all cached partial-movie-files.", + help="Remove all cached partial-movie-files.", ) # The default value of the following is set in manim.cfg parser.add_argument( diff --git a/manim/utils/file_ops.py b/manim/utils/file_ops.py index 21fe390494..7227c6dd6a 100644 --- a/manim/utils/file_ops.py +++ b/manim/utils/file_ops.py @@ -2,6 +2,7 @@ import numpy as np import time + def add_extension_if_not_present(file_name, extension): # This could conceivably be smarter about handling existing differing extensions if file_name[-len(extension) :] != extension: @@ -28,7 +29,7 @@ def seek_full_path_from_defaults(file_name, default_dir, extensions): raise IOError("File {} not Found".format(file_name)) -def modify_atime(file_path): +def modify_atime(file_path): """Will manually change the accessed time (called `atime`) of the file, as on a lot of OS the accessed time refresh is disabled by default. Paramaters @@ -36,4 +37,4 @@ def modify_atime(file_path): file_path : :class:`str` The path of the file. """ - os.utime(file_path, times=(time.time(),os.path.getmtime(file_path))) + os.utime(file_path, times=(time.time(), os.path.getmtime(file_path))) diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py index de29a62e0d..436c82e122 100644 --- a/manim/utils/hashing.py +++ b/manim/utils/hashing.py @@ -36,12 +36,11 @@ def default(self, obj): # NOTE : All module types objects are removed, because otherwise it throws ValueError: Circular reference detected if not. TODO if isinstance(x[i], ModuleType): del x[i] - return {'code': inspect.getsource(obj), - 'nonlocals': x} + return {"code": inspect.getsource(obj), "nonlocals": x} elif isinstance(obj, np.ndarray): return list(obj) elif hasattr(obj, "__dict__"): - temp = getattr(obj, '__dict__') + temp = getattr(obj, "__dict__") return self._encode_dict(temp) elif isinstance(obj, np.uint8): return int(obj) @@ -67,11 +66,13 @@ def _encode_dict(self, obj): `any` The object cleaned following the processus above. """ + def key_to_hash(key): if not isinstance(key, (str, int, float, bool)) and key is not None: # print('called') return zlib.crc32(json.dumps(key, cls=CustomEncoder).encode()) return key + if isinstance(obj, dict): return {key_to_hash(k): self._encode_dict(v) for k, v in obj.items()} return obj @@ -113,7 +114,7 @@ def get_camera_dict_for_hashing(camera_object): # We have to clean a little bit of camera_dict, as pixel_array and background are two very big numpy arrays. # They are not essential to caching process. # We also have to remove pixel_array_to_cairo_context as it contains used memory adress (set randomly). See l.516 get_cached_cairo_context in camera.py - for to_clean in ['background', 'pixel_array', 'pixel_array_to_cairo_context']: + for to_clean in ["background", "pixel_array", "pixel_array_to_cairo_context"]: camera_object_dict.pop(to_clean, None) return camera_object_dict @@ -138,10 +139,12 @@ def get_hash_from_play_call(camera_object, animations_list, current_mobjects_lis A string concatenation of the respective hashes of `camera_object`, `animations_list` and `current_mobjects_list`, separated by `_`. """ camera_json = get_json(get_camera_dict_for_hashing(camera_object)) - animations_list_json = [get_json(x) for x in sorted( - animations_list, key=lambda obj: str(obj))] - current_mobjects_list_json = [get_json(x) for x in sorted( - current_mobjects_list, key=lambda obj: str(obj))] + animations_list_json = [ + get_json(x) for x in sorted(animations_list, key=lambda obj: str(obj)) + ] + current_mobjects_list_json = [ + get_json(x) for x in sorted(current_mobjects_list, key=lambda obj: str(obj)) + ] hash_camera, hash_animations, hash_current_mobjects = [ zlib.crc32(repr(json_val).encode()) for json_val in [camera_json, animations_list_json, current_mobjects_list_json] @@ -149,7 +152,9 @@ def get_hash_from_play_call(camera_object, animations_list, current_mobjects_lis return "{}_{}_{}".format(hash_camera, hash_animations, hash_current_mobjects) -def get_hash_from_wait_call(camera_object, wait_time, stop_condition_function, current_mobjects_list): +def get_hash_from_wait_call( + camera_object, wait_time, stop_condition_function, current_mobjects_list +): """Take a wait time, a boolean function as a stop condition and a list of mobjects, and then output their individual hashes. This is meant to be used for `scene.wait` function. Parameters @@ -166,13 +171,20 @@ def get_hash_from_wait_call(camera_object, wait_time, stop_condition_function, c A concatenation of the respective hashes of `animations_list and `current_mobjects_list`, separated by `_`. """ camera_json = get_json(get_camera_dict_for_hashing(camera_object)) - current_mobjects_list_json = [get_json(x) for x in sorted( - current_mobjects_list, key=lambda obj: str(obj))] - hash_current_mobjects = zlib.crc32( - repr(current_mobjects_list_json).encode()) + current_mobjects_list_json = [ + get_json(x) for x in sorted(current_mobjects_list, key=lambda obj: str(obj)) + ] + hash_current_mobjects = zlib.crc32(repr(current_mobjects_list_json).encode()) hash_camera = zlib.crc32(repr(camera_json).encode()) if stop_condition_function != None: hash_function = zlib.crc32(get_json(stop_condition_function).encode()) - return "{}_{}{}_{}".format(hash_camera, str(wait_time).replace('.', '-'), hash_function, hash_current_mobjects) + return "{}_{}{}_{}".format( + hash_camera, + str(wait_time).replace(".", "-"), + hash_function, + hash_current_mobjects, + ) else: - return "{}_{}_{}".format(hash_camera, str(wait_time).replace('.', '-'), hash_current_mobjects) + return "{}_{}_{}".format( + hash_camera, str(wait_time).replace(".", "-"), hash_current_mobjects + ) diff --git a/tests/testing_utils.py b/tests/testing_utils.py index 6110b06053..92cfef9dbc 100644 --- a/tests/testing_utils.py +++ b/tests/testing_utils.py @@ -40,17 +40,19 @@ def __init__(self, scene_object, module_tested, caching_needed=False): self.path_tests_data = os.path.join("tests", "tests_data", module_tested) if caching_needed: - file_writer_config['text_dir'] = os.path.join( - self.path_tests_medias_cache, scene_object.__name__, 'Text') - file_writer_config['tex_dir'] = os.path.join( - self.path_tests_medias_cache, scene_object.__name__, 'Tex') + file_writer_config["text_dir"] = os.path.join( + self.path_tests_medias_cache, scene_object.__name__, "Text" + ) + file_writer_config["tex_dir"] = os.path.join( + self.path_tests_medias_cache, scene_object.__name__, "Tex" + ) - file_writer_config['skip_animations'] = True - file_writer_config['disable_caching'] = True + file_writer_config["skip_animations"] = True + file_writer_config["disable_caching"] = True file_writer_config["write_to_movie"] = False - config['pixel_height'] = 480 - config['pixel_width'] = 854 - config['frame_rate'] = 15 + config["pixel_height"] = 480 + config["pixel_width"] = 854 + config["frame_rate"] = 15 # By invoking this, the scene is rendered. self.scene = scene_object() From 64403c045fc3f601c6cf7adac9cf31a94b594b00 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Sun, 9 Aug 2020 19:29:49 +0200 Subject: [PATCH 38/47] fixed format --- manim/utils/file_ops.py | 1 + 1 file changed, 1 insertion(+) diff --git a/manim/utils/file_ops.py b/manim/utils/file_ops.py index 5eb547433b..b159e932d3 100644 --- a/manim/utils/file_ops.py +++ b/manim/utils/file_ops.py @@ -41,6 +41,7 @@ def modify_atime(file_path): """ os.utime(file_path, times=(time.time(), os.path.getmtime(file_path))) + def open_file(file_path): current_os = platform.system() if current_os == "Windows": From 3d0cd016c8b197ccc5e92f264c665ed4a192eacb Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Mon, 10 Aug 2020 16:13:35 +0200 Subject: [PATCH 39/47] fixed tests --- tests/conftest.py | 8 ++++++++ tests/test_cli/test_cfg_subcmd.py | 3 +-- tests/test_cli/write_cfg_sbcmd_input.txt | 2 ++ tests/test_logging/expected.txt | 12 +++++++++--- tests/test_logging/test_logging.py | 8 ++++---- 5 files changed, 24 insertions(+), 9 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 95f158842c..3236483e55 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -4,6 +4,7 @@ import os import sys import logging +from shutil import rmtree def pytest_addoption(parser): @@ -44,3 +45,10 @@ def reset_cfg_file(): yield with open(cfgfilepath, "w") as cfgfile: cfgfile.write(original) + + +@pytest.fixture +def clean_tests_cache(): + yield + path_output = os.path.join("tests", "tests_cache", "media_temp") + rmtree(path_output) diff --git a/tests/test_cli/test_cfg_subcmd.py b/tests/test_cli/test_cfg_subcmd.py index 2643ca2532..e994febc07 100644 --- a/tests/test_cli/test_cfg_subcmd.py +++ b/tests/test_cli/test_cfg_subcmd.py @@ -52,7 +52,6 @@ def test_cfg_write(python_version): ) assert ( exitcode == 0 - ), f"The cfg subcommand write is not working as intended.\nError : {err}" - + ), f"The cfg subcommand write is not working as intended.\nError : {err.decode()}" with open(cfgfilepath, "r") as cfgfile: assert "sound = False" in cfgfile.read() diff --git a/tests/test_cli/write_cfg_sbcmd_input.txt b/tests/test_cli/write_cfg_sbcmd_input.txt index a9fd7d0d55..ee5bbcfcae 100644 --- a/tests/test_cli/write_cfg_sbcmd_input.txt +++ b/tests/test_cli/write_cfg_sbcmd_input.txt @@ -25,5 +25,7 @@ False + + diff --git a/tests/test_logging/expected.txt b/tests/test_logging/expected.txt index ac9ee92030..317aad3a5e 100644 --- a/tests/test_logging/expected.txt +++ b/tests/test_logging/expected.txt @@ -1,4 +1,10 @@ - INFO Read configuration files: config.py: - INFO scene_file_writer.py: - File ready at + INFO Read configuration files: ['C:\\Users\\User\\OneDrive\\Bureau\\Programmation\\PYTHON\\MANIM-DEV\\manim\\manim\\default.cfg', 'C:\\Users\\User\\AppData\\Roaming\\Manim\\manim.cfg', config.py: + 'C:\\Users\\User\\OneDrive\\Bureau\\Programmation\\PYTHON\\MANIM-DEV\\manim\\tests\\tests_data\\manim.cfg'] + DEBUG Animation : Partial movie file written in scene_file_writer.py: + + DEBUG Animation : Partial movie file written in scene_file_writer.py: + DEBUG Animation : Partial movie file written in scene_file_writer.py: + + INFO scene_file_writer.py: + File ready at diff --git a/tests/test_logging/test_logging.py b/tests/test_logging/test_logging.py index 892889a6ef..05b9dbbefa 100644 --- a/tests/test_logging/test_logging.py +++ b/tests/test_logging/test_logging.py @@ -1,7 +1,6 @@ import subprocess import os import sys -from shutil import rmtree import pytest import re @@ -14,6 +13,7 @@ def capture(command, instream=None): return out, err, proc.returncode +@pytest.mark.usefixtures("clean_tests_cache") def test_logging_to_file(python_version): """Test logging Terminal output to a log file. As some data will differ with each log (the timestamps, file paths, line nums etc) @@ -21,7 +21,7 @@ def test_logging_to_file(python_version): whitespace. """ path_basic_scene = os.path.join("tests", "tests_data", "basic_scenes.py") - path_output = os.path.join("tests_cache", "media_temp") + path_output = os.path.join("tests", "tests_cache", "media_temp") command = [ python_version, "-m", @@ -37,8 +37,8 @@ def test_logging_to_file(python_version): ] out, err, exitcode = capture(command) log_file_path = os.path.join(path_output, "logs", "SquareToCircle.log") - assert exitcode == 0, err - assert os.path.exists(log_file_path), err + assert exitcode == 0, err.decode() + assert os.path.exists(log_file_path), err.decode() if sys.platform.startswith("win32") or sys.platform.startswith("cygwin"): enc = "Windows-1252" else: From 8bcc05a18c9a3b0e19f868d78e159ac8150e3ce8 Mon Sep 17 00:00:00 2001 From: Aathish Date: Tue, 11 Aug 2020 10:37:30 +0530 Subject: [PATCH 40/47] Update expected.txt Fixes logging tests. --- tests/test_logging/expected.txt | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/test_logging/expected.txt b/tests/test_logging/expected.txt index 317aad3a5e..ac9ee92030 100644 --- a/tests/test_logging/expected.txt +++ b/tests/test_logging/expected.txt @@ -1,10 +1,4 @@ - INFO Read configuration files: ['C:\\Users\\User\\OneDrive\\Bureau\\Programmation\\PYTHON\\MANIM-DEV\\manim\\manim\\default.cfg', 'C:\\Users\\User\\AppData\\Roaming\\Manim\\manim.cfg', config.py: - 'C:\\Users\\User\\OneDrive\\Bureau\\Programmation\\PYTHON\\MANIM-DEV\\manim\\tests\\tests_data\\manim.cfg'] - DEBUG Animation : Partial movie file written in scene_file_writer.py: - - DEBUG Animation : Partial movie file written in scene_file_writer.py: - - DEBUG Animation : Partial movie file written in scene_file_writer.py: - - INFO scene_file_writer.py: + INFO Read configuration files: config.py: + INFO scene_file_writer.py: File ready at + From e0fc6e5ff2573c68ecc5d17deac27785c4ace18f Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Tue, 11 Aug 2020 15:34:50 +0200 Subject: [PATCH 41/47] fixed merge conflict --- manim/utils/config_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/manim/utils/config_utils.py b/manim/utils/config_utils.py index cb96c67645..7180e02901 100644 --- a/manim/utils/config_utils.py +++ b/manim/utils/config_utils.py @@ -137,6 +137,9 @@ def _parse_file_writer_config(config_parser, args): fw_config["skip_animations"] = any( [fw_config["save_last_frame"], fw_config["from_animation_number"]] ) + fw_config["max_files_cached"] = default.getint("max_files_cached") + if fw_config["max_files_cached"] == -1: + fw_config["max_files_cached"] = float("inf") # Parse the verbose flag to read in the log level verbose = getattr(args, "verbose") verbose = default["verbose"] if verbose is None else verbose From 2d1d58df1d266fafb594d992dd1e1175f6dbcd23 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Tue, 11 Aug 2020 18:15:56 +0200 Subject: [PATCH 42/47] fixed logging test --- tests/test_logging/expected.txt | 8 ++++---- tests/tests_data/manim.cfg | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/test_logging/expected.txt b/tests/test_logging/expected.txt index ac9ee92030..9708fae5ed 100644 --- a/tests/test_logging/expected.txt +++ b/tests/test_logging/expected.txt @@ -1,4 +1,4 @@ - INFO Read configuration files: config.py: - INFO scene_file_writer.py: - File ready at - + INFO Read configuration files: config.py: + INFO scene_file_writer.py: + File ready at + diff --git a/tests/tests_data/manim.cfg b/tests/tests_data/manim.cfg index 6a3178abfb..9986617c4c 100644 --- a/tests/tests_data/manim.cfg +++ b/tests/tests_data/manim.cfg @@ -6,4 +6,4 @@ save_last_frame = False # save_pngs = False [logger] -log_width = 256 \ No newline at end of file +log_width = 512 \ No newline at end of file From e5e15976f4993129411175d7e42bd855a2832cd5 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Tue, 11 Aug 2020 18:35:14 +0200 Subject: [PATCH 43/47] fixed docstrings --- manim/scene/scene.py | 10 ++++++---- manim/scene/scene_file_writer.py | 8 ++++---- manim/utils/hashing.py | 1 + 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/manim/scene/scene.py b/manim/scene/scene.py index e0f4417ced..efc0ff479c 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -850,9 +850,10 @@ def update_skipping_status(self): def handle_caching_play(func): """ - This method is used internally to wrap the passed function into a function that will compute the hash of the play invocation, - and will act accordingly : either skip the animation because already cached, either nothing and let the play invocation be processed normally. + Decorator that returns a wrapped version of func that will compute the hash of the play invocation. + The returned function will act according to the computed hash: either skip the animation because it's already cached, or let the invoked function play normally. + Parameters ---------- func : :class:`Callable[[...], None]` @@ -883,8 +884,9 @@ def wrapper(self, *args, **kwargs): def handle_caching_wait(func): """ - This method is used internally to wrap the passed function into a function that will compute the hash of the wait invocation, - and will act accordingly : either skip the animation because already cached or nothing and let the play invocation be processed normally + Decorator that returns a wrapped version of func that will compute the hash of the wait invocation. + + The returned function will act according to the computed hash: either skip the animation because it's already cached, or let the invoked function play normally. Parameters ---------- diff --git a/manim/scene/scene_file_writer.py b/manim/scene/scene_file_writer.py index 302f9a33b4..4edbb1b674 100644 --- a/manim/scene/scene_file_writer.py +++ b/manim/scene/scene_file_writer.py @@ -430,12 +430,12 @@ def close_movie_pipe(self): f"Animation {self.scene.num_plays} : Partial movie file written in {self.partial_movie_file_path}" ) - def is_already_cached(self, hash_invokation): - """Will check if a file named with `hash_invokation` exists. + def is_already_cached(self, hash_invocation): + """Will check if a file named with `hash_invocation` exists. Parameters ---------- - hash_invokation : :class:`str` + hash_invocation : :class:`str` The hash corresponding to an invocation to either `scene.play` or `scene.wait`. Returns @@ -445,7 +445,7 @@ def is_already_cached(self, hash_invokation): """ path = os.path.join( self.partial_movie_directory, - "{}{}".format(hash_invokation, self.movie_file_extension), + "{}{}".format(hash_invocation, self.movie_file_extension), ) return os.path.exists(path) diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py index 436c82e122..46478a7faa 100644 --- a/manim/utils/hashing.py +++ b/manim/utils/hashing.py @@ -13,6 +13,7 @@ class CustomEncoder(json.JSONEncoder): def default(self, obj): """ This method is used to serialize objects to JSON format. + If obj is a function, then it will return a dict with two keys : 'code', for the code source, and 'nonlocals' for all nonlocalsvalues. (including nonlocals functions, that will be serialized as this is recursive.) if obj is a np.darray, it converts it into a list. if obj is an object with __dict__ attribute, it returns its __dict__. From f9cc9ede8604f7cccbf0e2aefd2b3cebfe487c94 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Tue, 11 Aug 2020 18:40:13 +0200 Subject: [PATCH 44/47] minor doc improvement --- manim/default.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/manim/default.cfg b/manim/default.cfg index fed3a1d82a..a5006bd2f7 100644 --- a/manim/default.cfg +++ b/manim/default.cfg @@ -99,6 +99,7 @@ pixel_width = 2560 # Use -1 to set max_files_cached to infinity. max_files_cached = 100 +#Flush cache will delete all the cached partial-movie-files. flush_cache = False disable_caching = False From e3ab30a6a49663846343c049c91ac0b049313dca Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Tue, 11 Aug 2020 20:25:03 +0200 Subject: [PATCH 45/47] Apply suggestions from code review Thank you Captain Docs ! Co-authored-by: Pg Biel <9021226+PgBiel@users.noreply.github.com> --- manim/scene/scene.py | 4 ++-- manim/utils/file_ops.py | 2 +- manim/utils/hashing.py | 12 ++++++------ 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/manim/scene/scene.py b/manim/scene/scene.py index efc0ff479c..dd19d1ac71 100644 --- a/manim/scene/scene.py +++ b/manim/scene/scene.py @@ -856,7 +856,7 @@ def handle_caching_play(func): Parameters ---------- - func : :class:`Callable[[...], None]` + func : Callable[[...], None] The play like function that has to be written to the video file stream. Take the same parameters as `scene.play`. """ @@ -890,7 +890,7 @@ def handle_caching_wait(func): Parameters ---------- - func : :class:`Callable[[...], None]` + func : Callable[[...], None] The wait like function that has to be written to the video file stream. Take the same parameters as `scene.wait`. """ diff --git a/manim/utils/file_ops.py b/manim/utils/file_ops.py index b159e932d3..2f975704b6 100644 --- a/manim/utils/file_ops.py +++ b/manim/utils/file_ops.py @@ -34,7 +34,7 @@ def seek_full_path_from_defaults(file_name, default_dir, extensions): def modify_atime(file_path): """Will manually change the accessed time (called `atime`) of the file, as on a lot of OS the accessed time refresh is disabled by default. - Paramaters + Parameters ---------- file_path : :class:`str` The path of the file. diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py index 46478a7faa..69be6878e4 100644 --- a/manim/utils/hashing.py +++ b/manim/utils/hashing.py @@ -21,12 +21,12 @@ def default(self, obj): Parameters ---------- - obj : `any` + obj : Any Arbitrary object to convert Returns ------- - `any` + Any Python object that JSON encoder will recognize """ @@ -59,12 +59,12 @@ def _encode_dict(self, obj): Parameters ---------- - obj : :class:`any` + obj : Any The obj to be cleaned. Returns ------- - `any` + Any The object cleaned following the processus above. """ @@ -128,10 +128,10 @@ def get_hash_from_play_call(camera_object, animations_list, current_mobjects_lis camera_object : :class:`~.Camera` The camera object used in the scene. - animations_list : :class:`list` + animations_list : Iterable[:class:`~.Animation`] The list of animations. - current_mobjects_list : :class:`list` + current_mobjects_list : Iterable[:class:`~.Mobject`] The list of mobjects. Returns From 947e14647446e54d3b3d35449b5e0ee406111db3 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Wed, 12 Aug 2020 14:48:27 +0200 Subject: [PATCH 46/47] Apply suggestions from code review Co-authored-by: Pg Biel <9021226+PgBiel@users.noreply.github.com> --- manim/utils/hashing.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py index 69be6878e4..5d9b1bcd91 100644 --- a/manim/utils/hashing.py +++ b/manim/utils/hashing.py @@ -31,12 +31,12 @@ def default(self, obj): """ if inspect.isfunction(obj) and not isinstance(obj, ModuleType): - r = inspect.getclosurevars(obj) - x = {**copy.copy(r.globals), **copy.copy(r.nonlocals)} - for i in list(x): + cvars = inspect.getclosurevars(obj) + cvardict = {**copy.copy(cvars.globals), **copy.copy(cvars.nonlocals)} + for i in list(cvardict): # NOTE : All module types objects are removed, because otherwise it throws ValueError: Circular reference detected if not. TODO - if isinstance(x[i], ModuleType): - del x[i] + if isinstance(cvardict[i], ModuleType): + del cvardict[i] return {"code": inspect.getsource(obj), "nonlocals": x} elif isinstance(obj, np.ndarray): return list(obj) @@ -177,7 +177,7 @@ def get_hash_from_wait_call( ] hash_current_mobjects = zlib.crc32(repr(current_mobjects_list_json).encode()) hash_camera = zlib.crc32(repr(camera_json).encode()) - if stop_condition_function != None: + if stop_condition_function is not None: hash_function = zlib.crc32(get_json(stop_condition_function).encode()) return "{}_{}{}_{}".format( hash_camera, From 7a3e1dc5d2ef47a9b20b65923f3a4fbebe42d0e5 Mon Sep 17 00:00:00 2001 From: Hugues Devimeux <36239975+huguesdevimeux@users.noreply.github.com> Date: Wed, 12 Aug 2020 14:58:28 +0200 Subject: [PATCH 47/47] fixed typo of the last commit -_- --- manim/utils/hashing.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manim/utils/hashing.py b/manim/utils/hashing.py index 5d9b1bcd91..563dd330e0 100644 --- a/manim/utils/hashing.py +++ b/manim/utils/hashing.py @@ -37,7 +37,7 @@ def default(self, obj): # NOTE : All module types objects are removed, because otherwise it throws ValueError: Circular reference detected if not. TODO if isinstance(cvardict[i], ModuleType): del cvardict[i] - return {"code": inspect.getsource(obj), "nonlocals": x} + return {"code": inspect.getsource(obj), "nonlocals": cvardict} elif isinstance(obj, np.ndarray): return list(obj) elif hasattr(obj, "__dict__"):