Automatically redirect to articles with same checksum

Author: benoit74

src/zimscraperlib/zim/creator.py

@@ -19,7 +19,9 @@
     - can be used to store a filepath and content read from it (not stored) """
 
 import datetime
+import hashlib
 import pathlib
+import re
 import weakref
 from typing import Any, Callable, Dict, Optional, Tuple, Union
 
@@ -104,6 +106,10 @@ def __init__(
 
         self.workaround_nocancel = workaround_nocancel
 
+        self.autodedup_filters = []
+
+        self.dedup_items = dict()
+
     def start(self):
         super().__enter__()
 
@@ -119,6 +125,9 @@ def update_metadata(self, **kwargs):
             for name, value in kwargs.items():
                 self.add_metadata(name, value)
 
+    def add_autodedup_filter(self, filter_regex: str):
+        self.autodedup_filters.append(re.compile(filter_regex))
+
     def add_item_for(
         self,
         path: str,
@@ -151,6 +160,31 @@ def add_item_for(
         if fpath is None and content is None:
             raise ValueError("One of fpath or content is required")
 
+        for dedup_filter in self.autodedup_filters:
+            if dedup_filter.match(path):
+                if content:
+                    digest = hashlib.sha256(content).digest()
+                else:
+                    sha256 = hashlib.sha256()
+                    with open(fpath, "rb") as f:
+                        while True:
+                            data = f.read(65536)  # lets read stuff in 64kb chunks!
+                            if not data:
+                                break
+                            sha256.update(data)
+                    digest = sha256.digest()
+
+                if digest in self.dedup_items:
+                    self.add_redirect(
+                        path=path,
+                        target_path=self.dedup_items[digest],
+                        title=title,
+                        is_front=is_front,
+                    )
+                    return path
+                self.dedup_items[digest] = path
+                break
+
         mimetype = mimetype_for(
             path=path, content=content, fpath=fpath, mimetype=mimetype
         )

tests/zim/test_zim_creator.py

@@ -125,6 +125,29 @@ def test_noindexlanguage(tmp_path):
     assert reader.has_title_index
     assert not reader.has_fulltext_index
 
+def test_duplicatefiles(tmp_path, png_image, html_file):
+    fpath = tmp_path / "test.zim"
+    with Creator(fpath, "welcome", "") as creator:
+        creator.add_autodedup_filter(r"^images/.*$")
+        # add a file not matching filter patterns
+        creator.add_item_for("other_folder1/yahoo0.png", "Image1", fpath=png_image)
+        # add same file but first matching filter patterns => will be added as-is
+        creator.add_item_for("images/yahoo1.png", "Image1", fpath=png_image)
+        # add same file but second matching filter patterns => will be replaced by a redirect
+        creator.add_item_for("images/yahoo2.png", "Image2", fpath=png_image)
+        # add same file but not matching filter patterns => will be added as-is
+        creator.add_item_for("other_folder2/yahoo3.png", "Image1", fpath=png_image)
+
+    reader = Archive(fpath)
+    # make sure we have our image
+    assert reader.get_item("images/yahoo1.png")
+    assert not reader.get_entry_by_path("images/yahoo1.png").is_redirect
+    assert reader.get_item("images/yahoo2.png")
+    assert reader.get_entry_by_path("images/yahoo2.png").is_redirect
+    assert reader.get_item("other_folder1/yahoo0.png")
+    assert not reader.get_entry_by_path("other_folder1/yahoo0.png").is_redirect
+    assert reader.get_item("other_folder2/yahoo3.png")
+    assert not reader.get_entry_by_path("other_folder2/yahoo3.png").is_redirect
 
 def test_add_item_for(tmp_path):
     fpath = tmp_path / "test.zim"