added setting to run all cronjobs in parallel

django_cron/admin.py

@@ -2,7 +2,7 @@
 
 from django.contrib import admin
 from django.db.models import F
-from django.utils.translation import ugettext_lazy as _
+from django.utils.translation import gettext_lazy as _
 
 from django_cron.models import CronJobLog
 from django_cron.helpers import humanize_duration

django_cron/backends/lock/file.py

@@ -15,6 +15,11 @@ class FileLock(DjangoCronJobLock):
     def lock(self):
         try:
             lock_name = self.get_lock_name()
+
+            ## just quit, do not touch the file so we have date modified
+            if os.path.exists(lock_name):
+                return False
+
             # need loop to avoid races on file unlinking
             while True:
                 f = open(lock_name, 'wb+', 0)
@@ -54,8 +59,10 @@ def release(self):
         f = self.lockfile
         # unlink before release lock to avoid race
         # see comment in self.lock for description
-        os.unlink(f.name)
-        f.close()
+        try:
+            os.unlink(f.name)
+            f.close()
+        except FileNotFoundError: pass  
 
     def get_lock_name(self):
         default_path = '/tmp'

django_cron/helpers.py

@@ -1,4 +1,4 @@
-from django.utils.translation import ugettext as _
+from django.utils.translation import gettext as _
 from django.template.defaultfilters import pluralize
 
 

django_cron/management/commands/runcrons.py

@@ -1,5 +1,6 @@
 import traceback
 from datetime import timedelta
+import threading
 
 from django.core.management.base import BaseCommand
 from django.conf import settings
@@ -8,6 +9,8 @@
 from django_cron import CronJobManager, get_class, get_current_time
 from django_cron.models import CronJobLog
 
+from datetime import datetime
+import pytz, os, logging
 
 DEFAULT_LOCK_TIME = 24 * 60 * 60  # 24 hours
 
@@ -34,28 +37,87 @@ def handle(self, *args, **options):
         Iterates over all the CRON_CLASSES (or if passed in as a commandline argument)
         and runs them.
         """
+
+        main_cron = False
+
         cron_classes = options['cron_classes']
         if cron_classes:
             cron_class_names = cron_classes
         else:
+            #main_cron = True
             cron_class_names = getattr(settings, 'CRON_CLASSES', [])
 
+        if main_cron:
+            #for handler in logging.root.handlers[:]:
+            #    logging.root.removeHandler(handler)
+
+            logger = logging.getLogger(__name__)
+
+            today = datetime.utcnow().replace(tzinfo=pytz.UTC).astimezone(pytz.timezone('Europe/Berlin'))
+            basename = os.path.join(os.path.dirname(__file__),"../../../log/",today.strftime("%Y/%m/%d"))
+            try: os.makedirs(basename)
+            except Exception: pass
+            folder = os.path.normpath(os.path.join(basename,today.strftime("%d-%m-%Y_%H-%M")+"_runcrons.log"))
+
+            handler = logging.FileHandler(folder)
+            handler.setLevel(logging.INFO)
+            formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+            handler.setFormatter(formatter)
+            logger.addHandler(handler)
+
+            #logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',filename=folder,level=logging.INFO)
+
+            logger.info("Running crons started "+today.strftime("%d-%m-%Y_%H-%M"))
+            #logger.info("Crons to run: "+ " ".join(cron_class_names))
+
         try:
             crons_to_run = [get_class(x) for x in cron_class_names]
         except Exception:
             error = traceback.format_exc()
             self.stdout.write('Make sure these are valid cron class names: %s\n%s' % (cron_class_names, error))
             return
 
+        threads = []
+        single_threaded = []
         for cron_class in crons_to_run:
+            if getattr(settings,'DJANGO_CRON_MULTITHREADED',False):
+                if hasattr(cron_class,"single_threaded") and cron_class.single_threaded:
+                    single_threaded.append(cron_class)
+                else:
+                    ## run all cron jobs in parallel as thread
+                    th = threading.Thread(
+                        target = run_cron_with_cache_check, 
+                        kwargs={
+                            "cron_class":cron_class,
+                            "force":options['force'],
+                            "silent":options['silent']
+                        }
+                    )
+                    if main_cron: logger.info("Thread starting: "+str(cron_class))
+                    th.start()
+                    if main_cron: logger.info("\tdone")
+                    threads.append([th,cron_class])
+            else: single_threaded.append(cron_class)
+
+        for cron_class in single_threaded:
+            print("run singlethreaded: "+str(cron_class))
             run_cron_with_cache_check(
                 cron_class,
                 force=options['force'],
                 silent=options['silent']
             )
 
+        for th in threads:
+            if main_cron: logger.info("Wait for thread: "+str(th[1]))
+            th[0].join()
+            if main_cron: logger.info("done")
+        if main_cron: logger.info("clear old log entries")
         clear_old_log_entries()
+        if main_cron: logger.info("done")
+        if main_cron: logger.info("close old connections")
         close_old_connections()
+        if main_cron: logger.info("done")
+        if main_cron: logger.info("exit")
 
 
 def run_cron_with_cache_check(cron_class, force=False, silent=False):

django_cron/migrations/0003_rename_cronjoblog_code_is_success_ran_at_time_django_cron_code_89ad04_idx_and_more.py

@@ -0,0 +1,28 @@
+# Generated by Django 4.2.6 on 2023-11-10 13:16
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('django_cron', '0002_remove_max_length_from_CronJobLog_message'),
+    ]
+
+    operations = [
+        migrations.RenameIndex(
+            model_name='cronjoblog',
+            new_name='django_cron_code_89ad04_idx',
+            old_fields=('code', 'is_success', 'ran_at_time'),
+        ),
+        migrations.RenameIndex(
+            model_name='cronjoblog',
+            new_name='django_cron_code_21f381_idx',
+            old_fields=('code', 'start_time', 'ran_at_time'),
+        ),
+        migrations.RenameIndex(
+            model_name='cronjoblog',
+            new_name='django_cron_code_966ed3_idx',
+            old_fields=('code', 'start_time'),
+        ),
+    ]

django_cron/models.py

@@ -20,9 +20,9 @@ def __unicode__(self):
         return '%s (%s)' % (self.code, 'Success' if self.is_success else 'Fail')
 
     class Meta:
-        index_together = [
-            ('code', 'is_success', 'ran_at_time'),
-            ('code', 'start_time', 'ran_at_time'),
-            ('code', 'start_time')  # useful when finding latest run (order by start_time) of cron
+        indexes = [
+            models.Index(fields=('code', 'is_success', 'ran_at_time')),
+            models.Index(fields=('code', 'start_time', 'ran_at_time')),
+            models.Index(fields=('code', 'start_time'))  # useful when finding latest run (order by start_time) of cron
         ]
         app_label = 'django_cron'

docs/configuration.rst

@@ -13,5 +13,6 @@ Configuration
 
 **DJANGO_CRON_DELETE_LOGS_OLDER_THAN** - integer, number of days after which log entries will be clear (optional - if not set no entries will be deleted)
 
+**DJANGO_CRON_MULTITHREADED** - run all cronjobs in parallel by using threads, default: ``False``
 
 For more details, see :doc:`Sample Cron Configurations <sample_cron_configurations>` and :doc:`Locking backend <locking_backend>`