diff --git a/django_cron/admin.py b/django_cron/admin.py index af70b5a..3b6767f 100644 --- a/django_cron/admin.py +++ b/django_cron/admin.py @@ -2,7 +2,7 @@ from django.contrib import admin from django.db.models import F -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django_cron.models import CronJobLog from django_cron.helpers import humanize_duration diff --git a/django_cron/backends/lock/file.py b/django_cron/backends/lock/file.py index 7c54bef..82ee59c 100644 --- a/django_cron/backends/lock/file.py +++ b/django_cron/backends/lock/file.py @@ -15,6 +15,11 @@ class FileLock(DjangoCronJobLock): def lock(self): try: lock_name = self.get_lock_name() + + ## just quit, do not touch the file so we have date modified + if os.path.exists(lock_name): + return False + # need loop to avoid races on file unlinking while True: f = open(lock_name, 'wb+', 0) @@ -54,8 +59,10 @@ def release(self): f = self.lockfile # unlink before release lock to avoid race # see comment in self.lock for description - os.unlink(f.name) - f.close() + try: + os.unlink(f.name) + f.close() + except FileNotFoundError: pass def get_lock_name(self): default_path = '/tmp' diff --git a/django_cron/helpers.py b/django_cron/helpers.py index 5b40ac4..dedbae8 100644 --- a/django_cron/helpers.py +++ b/django_cron/helpers.py @@ -1,4 +1,4 @@ -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.template.defaultfilters import pluralize diff --git a/django_cron/management/commands/runcrons.py b/django_cron/management/commands/runcrons.py index 2e506ba..a300ebf 100644 --- a/django_cron/management/commands/runcrons.py +++ b/django_cron/management/commands/runcrons.py @@ -1,5 +1,6 @@ import traceback from datetime import timedelta +import threading from django.core.management.base import BaseCommand from django.conf import settings @@ -8,6 +9,8 @@ from django_cron import CronJobManager, get_class, get_current_time from django_cron.models import CronJobLog +from datetime import datetime +import pytz, os, logging DEFAULT_LOCK_TIME = 24 * 60 * 60 # 24 hours @@ -34,12 +37,39 @@ def handle(self, *args, **options): Iterates over all the CRON_CLASSES (or if passed in as a commandline argument) and runs them. """ + + main_cron = False + cron_classes = options['cron_classes'] if cron_classes: cron_class_names = cron_classes else: + #main_cron = True cron_class_names = getattr(settings, 'CRON_CLASSES', []) + if main_cron: + #for handler in logging.root.handlers[:]: + # logging.root.removeHandler(handler) + + logger = logging.getLogger(__name__) + + today = datetime.utcnow().replace(tzinfo=pytz.UTC).astimezone(pytz.timezone('Europe/Berlin')) + basename = os.path.join(os.path.dirname(__file__),"../../../log/",today.strftime("%Y/%m/%d")) + try: os.makedirs(basename) + except Exception: pass + folder = os.path.normpath(os.path.join(basename,today.strftime("%d-%m-%Y_%H-%M")+"_runcrons.log")) + + handler = logging.FileHandler(folder) + handler.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + logger.addHandler(handler) + + #logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',filename=folder,level=logging.INFO) + + logger.info("Running crons started "+today.strftime("%d-%m-%Y_%H-%M")) + #logger.info("Crons to run: "+ " ".join(cron_class_names)) + try: crons_to_run = [get_class(x) for x in cron_class_names] except Exception: @@ -47,15 +77,47 @@ def handle(self, *args, **options): self.stdout.write('Make sure these are valid cron class names: %s\n%s' % (cron_class_names, error)) return + threads = [] + single_threaded = [] for cron_class in crons_to_run: + if getattr(settings,'DJANGO_CRON_MULTITHREADED',False): + if hasattr(cron_class,"single_threaded") and cron_class.single_threaded: + single_threaded.append(cron_class) + else: + ## run all cron jobs in parallel as thread + th = threading.Thread( + target = run_cron_with_cache_check, + kwargs={ + "cron_class":cron_class, + "force":options['force'], + "silent":options['silent'] + } + ) + if main_cron: logger.info("Thread starting: "+str(cron_class)) + th.start() + if main_cron: logger.info("\tdone") + threads.append([th,cron_class]) + else: single_threaded.append(cron_class) + + for cron_class in single_threaded: + print("run singlethreaded: "+str(cron_class)) run_cron_with_cache_check( cron_class, force=options['force'], silent=options['silent'] ) + for th in threads: + if main_cron: logger.info("Wait for thread: "+str(th[1])) + th[0].join() + if main_cron: logger.info("done") + if main_cron: logger.info("clear old log entries") clear_old_log_entries() + if main_cron: logger.info("done") + if main_cron: logger.info("close old connections") close_old_connections() + if main_cron: logger.info("done") + if main_cron: logger.info("exit") def run_cron_with_cache_check(cron_class, force=False, silent=False): diff --git a/django_cron/migrations/0003_rename_cronjoblog_code_is_success_ran_at_time_django_cron_code_89ad04_idx_and_more.py b/django_cron/migrations/0003_rename_cronjoblog_code_is_success_ran_at_time_django_cron_code_89ad04_idx_and_more.py new file mode 100644 index 0000000..6e27adf --- /dev/null +++ b/django_cron/migrations/0003_rename_cronjoblog_code_is_success_ran_at_time_django_cron_code_89ad04_idx_and_more.py @@ -0,0 +1,28 @@ +# Generated by Django 4.2.6 on 2023-11-10 13:16 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('django_cron', '0002_remove_max_length_from_CronJobLog_message'), + ] + + operations = [ + migrations.RenameIndex( + model_name='cronjoblog', + new_name='django_cron_code_89ad04_idx', + old_fields=('code', 'is_success', 'ran_at_time'), + ), + migrations.RenameIndex( + model_name='cronjoblog', + new_name='django_cron_code_21f381_idx', + old_fields=('code', 'start_time', 'ran_at_time'), + ), + migrations.RenameIndex( + model_name='cronjoblog', + new_name='django_cron_code_966ed3_idx', + old_fields=('code', 'start_time'), + ), + ] diff --git a/django_cron/models.py b/django_cron/models.py index 73e093a..b42f0d6 100644 --- a/django_cron/models.py +++ b/django_cron/models.py @@ -20,9 +20,9 @@ def __unicode__(self): return '%s (%s)' % (self.code, 'Success' if self.is_success else 'Fail') class Meta: - index_together = [ - ('code', 'is_success', 'ran_at_time'), - ('code', 'start_time', 'ran_at_time'), - ('code', 'start_time') # useful when finding latest run (order by start_time) of cron + indexes = [ + models.Index(fields=('code', 'is_success', 'ran_at_time')), + models.Index(fields=('code', 'start_time', 'ran_at_time')), + models.Index(fields=('code', 'start_time')) # useful when finding latest run (order by start_time) of cron ] app_label = 'django_cron' diff --git a/docs/configuration.rst b/docs/configuration.rst index a5974e3..123578c 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -13,5 +13,6 @@ Configuration **DJANGO_CRON_DELETE_LOGS_OLDER_THAN** - integer, number of days after which log entries will be clear (optional - if not set no entries will be deleted) +**DJANGO_CRON_MULTITHREADED** - run all cronjobs in parallel by using threads, default: ``False`` For more details, see :doc:`Sample Cron Configurations ` and :doc:`Locking backend `