diff --git a/.gitignore b/.gitignore index fad4983..be1f18c 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,8 @@ db.sqlite3 # Vim *.swp +#Amber cscope files +cscope.files +cscope.out +mk_cscope +tags diff --git a/django_cron/admin.py b/django_cron/admin.py index c43365d..c68adc7 100644 --- a/django_cron/admin.py +++ b/django_cron/admin.py @@ -37,7 +37,7 @@ class Meta: search_fields = ('code', 'message') ordering = ('-start_time',) - list_display = ('code', 'start_time', 'end_time', 'humanize_duration', 'is_success') + list_display = ('code', 'start_time', 'end_time', 'humanize_duration', 'is_success', 'ran_at_time') list_filter = ('code', 'start_time', 'is_success', DurationFilter) def get_queryset(self, request): @@ -58,5 +58,9 @@ def humanize_duration(self, obj): humanize_duration.admin_order_field = 'duration' +class CronJobLockAdmin(admin.ModelAdmin): + list_display = ('pk', 'job_name', 'locked') + list_filter = ('locked',) + admin.site.register(CronJobLog, CronJobLogAdmin) -admin.site.register(CronJobLock) +admin.site.register(CronJobLock, CronJobLockAdmin) diff --git a/django_cron/backends/lock/base.py b/django_cron/backends/lock/base.py index a003b4a..6a09cdc 100644 --- a/django_cron/backends/lock/base.py +++ b/django_cron/backends/lock/base.py @@ -28,7 +28,7 @@ def __init__(self, cron_class, silent, *args, **kwargs): for you. The rest is backend-specific. """ self.job_name = '.'.join([cron_class.__module__, cron_class.__name__]) - self.job_code = cron_class.code + self.job_code = cron_class.get_code() self.parallel = getattr(cron_class, 'ALLOW_PARALLEL_RUNS', False) self.silent = silent @@ -55,7 +55,7 @@ def release(self): ) def lock_failed_message(self): - return "%s: lock found. Will try later." % self.job_name + return "%s: lock found. Will try later." % self.job_code def __enter__(self): if not self.parallel and not self.lock(): diff --git a/django_cron/backends/lock/cache.py b/django_cron/backends/lock/cache.py index 00c9353..dec9505 100644 --- a/django_cron/backends/lock/cache.py +++ b/django_cron/backends/lock/cache.py @@ -55,7 +55,9 @@ def get_cache_by_name(self): return caches[cache_name] def get_lock_name(self): - return self.job_name + #We are using IP based codes. Name is Check_Documents_Cron ie the Class name. This is terrible since 2 apps can have the same class name + #Code is fmeca.check_documents_cron.203.23.34.56 + return self.job_code def get_cache_timeout(self, cron_class): try: diff --git a/django_cron/backends/lock/database.py b/django_cron/backends/lock/database.py index 3bf78a1..b89d4f7 100644 --- a/django_cron/backends/lock/database.py +++ b/django_cron/backends/lock/database.py @@ -11,7 +11,7 @@ class DatabaseLock(DjangoCronJobLock): @transaction.atomic def lock(self): - lock, created = CronJobLock.objects.get_or_create(job_name=self.job_name) + lock, created = CronJobLock.objects.using("default").get_or_create(job_name=self.job_code) if lock.locked: return False else: @@ -21,6 +21,7 @@ def lock(self): @transaction.atomic def release(self): - lock = CronJobLock.objects.filter(job_name=self.job_name, locked=True).first() - lock.locked = False - lock.save() + lock = CronJobLock.objects.using("default").filter(job_name=self.job_code, locked=True).first() + if lock: + lock.locked = False + lock.save() diff --git a/django_cron/backends/lock/file.py b/django_cron/backends/lock/file.py index 1d14d31..dd3621b 100644 --- a/django_cron/backends/lock/file.py +++ b/django_cron/backends/lock/file.py @@ -33,5 +33,5 @@ def get_lock_name(self): # let it die if failed, can't run further anyway os.makedirs(path, exist_ok=True) - filename = self.job_name + '.lock' + filename = self.job_code + '.lock' return os.path.join(path, filename) diff --git a/django_cron/core.py b/django_cron/core.py index 9d5a7e6..8b8f3a1 100644 --- a/django_cron/core.py +++ b/django_cron/core.py @@ -1,5 +1,5 @@ import logging -from datetime import datetime, timedelta +from datetime import timedelta import traceback import time import sys @@ -7,9 +7,12 @@ from django.conf import settings from django.utils.timezone import now as utc_now from django.db.models import Q - +from subprocess import check_output from django_cron.helpers import get_class, get_current_time +from django_common.helper import send_mail + + DEFAULT_LOCK_BACKEND = 'django_cron.backends.lock.cache.CacheLock' DJANGO_CRON_OUTPUT_ERRORS = False logger = logging.getLogger('django_cron') @@ -48,12 +51,29 @@ class CronJobBase(object): Following functions: + do - This is the actual business logic to be run at the given schedule """ - + SEND_FAILED_EMAIL = [] remove_successful_cron_logs = False def __init__(self): self.prev_success_cron = None + @classmethod + def get_code(self): + try: + if self.APPEND_IP_TO_CODE: + myserverip = None + try: + myserverip = check_output(['/usr/bin/ec2metadata', '--public-ipv4']) + myserverip = myserverip.strip() + except: + pass + if not myserverip: + myserverip = check_output(['hostname', '-I']) + myserverip = myserverip.strip() + return self.code + '-' + myserverip.decode('utf-8') + except: + return self.code + def set_prev_success_cron(self, prev_success_cron): self.prev_success_cron = prev_success_cron @@ -93,6 +113,9 @@ def __init__(self, cron_job_class, silent=False, dry_run=False, stdout=None): self.write_log = getattr( settings, 'DJANGO_CRON_OUTPUT_ERRORS', DJANGO_CRON_OUTPUT_ERRORS ) + self.send_error_email = getattr(settings, + 'DJANGO_CRON_SEND_IMMEDIATE_ERROR_EMAIL', + False) def should_run_now(self, force=False): from django_cron.models import CronJobLog @@ -109,19 +132,19 @@ def should_run_now(self, force=False): return True if cron_job.schedule.run_monthly_on_days is not None: - if not datetime.today().day in cron_job.schedule.run_monthly_on_days: + if not get_current_time().day in cron_job.schedule.run_monthly_on_days: return False if cron_job.schedule.run_weekly_on_days is not None: - if not datetime.today().weekday() in cron_job.schedule.run_weekly_on_days: + if not get_current_time().weekday() in cron_job.schedule.run_weekly_on_days: return False if cron_job.schedule.retry_after_failure_mins: # We check last job - success or not last_job = ( - CronJobLog.objects.filter(code=cron_job.code) + CronJobLog.objects.filter(code=cron_job.get_code()) .order_by('-start_time') - .exclude(start_time__gt=datetime.today()) + .exclude(start_time__gt=get_current_time()) .first() ) if ( @@ -136,8 +159,8 @@ def should_run_now(self, force=False): if cron_job.schedule.run_every_mins is not None: try: self.previously_ran_successful_cron = CronJobLog.objects.filter( - code=cron_job.code, is_success=True - ).exclude(start_time__gt=datetime.today()).latest('start_time') + code=cron_job.get_code(), is_success=True + ).exclude(start_time__gt=get_current_time()).latest('start_time') except CronJobLog.DoesNotExist: pass @@ -158,7 +181,7 @@ def should_run_now(self, force=False): actual_time = time.strptime("%s:%s" % (now.hour, now.minute), "%H:%M") if actual_time >= user_time: qset = CronJobLog.objects.filter( - code=cron_job.code, ran_at_time=time_data, is_success=True + code=cron_job.get_code(), ran_at_time=time_data, is_success=True ).filter( Q(start_time__gt=now) | Q( @@ -177,7 +200,7 @@ def make_log(self, *messages, **kwargs): cron_log = self.cron_log cron_job = getattr(self, 'cron_job', self.cron_job_class) - cron_log.code = cron_job.code + cron_log.code = cron_job.get_code() cron_log.is_success = kwargs.get('success', True) cron_log.message = self.make_log_msg(messages) @@ -187,6 +210,36 @@ def make_log(self, *messages, **kwargs): if not cron_log.is_success and self.write_log: logger.error("%s cronjob error:\n%s" % (cron_log.code, cron_log.message)) + + if self.send_error_email and not cron_log.is_success: + from django_cron.models import CronJobLog + try: + emails = [admin[1] for admin in settings.ADMINS] + if getattr(cron_job, "SEND_FAILED_EMAIL", []): + emails.extend(cron_job.SEND_FAILED_EMAIL) + + failed_runs_cronjob_email_prefix = getattr(settings, 'FAILED_RUNS_CRONJOB_EMAIL_PREFIX', '') + min_failures = getattr(cron_job, 'MIN_NUM_FAILURES', 10) + if not min_failures: + min_failures = 10 + + last_min_cron_status = list(CronJobLog.objects.using("default").filter( + code=cron_log.code).order_by("-end_time").values_list("is_success", flat=True)[:min_failures]) + + #All of them should be failed ie false. Then only we have to send email + # Send on 3 failures. [True, False, False] ie [success, failed, failed] does not trigger email + if not any(last_min_cron_status): + send_mail( + '%s%s failed %s times in a row!' % ( + failed_runs_cronjob_email_prefix, + cron_log.code, + min_failures, + ), + cron_log.message, + settings.DEFAULT_FROM_EMAIL, emails + ) + except Exception as e: + logger.exception(e) def make_log_msg(self, messages): full_message = '' @@ -260,7 +313,7 @@ def run(self, force=False): logger.debug( "Running cron: %s code %s", cron_job_class.__name__, - self.cron_job.code, + self.cron_job.get_code(), ) self.make_log('Job in progress', success=True) self.msg = self.cron_job.do() diff --git a/django_cron/cron.py b/django_cron/cron.py index 8d65f49..44b3104 100644 --- a/django_cron/cron.py +++ b/django_cron/cron.py @@ -28,7 +28,7 @@ def do(self): for cron in crons_to_check: min_failures = getattr(cron, 'MIN_NUM_FAILURES', 10) - jobs = CronJobLog.objects.filter(code=cron.code).order_by('-end_time')[ + jobs = CronJobLog.objects.filter(code=cron.get_code()).order_by('-end_time')[ :min_failures ] failures = 0 diff --git a/django_cron/management/commands/runcrons.py b/django_cron/management/commands/runcrons.py index 07fa567..9d0f8c2 100644 --- a/django_cron/management/commands/runcrons.py +++ b/django_cron/management/commands/runcrons.py @@ -5,6 +5,7 @@ from django.core.management.base import BaseCommand from django.conf import settings from django.db import close_old_connections +from django_common.helper import send_mail from django_cron import CronJobManager, get_class, get_current_time from django_cron.models import CronJobLog @@ -25,17 +26,28 @@ def add_arguments(self, parser): action='store_true', help="Just show what crons would be run; don't actually run them", ) + parser.add_argument( + '--run_class_list_name', + nargs='?', + help='Runs all the crons in the specified class list from settings. This is to override CRON_CLASSES hardcoding' + ) def handle(self, *args, **options): """ Iterates over all the CRON_CLASSES (or if passed in as a commandline argument) and runs them. """ + if not options['silent']: self.stdout.write("Running Crons\n") self.stdout.write("{0}\n".format("=" * 40)) cron_classes = options['cron_classes'] + + if options['run_class_list_name']: + list_name = options['run_class_list_name'] + cron_classes = getattr(settings, list_name, []) + if cron_classes: cron_class_names = cron_classes else: @@ -44,11 +56,20 @@ def handle(self, *args, **options): try: crons_to_run = [get_class(x) for x in cron_class_names] except ImportError: + # Send an email to admin when the module load fails error = traceback.format_exc() - self.stdout.write( - 'ERROR: Make sure these are valid cron class names: %s\n\n%s' - % (cron_class_names, error) - ) + self.stdout.write('ERROR: Make sure these are valid cron class names: %s\n\n%s' % (cron_class_names, error)) + try: + emails = [admin[1] for admin in settings.ADMINS] + failed_runs_cronjob_email_prefix = getattr(settings, 'FAILED_RUNS_CRONJOB_EMAIL_PREFIX', '') + send_mail( + "URGENT!!! {} Error while importing crons".format(failed_runs_cronjob_email_prefix), + error, + settings.DEFAULT_FROM_EMAIL, emails + ) + except Exception as e: + self.stdout.write( + 'ERROR: While sending email: %s' % (e)) return for cron_class in crons_to_run: diff --git a/django_cron/models.py b/django_cron/models.py index c109e09..8e49cc2 100644 --- a/django_cron/models.py +++ b/django_cron/models.py @@ -15,13 +15,14 @@ class CronJobLog(models.Model): # This field is used to mark jobs executed in exact time. # Jobs that run every X minutes, have this field empty. - ran_at_time = models.TimeField(null=True, blank=True, db_index=True, editable=False) + ran_at_time = models.TimeField(null=True, blank=True, db_index=True, editable=True) def __unicode__(self): return '%s (%s)' % (self.code, 'Success' if self.is_success else 'Fail') - + def __str__(self): - return "%s (%s)" % (self.code, "Success" if self.is_success else "Fail") + return '%s (%s)' % (self.code, 'Success' if self.is_success else 'Fail') + class Meta: index_together = [ @@ -38,3 +39,6 @@ class Meta: class CronJobLock(models.Model): job_name = models.CharField(max_length=200, unique=True) locked = models.BooleanField(default=False) + + def __str__(self): + return '%s (%s)' % (self.job_name, self.locked) \ No newline at end of file diff --git a/setup.py b/setup.py index 3d057dc..4ee5bea 100644 --- a/setup.py +++ b/setup.py @@ -20,10 +20,10 @@ setup( name='django-cron', - version='0.6.0', + version='0.6.3', author='Sumit Chachra', author_email='chachra@tivix.com', - url='http://github.com/tivix/django-cron', + url='https://github.com/Trendlyne-technologies/django-cron', description='Running python crons in a Django project', packages=find_packages(), long_description=long_description,