diff --git a/docker/Dockerfile b/docker/Dockerfile index 31e8f16..6a52141 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -93,12 +93,7 @@ RUN ln -s ${HOME}/etc/nginx/sites-available/sigi.vhost /etc/nginx/sites-enabled/ RUN mkdir -p /var/log/sigi # schedule cron jobs -RUN crontab -l | { cat; echo "* * * * * /usr/local/bin/python ${HOME}/manage.py runjobs minutely >> /var/log/sigi/cron.log 2>&1"; } | crontab - -RUN crontab -l | { cat; echo "0 * * * * /usr/local/bin/python ${HOME}/manage.py runjobs hourly >> /var/log/sigi/cron.log 2>&1"; } | crontab - -RUN crontab -l | { cat; echo "0 0 * * * /usr/local/bin/python ${HOME}/manage.py runjobs daily >> /var/log/sigi/cron.log 2>&1"; } | crontab - -RUN crontab -l | { cat; echo "0 0 * * 0 /usr/local/bin/python ${HOME}/manage.py runjobs weekly >> /var/log/sigi/cron.log 2>&1"; } | crontab - -RUN crontab -l | { cat; echo "0 0 1 * * /usr/local/bin/python ${HOME}/manage.py runjobs monthly >> /var/log/sigi/cron.log 2>&1"; } | crontab - -RUN crontab -l | { cat; echo "0 4 * * * /usr/local/bin/python ${HOME}/manage.py runjob sincroniza_saberes >> /var/log/sigi/cron.log 2>&1"; } | crontab - +RUN crontab -l | { cat; echo "* * * * * /usr/local/bin/python ${HOME}/manage.py runjob job_controller >> /var/log/sigi/cron.log 2>&1"; } | crontab - EXPOSE 80/tcp 443/tcp ENV DEBIAN_FRONTEND=teletype diff --git a/requirements/requirements.txt b/requirements/requirements.txt index b29a62e..022fd2c 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -1,3 +1,4 @@ +cron-converter==1.0.2 dnspython==2.3.0 docutils==0.20.1 gunicorn==20.1.0 diff --git a/sigi/apps/utils/admin.py b/sigi/apps/utils/admin.py index ac8a685..0dcbf20 100644 --- a/sigi/apps/utils/admin.py +++ b/sigi/apps/utils/admin.py @@ -1,7 +1,44 @@ -from django.contrib import admin -from sigi.apps.utils.models import SigiAlert +from django.contrib import admin, messages +from django.core.exceptions import PermissionDenied +from django.http import HttpRequest +from django.shortcuts import get_object_or_404, redirect +from django.urls import reverse, path +from django.utils import timezone +from django.utils.formats import localize +from django.utils.safestring import mark_safe +from django.utils.translation import gettext as _ +from django_extensions.management.jobs import get_job, get_jobs from tinymce.models import HTMLField from tinymce.widgets import AdminTinyMCE +from sigi.apps.utils.models import SigiAlert, Cronjob, JobSchedule + + +class JobScheduleInline(admin.TabularInline): + model = JobSchedule + fields = ["status", "iniciar", "iniciado", "tempo_gasto", "get_runner"] + readonly_fields = [ + "status", + "iniciar", + "iniciado", + "tempo_gasto", + "get_runner", + ] + can_delete = False + can_add = False + extra = 0 + + def has_add_permission(self, request, obj): + return False + + @mark_safe + @admin.display(description=_("executar")) + def get_runner(self, sched): + if sched.status == JobSchedule.STATUS_AGENDADO: + url = reverse("admin:utils_jobschedule_runjob", args=[sched.id]) + return ( + f"play_arrow" + ) + return "" @admin.register(SigiAlert) @@ -10,3 +47,161 @@ class SigiAlertAdmin(admin.ModelAdmin): search_fields = ("titulo", "caminho") formfield_overrides = {HTMLField: {"widget": AdminTinyMCE}} list_filter = ("destinatarios",) + + +@admin.register(Cronjob) +class CronjobAdmin(admin.ModelAdmin): + list_display = ( + "job_name", + "app_name", + "get_help", + "expressao_cron", + "get_schedule", + "get_runner", + ) + fields = ["job_name", "app_name", "get_help", "expressao_cron"] + readonly_fields = ("job_name", "app_name", "get_help") + inlines = [JobScheduleInline] + + def get_urls(self): + urls = super().get_urls() + model_info = (self.model._meta.app_label, self.model._meta.model_name) + + my_urls = [ + path( + "/runjob/", + self.admin_site.admin_view(self.run_job), + name="%s_%s_runjob" % model_info, + ), + ] + return my_urls + urls + + @admin.display(description=_("descrição")) + def get_help(self, job): + try: + JobClass = get_job(job.app_name, job.job_name) + except KeyError: + return _( + f"A rotina de JOB {job.app_name}.{job.job_name} " + "não foi encontrada." + ) + job_obj = JobClass() + return job_obj.help + + @admin.display(description=_("agenda")) + def get_schedule(self, job): + sched = job.jobschedule_set.first() + if sched is None: + return _("Nenhum agendamento para este job") + if sched.status == JobSchedule.STATUS_AGENDADO: + return _( + "início agendado para " + f"{localize(timezone.localtime(sched.iniciar))}." + ) + if sched.status == JobSchedule.STATUS_EXECUTANDO: + return _( + "em execução desde " + f"{localize(timezone.localtime(sched.iniciado))}" + ) + return _( + f"executado em {localize(timezone.localtime(sched.iniciado))}, " + f"levando {sched.tempo_gasto} minutos para concluir" + ) + + @mark_safe + @admin.display(description=_("executar")) + def get_runner(self, job): + url = reverse("admin:utils_cronjob_runjob", args=[job.id]) + return ( + f"" "play_arrow" + ) + + def run_job(self, request, object_id): + cronjob = get_object_or_404(Cronjob, id=object_id) + sched = cronjob.next_schedule() + if sched.status != JobSchedule.STATUS_AGENDADO: + raise PermissionDenied( + _( + "Este agendamento não pode ser executado pois " + f"está com status {sched.get_status_display()}" + ) + ) + sched.run_job() + self.message_user( + request, + _("JOB executado!"), + messages.SUCCESS, + ) + return redirect("admin:utils_jobschedule_change", object_id=sched.id) + + +@admin.register(JobSchedule) +class JobScheduleAdmin(admin.ModelAdmin): + list_display = [ + "job", + "status", + "iniciar", + "iniciado", + "tempo_gasto", + "get_runner", + ] + fields = [ + "job", + "status", + "iniciar", + "iniciado", + "tempo_gasto", + ] + readonly_fields = fields + list_filter = ("status", "job") + date_hierarchy = "iniciar" + + def get_urls(self): + urls = super().get_urls() + model_info = (self.model._meta.app_label, self.model._meta.model_name) + + my_urls = [ + path( + "/runjob/", + self.admin_site.admin_view(self.run_job), + name="%s_%s_runjob" % model_info, + ), + ] + return my_urls + urls + + def has_add_permission(self, request): + return False + + def has_delete_permission(self, request, obj=None): + return False + + def has_change_permission(self, request, obj=None): + return False + + @mark_safe + @admin.display(description=_("executar")) + def get_runner(self, sched): + if sched.status == JobSchedule.STATUS_AGENDADO: + url = reverse("admin:utils_jobschedule_runjob", args=[sched.id]) + return ( + f"" + "play_arrow" + ) + return "" + + def run_job(self, request, object_id): + sched = get_object_or_404(JobSchedule, id=object_id) + if sched.status != JobSchedule.STATUS_AGENDADO: + raise PermissionDenied( + _( + "Este agendamento não pode ser executado pois " + f"está com status {sched.get_status_display()}" + ) + ) + sched.run_job() + self.message_user( + request, + _("JOB executado!"), + messages.SUCCESS, + ) + return redirect("admin:utils_jobschedule_change", object_id=object_id) diff --git a/sigi/apps/utils/jobs/__init__.py b/sigi/apps/utils/jobs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sigi/apps/utils/jobs/daily/__init__.py b/sigi/apps/utils/jobs/daily/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sigi/apps/utils/jobs/hourly/__init__.py b/sigi/apps/utils/jobs/hourly/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sigi/apps/utils/jobs/job_controller.py b/sigi/apps/utils/jobs/job_controller.py new file mode 100644 index 0000000..a53c500 --- /dev/null +++ b/sigi/apps/utils/jobs/job_controller.py @@ -0,0 +1,90 @@ +from django_extensions.management.jobs import BaseJob +from django_extensions.management.jobs import get_jobs +from django.utils import timezone +from django.utils.formats import localize +from django.utils.translation import gettext as _ +from sigi.apps.utils.models import Cronjob, JobSchedule + +WHEN_SETS = { + "daily": "0 0 * * *", + "hourly": "0 * * * *", + "monthly": "0 0 1 * *", + "weekly": "0 0 * * 0", + "yearly": "0 0 1 1 *", + "minutely": "* * * * *", +} + + +class Job(BaseJob): + help = "Controlador de cronjobs do SIGI." + + def execute(self): + print("Rodando controlador de jobs...") + self.remove_old_jobs() + self.sync_new_jobs() + self.run_scheduled() + self.schedule_jobs() + + def remove_old_jobs(self): + """Remover das tabelas os jobs que foram removidos do código""" + print("\tRemover das tabelas os jobs que foram removidos do código...") + all_jobs = get_jobs() + excludes = Cronjob.objects.all() + for app_name, job_name in all_jobs.keys(): + excludes = excludes.exclude(app_name=app_name, job_name=job_name) + print("\t\t", excludes.delete()) + + def sync_new_jobs(self): + """ + Atualizar a tabela de JOBS com os novos JOBS que tenham sido criados + """ + print( + "\tAtualizar a tabela de JOBS com os novos JOBS que tenham " + "sido criados..." + ) + all_jobs = get_jobs() + for (app_name, job_name), JobClass in all_jobs.items(): + if app_name == "sigi.apps.utils" and job_name == "job_controller": + # Ignorar job_controller + continue + try: + job = Cronjob.objects.get(app_name=app_name, job_name=job_name) + except Cronjob.DoesNotExist: + # Inserir o JOB na tabela de JOBS # + job_obj = JobClass() + if job_obj.when in WHEN_SETS: + expressao_cron = WHEN_SETS[job_obj.when] + else: + expressao_cron = WHEN_SETS["daily"] # Default + job = Cronjob( + app_name=app_name, + job_name=job_name, + expressao_cron=expressao_cron, + ) + job.save() + print(f"\t\tNovo job encontrado: {job_name}: {job_obj.help}") + + def run_scheduled(self): + """Executa os jobs que estão agendados""" + print("\tExecutar os jobs que estão agendados...") + for sched in JobSchedule.objects.filter( + status=JobSchedule.STATUS_AGENDADO + ): + agora = timezone.localtime() + if sched.iniciar <= agora: + sched.run_job() + + def schedule_jobs(self): + """Criar agenda para próxima execução""" + print("\tCriar agenda para próxima execução...") + for job in Cronjob.objects.exclude( + jobschedule__status__in=[ + JobSchedule.STATUS_AGENDADO, + JobSchedule.STATUS_EXECUTANDO, + ] + ): + sched = job.next_schedule() + print( + f"\t\tAgendado job {sched.job.job_name} " + f"para {localize(sched.iniciar)}" + ) diff --git a/sigi/apps/utils/jobs/monthly/__init__.py b/sigi/apps/utils/jobs/monthly/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sigi/apps/utils/jobs/weekly/__init__.py b/sigi/apps/utils/jobs/weekly/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sigi/apps/utils/jobs/yearly/__init__.py b/sigi/apps/utils/jobs/yearly/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/sigi/apps/utils/migrations/0002_cronjob_jobschedule.py b/sigi/apps/utils/migrations/0002_cronjob_jobschedule.py new file mode 100644 index 0000000..1c45a0e --- /dev/null +++ b/sigi/apps/utils/migrations/0002_cronjob_jobschedule.py @@ -0,0 +1,45 @@ +# Generated by Django 4.2.7 on 2024-02-26 12:53 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('utils', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Cronjob', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('app_name', models.CharField(editable=False, max_length=100, verbose_name='app')), + ('job_name', models.CharField(editable=False, max_length=100, verbose_name='job')), + ('expressao_cron', models.CharField(default='* * * * *', help_text="Usar expressoões no formato padrão de CRON: 'minute hour day month day-of-week'. Mais detalhes: CronHowTo", max_length=100, verbose_name='expressão CRON')), + ], + options={ + 'verbose_name': 'Cron job', + 'verbose_name_plural': 'Cron jobs', + 'ordering': ('app_name', 'job_name'), + }, + ), + migrations.CreateModel( + name='JobSchedule', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('iniciar', models.DateTimeField(verbose_name='Iniciar em')), + ('iniciado', models.DateTimeField(blank=True, null=True, verbose_name='Iniciado em')), + ('status', models.CharField(choices=[('A', 'Agendado'), ('E', 'Executando'), ('C', 'Concluído')], default='A', max_length=1, verbose_name='estado')), + ('tempo_gasto', models.DurationField(blank=True, editable=False, null=True, verbose_name='tempo gasto')), + ('resultado', models.TextField(blank=True, editable=False, verbose_name='resultado da execução')), + ('job', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='utils.cronjob', verbose_name='Cron job')), + ], + options={ + 'verbose_name': 'Agenda de execução', + 'verbose_name_plural': 'Agenda de execuções', + 'ordering': ('iniciar',), + }, + ), + ] diff --git a/sigi/apps/utils/models.py b/sigi/apps/utils/models.py index 2858e3f..46c4af4 100644 --- a/sigi/apps/utils/models.py +++ b/sigi/apps/utils/models.py @@ -1,8 +1,15 @@ +import io +from contextlib import redirect_stderr, redirect_stdout +from cron_converter import Cron from pyexpat import model from django.db import models from django.contrib.auth.models import Group +from django.utils import timezone +from django.utils.formats import localize from django.utils.translation import gettext as _ +from django_extensions.management.jobs import get_job, get_jobs from tinymce.models import HTMLField +from sigi.apps.utils.management.jobs import JobReportMixin class SigiAlert(models.Model): @@ -26,3 +33,152 @@ class SigiAlert(models.Model): def __str__(self): return self.titulo + + +class Cronjob(models.Model): + app_name = models.CharField(_("app"), max_length=100, editable=False) + job_name = models.CharField(_("job"), max_length=100, editable=False) + expressao_cron = models.CharField( + _("expressão CRON"), + max_length=100, + default="* * * * *", + help_text=_( + "Usar expressoões no formato padrão de CRON: " + "'minute hour day month day-of-week'. " + "Mais detalhes: " + "" + "CronHowTo" + "" + ), + ) + + class Meta: + ordering = ("app_name", "job_name") + verbose_name = _("Cron job") + verbose_name_plural = _("Cron jobs") + + def __str__(self): + return self.job_name + + def run(self): + try: + JobClass = get_job(self.app_name, self.job_name) + except KeyError: + return ( + f"A rotina de JOB {self.job.job_name} do app " + f"{self.job.app_name} não foi encontrada." + ) + try: + job_obj = JobClass() + with io.StringIO() as so_buf, io.StringIO() as se_buf, redirect_stdout( + so_buf + ), redirect_stderr( + se_buf + ): + job_obj.execute() + messages = so_buf.getvalue() + errors = se_buf.getvalue() + report_data = ["", "MENSAGENS", "---------", ""] + if messages: + report_data.extend(messages.splitlines()) + else: + report_data.extend(["Nenhuma mensagem gerada", ""]) + report_data.extend(["", "ERROS", "-----", ""]) + if errors: + report_data.extend(errors.splitlines()) + else: + report_data.extend(["Nenhum erro gerado", ""]) + return "\n".join(report_data) + except Exception as e: + # Qualquer erro deve ser reportado + return _(f"JOB abortado com erro: {str(e)}") + + def next_schedule(self): + """Recupera a agenda da próxima execução. Se não existe, cria.""" + try: + sch = self.jobschedule_set.get( + status__in=[ + JobSchedule.STATUS_AGENDADO, + JobSchedule.STATUS_EXECUTANDO, + ] + ) + except JobSchedule.DoesNotExist: + iniciar = self.get_next_schedule_time() + sch = JobSchedule(job=self, iniciar=iniciar) + sch.save() + return sch + + def get_next_schedule_time(self): + cron_instance = Cron(self.expressao_cron) + scheduller = cron_instance.schedule(timezone.localtime()) + return scheduller.next() + + +class JobSchedule(models.Model): + STATUS_AGENDADO = "A" + STATUS_EXECUTANDO = "E" + STATUS_CONCLUIDO = "C" + STATUS_CHOICES = ( + (STATUS_AGENDADO, _("Agendado")), + (STATUS_EXECUTANDO, _("Executando")), + (STATUS_CONCLUIDO, _("Concluído")), + ) + job = models.ForeignKey( + Cronjob, verbose_name=_("Cron job"), on_delete=models.CASCADE + ) + iniciar = models.DateTimeField(_("Iniciar em")) + iniciado = models.DateTimeField(_("Iniciado em"), blank=True, null=True) + status = models.CharField( + _("estado"), + max_length=1, + choices=STATUS_CHOICES, + default=STATUS_AGENDADO, + ) + tempo_gasto = models.DurationField( + _("tempo gasto"), blank=True, null=True, editable=False + ) + resultado = models.TextField( + _("resultado da execução"), blank=True, editable=False + ) + + class Meta: + ordering = ("iniciar",) + verbose_name = _("Agenda de execução") + verbose_name_plural = _("Agenda de execuções") + + class DoesNotExecute(Exception): + """This scheduled job cannot be executed because not in AGENDADO state""" + + pass + + def __str__(self): + if self.status == JobSchedule.STATUS_AGENDADO: + return _( + f"{self.job.job_name}: início agendado para " + f"{localize(timezone.localtime(self.iniciar))}." + ) + elif self.status == JobSchedule.STATUS_EXECUTANDO: + return _( + f"{self.job.job_name}: em execução desde " + f"{localize(timezone.localtime(self.iniciado))}" + ) + return _( + f"{self.job.job_name}: executado em " + f"{localize(timezone.localtime(self.iniciado))}, " + f"levando {self.tempo_gasto} para concluir" + ) + + def run_job(self): + """Executa o job agendado. Esta rotina não verifica se a agenda está + na hora certa, apenas executa o job associado.""" + + if self.status != JobSchedule.STATUS_AGENDADO: + raise JobSchedule.DoesNotExecute() + + self.iniciado = timezone.localtime() + self.status = JobSchedule.STATUS_EXECUTANDO + self.save() + self.resultado = self.job.run() + self.status = JobSchedule.STATUS_CONCLUIDO + self.tempo_gasto = timezone.localtime() - self.iniciado + self.save() diff --git a/sigi/apps/utils/templates/admin/utils/jobschedule/change_form.html b/sigi/apps/utils/templates/admin/utils/jobschedule/change_form.html new file mode 100644 index 0000000..cb6c0d7 --- /dev/null +++ b/sigi/apps/utils/templates/admin/utils/jobschedule/change_form.html @@ -0,0 +1,33 @@ +{% extends "admin/change_form.html" %} +{% load static i18n %} + +{% block extrastyle %} +{{ block.super }} + +{% endblock %} + +{% block after_field_sets %} +
+
+
+
+
+ 
{{ original.resultado }}
+
+
+
+
+{% endblock %} \ No newline at end of file diff --git a/sigi/apps/utils/views.py b/sigi/apps/utils/views.py index dc279b9..0e4d6cb 100644 --- a/sigi/apps/utils/views.py +++ b/sigi/apps/utils/views.py @@ -1,9 +1,14 @@ +import docutils.core +import io +from contextlib import redirect_stdout, redirect_stderr from parsel import Selector from django.contrib.auth.decorators import user_passes_test from django.contrib.auth.decorators import login_required -from django.shortcuts import render +from django.shortcuts import render, get_object_or_404 +from django.template.loader import render_to_string from django.utils import timezone from django_extensions.management.jobs import get_job +from sigi.apps.utils.models import JobSchedule @login_required @@ -11,9 +16,46 @@ from django_extensions.management.jobs import get_job def user_run_job(request, job_name): job = get_job(None, job_name)() start_time = timezone.localtime() - job.do_job() - end_time = timezone.localtime() - rst, html = job.prepare_report(start_time, end_time) + if hasattr(job, "do_job"): + job.do_job() + end_time = timezone.localtime() + rst, html = job.prepare_report(start_time, end_time) + else: + with io.StringIO() as so_buf, io.StringIO() as se_buf, redirect_stdout( + so_buf + ), redirect_stderr(se_buf): + job.execute() + messages = so_buf.getvalue() + errors = se_buf.getvalue() + report_data = ["", "MENSAGENS", "---------", ""] + if messages: + report_data.extend(messages.splitlines()) + else: + report_data.extend(["Nenhuma mensagem gerada", ""]) + report_data.extend(["", "ERROS", "-----", ""]) + if errors: + report_data.extend(errors.splitlines()) + else: + report_data.extend(["Nenhum erro gerado", ""]) + end_time = timezone.localtime() + rst = render_to_string( + "emails/base_report.rst", + { + "title": job.help, + "start_time": start_time, + "end_time": end_time, + "report_data": report_data, + }, + ) + html = docutils.core.publish_string( + rst, + writer_name="html5", + settings_overrides={ + "input_encoding": "unicode", + "output_encoding": "unicode", + }, + ) + dp = Selector(text=html) return render( diff --git a/sigi/menu_conf.yaml b/sigi/menu_conf.yaml index 19b31d1..4238cf4 100644 --- a/sigi/menu_conf.yaml +++ b/sigi/menu_conf.yaml @@ -18,6 +18,11 @@ admin_menu: - title: Encerra inscrições de eventos view_name: utils_runjob view_param: encerra_inscricao + - title: Jobs de cron + view_name: admin:utils_cronjob_changelist + - title: Jobs agendados + view_name: admin_utils_jobschedule_changelist + querystr: status__exact=A main_menu: - title: Municípios