Aprimoramentos nos cronjobs

docker/Dockerfile

@@ -93,12 +93,7 @@ RUN ln -s ${HOME}/etc/nginx/sites-available/sigi.vhost /etc/nginx/sites-enabled/
 RUN mkdir -p /var/log/sigi
 
 # schedule cron jobs
-RUN crontab -l | { cat; echo "* * * * * /usr/local/bin/python ${HOME}/manage.py runjobs minutely >> /var/log/sigi/cron.log 2>&1"; } | crontab -
+RUN crontab -l | { cat; echo "* * * * * /usr/local/bin/python ${HOME}/manage.py runjob job_controller >> /var/log/sigi/cron.log 2>&1"; } | crontab -
-RUN crontab -l | { cat; echo "0 * * * * /usr/local/bin/python ${HOME}/manage.py runjobs hourly >> /var/log/sigi/cron.log 2>&1"; } | crontab -
-RUN crontab -l | { cat; echo "0 0 * * * /usr/local/bin/python ${HOME}/manage.py runjobs daily >> /var/log/sigi/cron.log 2>&1"; } | crontab -
-RUN crontab -l | { cat; echo "0 0 * * 0 /usr/local/bin/python ${HOME}/manage.py runjobs weekly >> /var/log/sigi/cron.log 2>&1"; } | crontab -
-RUN crontab -l | { cat; echo "0 0 1 * * /usr/local/bin/python ${HOME}/manage.py runjobs monthly >> /var/log/sigi/cron.log 2>&1"; } | crontab -
-RUN crontab -l | { cat; echo "0 4 * * * /usr/local/bin/python ${HOME}/manage.py runjob sincroniza_saberes >> /var/log/sigi/cron.log 2>&1"; } | crontab -
 EXPOSE 80/tcp 443/tcp
 
 ENV DEBIAN_FRONTEND=teletype

requirements/requirements.txt

@@ -1,3 +1,4 @@
+cron-converter==1.0.2
 dnspython==2.3.0
 docutils==0.20.1
 gunicorn==20.1.0

sigi/apps/utils/admin.py

@@ -1,7 +1,44 @@
-from django.contrib import admin
+from django.contrib import admin, messages
-from sigi.apps.utils.models import SigiAlert
+from django.core.exceptions import PermissionDenied
+from django.http import HttpRequest
+from django.shortcuts import get_object_or_404, redirect
+from django.urls import reverse, path
+from django.utils import timezone
+from django.utils.formats import localize
+from django.utils.safestring import mark_safe
+from django.utils.translation import gettext as _
+from django_extensions.management.jobs import get_job, get_jobs
 from tinymce.models import HTMLField
 from tinymce.widgets import AdminTinyMCE
+from sigi.apps.utils.models import SigiAlert, Cronjob, JobSchedule
+
+
+class JobScheduleInline(admin.TabularInline):
+    model = JobSchedule
+    fields = ["status", "iniciar", "iniciado", "tempo_gasto", "get_runner"]
+    readonly_fields = [
+        "status",
+        "iniciar",
+        "iniciado",
+        "tempo_gasto",
+        "get_runner",
+    ]
+    can_delete = False
+    can_add = False
+    extra = 0
+
+    def has_add_permission(self, request, obj):
+        return False
+
+    @mark_safe
+    @admin.display(description=_("executar"))
+    def get_runner(self, sched):
+        if sched.status == JobSchedule.STATUS_AGENDADO:
+            url = reverse("admin:utils_jobschedule_runjob", args=[sched.id])
+            return (
+                f"<a href='{url}'><i class='material-icons'>play_arrow</i></a>"
+            )
+        return ""
 
 
 @admin.register(SigiAlert)
@@ -10,3 +47,161 @@ class SigiAlertAdmin(admin.ModelAdmin):
     search_fields = ("titulo", "caminho")
     formfield_overrides = {HTMLField: {"widget": AdminTinyMCE}}
     list_filter = ("destinatarios",)
+
+
+@admin.register(Cronjob)
+class CronjobAdmin(admin.ModelAdmin):
+    list_display = (
+        "job_name",
+        "app_name",
+        "get_help",
+        "expressao_cron",
+        "get_schedule",
+        "get_runner",
+    )
+    fields = ["job_name", "app_name", "get_help", "expressao_cron"]
+    readonly_fields = ("job_name", "app_name", "get_help")
+    inlines = [JobScheduleInline]
+
+    def get_urls(self):
+        urls = super().get_urls()
+        model_info = (self.model._meta.app_label, self.model._meta.model_name)
+
+        my_urls = [
+            path(
+                "<path:object_id>/runjob/",
+                self.admin_site.admin_view(self.run_job),
+                name="%s_%s_runjob" % model_info,
+            ),
+        ]
+        return my_urls + urls
+
+    @admin.display(description=_("descrição"))
+    def get_help(self, job):
+        try:
+            JobClass = get_job(job.app_name, job.job_name)
+        except KeyError:
+            return _(
+                f"A rotina de JOB {job.app_name}.{job.job_name} "
+                "não foi encontrada."
+            )
+        job_obj = JobClass()
+        return job_obj.help
+
+    @admin.display(description=_("agenda"))
+    def get_schedule(self, job):
+        sched = job.jobschedule_set.first()
+        if sched is None:
+            return _("Nenhum agendamento para este job")
+        if sched.status == JobSchedule.STATUS_AGENDADO:
+            return _(
+                "início agendado para "
+                f"{localize(timezone.localtime(sched.iniciar))}."
+            )
+        if sched.status == JobSchedule.STATUS_EXECUTANDO:
+            return _(
+                "em execução desde "
+                f"{localize(timezone.localtime(sched.iniciado))}"
+            )
+        return _(
+            f"executado em {localize(timezone.localtime(sched.iniciado))}, "
+            f"levando {sched.tempo_gasto} minutos para concluir"
+        )
+
+    @mark_safe
+    @admin.display(description=_("executar"))
+    def get_runner(self, job):
+        url = reverse("admin:utils_cronjob_runjob", args=[job.id])
+        return (
+            f"<a href='{url}'>" "<i class='material-icons'>play_arrow</i></a>"
+        )
+
+    def run_job(self, request, object_id):
+        cronjob = get_object_or_404(Cronjob, id=object_id)
+        sched = cronjob.next_schedule()
+        if sched.status != JobSchedule.STATUS_AGENDADO:
+            raise PermissionDenied(
+                _(
+                    "Este agendamento não pode ser executado pois "
+                    f"está com status {sched.get_status_display()}"
+                )
+            )
+        sched.run_job()
+        self.message_user(
+            request,
+            _("JOB executado!"),
+            messages.SUCCESS,
+        )
+        return redirect("admin:utils_jobschedule_change", object_id=sched.id)
+
+
+@admin.register(JobSchedule)
+class JobScheduleAdmin(admin.ModelAdmin):
+    list_display = [
+        "job",
+        "status",
+        "iniciar",
+        "iniciado",
+        "tempo_gasto",
+        "get_runner",
+    ]
+    fields = [
+        "job",
+        "status",
+        "iniciar",
+        "iniciado",
+        "tempo_gasto",
+    ]
+    readonly_fields = fields
+    list_filter = ("status", "job")
+    date_hierarchy = "iniciar"
+
+    def get_urls(self):
+        urls = super().get_urls()
+        model_info = (self.model._meta.app_label, self.model._meta.model_name)
+
+        my_urls = [
+            path(
+                "<path:object_id>/runjob/",
+                self.admin_site.admin_view(self.run_job),
+                name="%s_%s_runjob" % model_info,
+            ),
+        ]
+        return my_urls + urls
+
+    def has_add_permission(self, request):
+        return False
+
+    def has_delete_permission(self, request, obj=None):
+        return False
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+    @mark_safe
+    @admin.display(description=_("executar"))
+    def get_runner(self, sched):
+        if sched.status == JobSchedule.STATUS_AGENDADO:
+            url = reverse("admin:utils_jobschedule_runjob", args=[sched.id])
+            return (
+                f"<a href='{url}'>"
+                "<i class='material-icons'>play_arrow</i></a>"
+            )
+        return ""
+
+    def run_job(self, request, object_id):
+        sched = get_object_or_404(JobSchedule, id=object_id)
+        if sched.status != JobSchedule.STATUS_AGENDADO:
+            raise PermissionDenied(
+                _(
+                    "Este agendamento não pode ser executado pois "
+                    f"está com status {sched.get_status_display()}"
+                )
+            )
+        sched.run_job()
+        self.message_user(
+            request,
+            _("JOB executado!"),
+            messages.SUCCESS,
+        )
+        return redirect("admin:utils_jobschedule_change", object_id=object_id)

sigi/apps/utils/jobs/job_controller.py

@@ -0,0 +1,90 @@
+from django_extensions.management.jobs import BaseJob
+from django_extensions.management.jobs import get_jobs
+from django.utils import timezone
+from django.utils.formats import localize
+from django.utils.translation import gettext as _
+from sigi.apps.utils.models import Cronjob, JobSchedule
+
+WHEN_SETS = {
+    "daily": "0 0 * * *",
+    "hourly": "0 * * * *",
+    "monthly": "0 0 1 * *",
+    "weekly": "0 0 * * 0",
+    "yearly": "0 0 1 1 *",
+    "minutely": "* * * * *",
+}
+
+
+class Job(BaseJob):
+    help = "Controlador de cronjobs do SIGI."
+
+    def execute(self):
+        print("Rodando controlador de jobs...")
+        self.remove_old_jobs()
+        self.sync_new_jobs()
+        self.run_scheduled()
+        self.schedule_jobs()
+
+    def remove_old_jobs(self):
+        """Remover das tabelas os jobs que foram removidos do código"""
+        print("\tRemover das tabelas os jobs que foram removidos do código...")
+        all_jobs = get_jobs()
+        excludes = Cronjob.objects.all()
+        for app_name, job_name in all_jobs.keys():
+            excludes = excludes.exclude(app_name=app_name, job_name=job_name)
+        print("\t\t", excludes.delete())
+
+    def sync_new_jobs(self):
+        """
+        Atualizar a tabela de JOBS com os novos JOBS que tenham sido criados
+        """
+        print(
+            "\tAtualizar a tabela de JOBS com os novos JOBS que tenham "
+            "sido criados..."
+        )
+        all_jobs = get_jobs()
+        for (app_name, job_name), JobClass in all_jobs.items():
+            if app_name == "sigi.apps.utils" and job_name == "job_controller":
+                # Ignorar job_controller
+                continue
+            try:
+                job = Cronjob.objects.get(app_name=app_name, job_name=job_name)
+            except Cronjob.DoesNotExist:
+                # Inserir o JOB na tabela de JOBS #
+                job_obj = JobClass()
+                if job_obj.when in WHEN_SETS:
+                    expressao_cron = WHEN_SETS[job_obj.when]
+                else:
+                    expressao_cron = WHEN_SETS["daily"]  # Default
+                job = Cronjob(
+                    app_name=app_name,
+                    job_name=job_name,
+                    expressao_cron=expressao_cron,
+                )
+                job.save()
+                print(f"\t\tNovo job encontrado: {job_name}: {job_obj.help}")
+
+    def run_scheduled(self):
+        """Executa os jobs que estão agendados"""
+        print("\tExecutar os jobs que estão agendados...")
+        for sched in JobSchedule.objects.filter(
+            status=JobSchedule.STATUS_AGENDADO
+        ):
+            agora = timezone.localtime()
+            if sched.iniciar <= agora:
+                sched.run_job()
+
+    def schedule_jobs(self):
+        """Criar agenda para próxima execução"""
+        print("\tCriar agenda para próxima execução...")
+        for job in Cronjob.objects.exclude(
+            jobschedule__status__in=[
+                JobSchedule.STATUS_AGENDADO,
+                JobSchedule.STATUS_EXECUTANDO,
+            ]
+        ):
+            sched = job.next_schedule()
+            print(
+                f"\t\tAgendado job {sched.job.job_name} "
+                f"para {localize(sched.iniciar)}"
+            )

sigi/apps/utils/migrations/0002_cronjob_jobschedule.py

@@ -0,0 +1,45 @@
+# Generated by Django 4.2.7 on 2024-02-26 12:53
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('utils', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Cronjob',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('app_name', models.CharField(editable=False, max_length=100, verbose_name='app')),
+                ('job_name', models.CharField(editable=False, max_length=100, verbose_name='job')),
+                ('expressao_cron', models.CharField(default='* * * * *', help_text="Usar expressoões no formato padrão de CRON: 'minute hour day month day-of-week'. Mais detalhes: <a href='https://help.ubuntu.com/community/CronHowto'>CronHowTo</a>", max_length=100, verbose_name='expressão CRON')),
+            ],
+            options={
+                'verbose_name': 'Cron job',
+                'verbose_name_plural': 'Cron jobs',
+                'ordering': ('app_name', 'job_name'),
+            },
+        ),
+        migrations.CreateModel(
+            name='JobSchedule',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('iniciar', models.DateTimeField(verbose_name='Iniciar em')),
+                ('iniciado', models.DateTimeField(blank=True, null=True, verbose_name='Iniciado em')),
+                ('status', models.CharField(choices=[('A', 'Agendado'), ('E', 'Executando'), ('C', 'Concluído')], default='A', max_length=1, verbose_name='estado')),
+                ('tempo_gasto', models.DurationField(blank=True, editable=False, null=True, verbose_name='tempo gasto')),
+                ('resultado', models.TextField(blank=True, editable=False, verbose_name='resultado da execução')),
+                ('job', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='utils.cronjob', verbose_name='Cron job')),
+            ],
+            options={
+                'verbose_name': 'Agenda de execução',
+                'verbose_name_plural': 'Agenda de execuções',
+                'ordering': ('iniciar',),
+            },
+        ),
+    ]

sigi/apps/utils/models.py

@@ -1,8 +1,15 @@
+import io
+from contextlib import redirect_stderr, redirect_stdout
+from cron_converter import Cron
 from pyexpat import model
 from django.db import models
 from django.contrib.auth.models import Group
+from django.utils import timezone
+from django.utils.formats import localize
 from django.utils.translation import gettext as _
+from django_extensions.management.jobs import get_job, get_jobs
 from tinymce.models import HTMLField
+from sigi.apps.utils.management.jobs import JobReportMixin
 
 
 class SigiAlert(models.Model):
@@ -26,3 +33,152 @@ class SigiAlert(models.Model):
 
     def __str__(self):
         return self.titulo
+
+
+class Cronjob(models.Model):
+    app_name = models.CharField(_("app"), max_length=100, editable=False)
+    job_name = models.CharField(_("job"), max_length=100, editable=False)
+    expressao_cron = models.CharField(
+        _("expressão CRON"),
+        max_length=100,
+        default="* * * * *",
+        help_text=_(
+            "Usar expressoões no formato padrão de CRON: "
+            "'minute hour day month day-of-week'. "
+            "Mais detalhes: "
+            "<a href='https://help.ubuntu.com/community/CronHowto'>"
+            "CronHowTo"
+            "</a>"
+        ),
+    )
+
+    class Meta:
+        ordering = ("app_name", "job_name")
+        verbose_name = _("Cron job")
+        verbose_name_plural = _("Cron jobs")
+
+    def __str__(self):
+        return self.job_name
+
+    def run(self):
+        try:
+            JobClass = get_job(self.app_name, self.job_name)
+        except KeyError:
+            return (
+                f"A rotina de JOB {self.job.job_name} do app "
+                f"{self.job.app_name} não foi encontrada."
+            )
+        try:
+            job_obj = JobClass()
+            with io.StringIO() as so_buf, io.StringIO() as se_buf, redirect_stdout(
+                so_buf
+            ), redirect_stderr(
+                se_buf
+            ):
+                job_obj.execute()
+                messages = so_buf.getvalue()
+                errors = se_buf.getvalue()
+            report_data = ["", "MENSAGENS", "---------", ""]
+            if messages:
+                report_data.extend(messages.splitlines())
+            else:
+                report_data.extend(["Nenhuma mensagem gerada", ""])
+            report_data.extend(["", "ERROS", "-----", ""])
+            if errors:
+                report_data.extend(errors.splitlines())
+            else:
+                report_data.extend(["Nenhum erro gerado", ""])
+            return "\n".join(report_data)
+        except Exception as e:
+            # Qualquer erro deve ser reportado
+            return _(f"JOB abortado com erro: {str(e)}")
+
+    def next_schedule(self):
+        """Recupera a agenda da próxima execução. Se não existe, cria."""
+        try:
+            sch = self.jobschedule_set.get(
+                status__in=[
+                    JobSchedule.STATUS_AGENDADO,
+                    JobSchedule.STATUS_EXECUTANDO,
+                ]
+            )
+        except JobSchedule.DoesNotExist:
+            iniciar = self.get_next_schedule_time()
+            sch = JobSchedule(job=self, iniciar=iniciar)
+            sch.save()
+        return sch
+
+    def get_next_schedule_time(self):
+        cron_instance = Cron(self.expressao_cron)
+        scheduller = cron_instance.schedule(timezone.localtime())
+        return scheduller.next()
+
+
+class JobSchedule(models.Model):
+    STATUS_AGENDADO = "A"
+    STATUS_EXECUTANDO = "E"
+    STATUS_CONCLUIDO = "C"
+    STATUS_CHOICES = (
+        (STATUS_AGENDADO, _("Agendado")),
+        (STATUS_EXECUTANDO, _("Executando")),
+        (STATUS_CONCLUIDO, _("Concluído")),
+    )
+    job = models.ForeignKey(
+        Cronjob, verbose_name=_("Cron job"), on_delete=models.CASCADE
+    )
+    iniciar = models.DateTimeField(_("Iniciar em"))
+    iniciado = models.DateTimeField(_("Iniciado em"), blank=True, null=True)
+    status = models.CharField(
+        _("estado"),
+        max_length=1,
+        choices=STATUS_CHOICES,
+        default=STATUS_AGENDADO,
+    )
+    tempo_gasto = models.DurationField(
+        _("tempo gasto"), blank=True, null=True, editable=False
+    )
+    resultado = models.TextField(
+        _("resultado da execução"), blank=True, editable=False
+    )
+
+    class Meta:
+        ordering = ("iniciar",)
+        verbose_name = _("Agenda de execução")
+        verbose_name_plural = _("Agenda de execuções")
+
+    class DoesNotExecute(Exception):
+        """This scheduled job cannot be executed because not in AGENDADO state"""
+
+        pass
+
+    def __str__(self):
+        if self.status == JobSchedule.STATUS_AGENDADO:
+            return _(
+                f"{self.job.job_name}: início agendado para "
+                f"{localize(timezone.localtime(self.iniciar))}."
+            )
+        elif self.status == JobSchedule.STATUS_EXECUTANDO:
+            return _(
+                f"{self.job.job_name}: em execução desde "
+                f"{localize(timezone.localtime(self.iniciado))}"
+            )
+        return _(
+            f"{self.job.job_name}: executado em "
+            f"{localize(timezone.localtime(self.iniciado))}, "
+            f"levando {self.tempo_gasto} para concluir"
+        )
+
+    def run_job(self):
+        """Executa o job agendado. Esta rotina não verifica se a agenda está
+        na hora certa, apenas executa o job associado."""
+
+        if self.status != JobSchedule.STATUS_AGENDADO:
+            raise JobSchedule.DoesNotExecute()
+
+        self.iniciado = timezone.localtime()
+        self.status = JobSchedule.STATUS_EXECUTANDO
+        self.save()
+        self.resultado = self.job.run()
+        self.status = JobSchedule.STATUS_CONCLUIDO
+        self.tempo_gasto = timezone.localtime() - self.iniciado
+        self.save()

sigi/apps/utils/templates/admin/utils/jobschedule/change_form.html

@@ -0,0 +1,33 @@
+{% extends "admin/change_form.html" %}
+{% load static i18n %}
+
+{% block extrastyle %}
+{{ block.super }}
+  <style type="text/css">
+    .display {
+        flex: 1;
+    }
+    h1 {
+      font-size: 2.5rem;
+    }
+    h2 {
+      font-size: 2rem;
+    }
+    h3 {
+      font-size: 1.5rem;
+    }
+  </style>
+{% endblock %}
+
+{% block after_field_sets %}
+<fieldset class="module aligned ">
+  <div class="form-row field-resultado">
+    <div class="input-field">
+      <div class="readonly-label"><label>Resultado:</label></div>
+      <div class="readonly">
+        <pre>{{ original.resultado }}</pre>
+      </div>
+    </div>
+  </div>
+</fieldset>
+{% endblock %}

sigi/apps/utils/views.py

@@ -1,9 +1,14 @@
+import docutils.core
+import io
+from contextlib import redirect_stdout, redirect_stderr
 from parsel import Selector
 from django.contrib.auth.decorators import user_passes_test
 from django.contrib.auth.decorators import login_required
-from django.shortcuts import render
+from django.shortcuts import render, get_object_or_404
+from django.template.loader import render_to_string
 from django.utils import timezone
 from django_extensions.management.jobs import get_job
+from sigi.apps.utils.models import JobSchedule
 
 
 @login_required
@@ -11,9 +16,46 @@ from django_extensions.management.jobs import get_job
 def user_run_job(request, job_name):
     job = get_job(None, job_name)()
     start_time = timezone.localtime()
-    job.do_job()
+    if hasattr(job, "do_job"):
-    end_time = timezone.localtime()
+        job.do_job()
-    rst, html = job.prepare_report(start_time, end_time)
+        end_time = timezone.localtime()
+        rst, html = job.prepare_report(start_time, end_time)
+    else:
+        with io.StringIO() as so_buf, io.StringIO() as se_buf, redirect_stdout(
+            so_buf
+        ), redirect_stderr(se_buf):
+            job.execute()
+            messages = so_buf.getvalue()
+            errors = se_buf.getvalue()
+        report_data = ["", "MENSAGENS", "---------", ""]
+        if messages:
+            report_data.extend(messages.splitlines())
+        else:
+            report_data.extend(["Nenhuma mensagem gerada", ""])
+        report_data.extend(["", "ERROS", "-----", ""])
+        if errors:
+            report_data.extend(errors.splitlines())
+        else:
+            report_data.extend(["Nenhum erro gerado", ""])
+        end_time = timezone.localtime()
+        rst = render_to_string(
+            "emails/base_report.rst",
+            {
+                "title": job.help,
+                "start_time": start_time,
+                "end_time": end_time,
+                "report_data": report_data,
+            },
+        )
+        html = docutils.core.publish_string(
+            rst,
+            writer_name="html5",
+            settings_overrides={
+                "input_encoding": "unicode",
+                "output_encoding": "unicode",
+            },
+        )
+
     dp = Selector(text=html)
 
     return render(

sigi/menu_conf.yaml

@@ -18,6 +18,11 @@ admin_menu:
       - title: Encerra inscrições de eventos
         view_name: utils_runjob
         view_param: encerra_inscricao
+      - title: Jobs de cron
+        view_name: admin:utils_cronjob_changelist
+      - title: Jobs agendados
+        view_name: admin_utils_jobschedule_changelist
+        querystr: status__exact=A
 
 main_menu:
   - title: Municípios