From 2b45e837ca2a4122f77ad90437d264d6cc8cd47e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ses=C3=B3stris=20Vieira?= Date: Wed, 29 Mar 2023 16:28:30 -0300 Subject: [PATCH] =?UTF-8?q?Importa=C3=A7=C3=A3o=20do=20Gescon=20a=20cada?= =?UTF-8?q?=20hora?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker/Dockerfile | 1 - .../jobs/{ => hourly}/importa_gescon.py | 5 +++-- .../migrations/0034_gescon_checksums.py | 17 +++++++++++++++++ sigi/apps/convenios/models.py | 18 ++++++++++++++++++ 4 files changed, 38 insertions(+), 3 deletions(-) rename sigi/apps/convenios/jobs/{ => hourly}/importa_gescon.py (70%) create mode 100644 sigi/apps/convenios/migrations/0034_gescon_checksums.py diff --git a/docker/Dockerfile b/docker/Dockerfile index a7bc6a1..fd53b72 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -93,7 +93,6 @@ RUN mkdir -p /var/log/sigi # schedule cron jobs RUN crontab -l | { cat; echo "* * * * * /usr/local/bin/python ${HOME}/manage.py runjobs minutely >> /var/log/sigi/cron.log 2>&1"; } | crontab - RUN crontab -l | { cat; echo "0 * * * * /usr/local/bin/python ${HOME}/manage.py runjobs hourly >> /var/log/sigi/cron.log 2>&1"; } | crontab - -RUN crontab -l | { cat; echo "0 */4 * * * /usr/local/bin/python ${HOME}/manage.py runjob importa_gescon >> /var/log/sigi/cron.log 2>&1"; } | crontab - RUN crontab -l | { cat; echo "0 0 * * * /usr/local/bin/python ${HOME}/manage.py runjobs daily >> /var/log/sigi/cron.log 2>&1"; } | crontab - RUN crontab -l | { cat; echo "0 0 * * 0 /usr/local/bin/python ${HOME}/manage.py runjobs weekly >> /var/log/sigi/cron.log 2>&1"; } | crontab - RUN crontab -l | { cat; echo "0 0 1 * * /usr/local/bin/python ${HOME}/manage.py runjobs monthly >> /var/log/sigi/cron.log 2>&1"; } | crontab - diff --git a/sigi/apps/convenios/jobs/importa_gescon.py b/sigi/apps/convenios/jobs/hourly/importa_gescon.py similarity index 70% rename from sigi/apps/convenios/jobs/importa_gescon.py rename to sigi/apps/convenios/jobs/hourly/importa_gescon.py index 2b56f43..9cc08d0 100644 --- a/sigi/apps/convenios/jobs/importa_gescon.py +++ b/sigi/apps/convenios/jobs/hourly/importa_gescon.py @@ -2,11 +2,12 @@ import datetime import docutils.core from django.core.mail import mail_admins from django.utils.translation import gettext as _ +from django_extensions.management.jobs import HourlyJob from sigi.apps.convenios.models import Gescon -from sigi.apps.utils.management.jobs import JobReportMixin, QuarterDailyJob +from sigi.apps.utils.management.jobs import JobReportMixin -class Job(JobReportMixin, QuarterDailyJob): +class Job(JobReportMixin, HourlyJob): help = "Carga de dados do Gescon." def do_job(self): diff --git a/sigi/apps/convenios/migrations/0034_gescon_checksums.py b/sigi/apps/convenios/migrations/0034_gescon_checksums.py new file mode 100644 index 0000000..456ac00 --- /dev/null +++ b/sigi/apps/convenios/migrations/0034_gescon_checksums.py @@ -0,0 +1,17 @@ +# Generated by Django 4.1.7 on 2023-03-29 18:58 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + dependencies = [ + ("convenios", "0033_alter_projeto_modelo_minuta_and_more"), + ] + + operations = [ + migrations.AddField( + model_name="gescon", + name="checksums", + field=models.JSONField(null=True), + ), + ] diff --git a/sigi/apps/convenios/models.py b/sigi/apps/convenios/models.py index e875564..ec64325 100644 --- a/sigi/apps/convenios/models.py +++ b/sigi/apps/convenios/models.py @@ -1,5 +1,6 @@ import re import requests +from hashlib import md5 from pathlib import Path from django.db import models from django.db.models import Q, fields @@ -561,6 +562,7 @@ class Gescon(models.Model): ultima_importacao = models.TextField( _("Resultado da última importação"), blank=True ) + checksums = models.JSONField(null=True) class Meta: verbose_name = _("Configuração do Gescon") @@ -599,6 +601,8 @@ class Gescon(models.Model): def importa_contratos(self): self.ultima_importacao = "" + if self.checksums is None: + self.checksums = {} self.add_message( _( f"Importação iniciada em {timezone.localtime():%d/%m/%Y %H:%M:%S}\n" @@ -642,6 +646,8 @@ class Gescon(models.Model): if re.sub("[^\d]", "", o.cnpj) != "" } + requests.packages.urllib3.disable_warnings() + for sigla_gescon, sigla_sigi in subespecies: self.add_message(_(f"\n**Importando subespécie {sigla_gescon}**")) url = self.url_gescon.format(s=sigla_gescon) @@ -669,6 +675,17 @@ class Gescon(models.Model): ) continue + md5sum = md5(response.text.encode(response.encoding)).hexdigest() + if ( + sigla_gescon in self.checksums + and self.checksums[sigla_gescon] == md5sum + ): + self.add_message( + f"Dados da subespécie {sigla_gescon} inalterados no Gescon." + " Processamento desnecessário." + ) + continue + contratos = response.json() # Pegar só os contratos que possuem alguma das palavras-chave @@ -914,6 +931,7 @@ class Gescon(models.Model): f"{erros} reportados com erro." ) ) + self.checksums[sigla_gescon] = md5sum self.save()