Implementando celery para email de tramitacao e indexacao do solr (#2647)

* implementando celery para tramitacao de email e indexacao do solr

corrigindo conflitos do cherry pick

celery email_queue

implementando solr com fila de menagem

* adicionando django-celery-results==1.0.4

* corrigindo quantidade de argumentos

* adicionando Whoosh

* dockerizando celery

* Startando celery no docker-composer junto ao sapl

docker-compose.yml

@@ -10,13 +10,26 @@ sapldb:
     - sapldb_data:/var/lib/postgresql/data/
   ports:
     - "5432:5432"
+
 saplredis:
   image: redis:5.0.3-stretch
   restart: always
   ports: 
     - "6379:6379"
+
+#saplsolr:
+#  image: solr:7.4-alpine
+#  restart: always
+#  command: bin/solr start -c -f
+#  volumes:
+#    - solr_data:/opt/solr/server/solr
+#    - solr_configsets:/opt/solr/server/solr/configsets
+#  ports:
+#    - "8983:8983"
+
 sapl:
   image: interlegis/sapl:master
+  #build: .
   restart: always
   environment:
     ADMIN_PASSWORD: interlegis
@@ -31,12 +44,17 @@ sapl:
     USE_CHANNEL_LAYERS: 'True'
     PORT_CHANNEL_LAYERS: 6379
     HOST_CHANNEL_LAYERS: saplredis
+#    USE_SOLR: 'True'
+#    SOLR_COLLECTION: sapl
+#    SOLR_URL: http://saplsolr:8983
     TZ: America/Sao_Paulo
+    CELERY_BROKER_URL: redis://saplredis:6379
   volumes:
     - sapl_data:/var/interlegis/sapl/data
     - sapl_media:/var/interlegis/sapl/media
   links:
     - sapldb
     - saplredis
+    #- saplsolr
   ports:
     - "80:80"

requirements/requirements.txt

@@ -26,6 +26,14 @@ unipath==1.1
 WeasyPrint==44
 Pillow==5.1.0
 gunicorn==19.9.0
+kombu==4.1.0
+billiard==3.5.0.2
+celery==4.1.0
+celery-haystack==0.10
+redis>=2.10.5,<2.11
+django-celery-results==1.0.4
+Whoosh==2.7.4
+
 
 pysolr==3.6.0
 

sapl/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

sapl/base/email_utils.py

@@ -11,8 +11,8 @@ from sapl.materia.models import AcompanhamentoMateria
 from sapl.protocoloadm.models import AcompanhamentoDocumento
 from sapl.settings import EMAIL_SEND_USER
 from sapl.utils import mail_service_configured
-from django.utils.translation import ugettext_lazy as _
 
+from django.utils.translation import ugettext_lazy as _
 
 def load_email_templates(templates, context={}):
 
@@ -115,8 +115,9 @@ def do_envia_email_confirmacao(base_url, casa, tipo, doc_mat, destinatario):
     # Envia email de confirmacao para atualizações de tramitação
     #
 
+    logger = logging.getLogger(__name__)
+
     if not mail_service_configured():
-        logger = logging.getLogger(__name__)
         logger.warning(_('Servidor de email não configurado.'))
         return
 
@@ -264,4 +265,5 @@ def do_envia_email_tramitacao(base_url, tipo, doc_mat, status, unidade_destino):
             raise Exception(
                 'Erro ao enviar e-mail de acompanhamento de matéria.')
 
+
     connection.close()

sapl/base/receivers.py

@@ -1,11 +1,12 @@
 from django.db.models.signals import post_delete
 from django.dispatch import receiver
 
-from sapl.base.email_utils import do_envia_email_tramitacao
 from sapl.base.signals import tramitacao_signal
 from sapl.protocoloadm.models import TramitacaoAdministrativo
 from sapl.utils import get_base_url
 
+from .tasks import task_envia_email_tramitacao
+
 
 @receiver(tramitacao_signal)
 def handle_tramitacao_signal(sender, **kwargs):
@@ -18,12 +19,11 @@ def handle_tramitacao_signal(sender, **kwargs):
         tipo = "materia"
         doc_mat = tramitacao.materia
 
-    do_envia_email_tramitacao(
+    kwargs = {'base_url': get_base_url(request), 'tipo': tipo, 'doc_mat_id': doc_mat.id,
-        get_base_url(request),
+              'tramitacao_status_id': tramitacao.status.id,
-        tipo,
+              'tramitacao_unidade_tramitacao_destino_id': tramitacao.unidade_tramitacao_destino.id}
-        doc_mat,
+
-        tramitacao.status,
+    task_envia_email_tramitacao.delay(kwargs)
-        tramitacao.unidade_tramitacao_destino)
 
 
 @receiver(post_delete)

sapl/base/search_indexes.py

@@ -8,9 +8,10 @@ from django.template import loader
 from haystack import connections
 from haystack.constants import Indexable
 from haystack.fields import CharField
-from haystack.indexes import SearchIndex
 from haystack.utils import get_model_ct_tuple
 
+from celery_haystack.indexes import CelerySearchIndex
+
 from sapl.compilacao.models import (STATUS_TA_IMMUTABLE_PUBLIC,
                                     STATUS_TA_PUBLIC, Dispositivo)
 from sapl.materia.models import DocumentoAcessorio, MateriaLegislativa
@@ -117,7 +118,7 @@ class TextExtractField(CharField):
                          'extracted': self.extract_data(obj)})
 
 
-class DocumentoAcessorioIndex(SearchIndex, Indexable):
+class DocumentoAcessorioIndex(CelerySearchIndex, Indexable):
     model = DocumentoAcessorio
     text = TextExtractField(
         document=True, use_template=True,

sapl/base/tasks.py

@@ -0,0 +1,27 @@
+from sapl.celery import app
+from sapl.base.email_utils import do_envia_email_tramitacao
+from sapl.materia.models import StatusTramitacao, UnidadeTramitacao, MateriaLegislativa
+from sapl.protocoloadm.models import StatusTramitacaoAdministrativo, DocumentoAdministrativo
+
+
+@app.task(queue='email_queue')
+def task_envia_email_tramitacao(kwargs):
+
+    tipo = kwargs.get("tipo")
+    doc_mat_id = kwargs.get("doc_mat_id")
+    tramitacao_status_id = kwargs.get("tramitacao_status_id")
+    tramitacao_unidade_tramitacao_destino_id = kwargs.get("tramitacao_unidade_tramitacao_destino_id")
+    base_url = kwargs.get("base_url")
+
+    if tipo == 'documento':
+        doc_mat = DocumentoAdministrativo.objects.get(id=doc_mat_id)
+        status = StatusTramitacaoAdministrativo.objects.get(id=tramitacao_status_id)
+
+    elif tipo == 'materia':
+        doc_mat = MateriaLegislativa.objects.get(id=doc_mat_id)
+        status = StatusTramitacao.objects.get(id=tramitacao_status_id)
+
+    unidade_destino = UnidadeTramitacao.objects.get(id=tramitacao_unidade_tramitacao_destino_id)
+
+    do_envia_email_tramitacao(base_url, tipo, doc_mat, status, unidade_destino)
+

sapl/celery.py

@@ -0,0 +1,10 @@
+import os
+from celery import Celery
+from django.conf import settings
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'sapl.settings')
+app = Celery('sapl')
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+
+app.autodiscover_tasks(lambda: settings.INSTALLED_APPS)

sapl/settings.py

@@ -94,7 +94,10 @@ INSTALLED_APPS = (
     'reversion',
     'reversion_compare',
 
+    'django_celery_results',
     'haystack',
+    'celery_haystack',
+    'whoosh',
     'speedinfo',
 
     'webpack_loader',
@@ -116,7 +119,7 @@ SOLR_URL = config('SOLR_URL', cast=str, default='http://localhost:8983')
 SOLR_COLLECTION = config('SOLR_COLLECTION', cast=str, default='sapl')
 
 if USE_SOLR:
-    HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.RealtimeSignalProcessor'  # enable auto-index
+    HAYSTACK_SIGNAL_PROCESSOR = 'celery_haystack.signals.CelerySignalProcessor'  # enable auto-index
     SEARCH_BACKEND = 'haystack.backends.solr_backend.SolrEngine'
     SEARCH_URL = ('URL', '{}/solr/{}'.format(SOLR_URL, SOLR_COLLECTION))
 
@@ -130,6 +133,11 @@ HAYSTACK_CONNECTIONS = {
     },
 }
 
+CELERY_BROKER_URL = 'redis://localhost:6379'
+
+CELERY_RESULT_BACKEND = 'django-db'
+
+
 MIDDLEWARE = [
     'reversion.middleware.RevisionMiddleware',
     'django.contrib.sessions.middleware.SessionMiddleware',

start.sh

@@ -101,6 +101,9 @@ if [ $lack_pwd -eq 0 ]; then
    # return -1
 fi
 
+rm -rf /var/interlegis/sapl/celery_log/*
+celery multi start 2 -A sapl -l info -Q:1 email_queue -c:1 1 -Q:2 celery -c:2 1 --pidfile=./celery_log/%n.pid --logfile=./celery_log/%n%I.log
+
 echo "-------------------------------------"
 echo "| ███████╗ █████╗ ██████╗ ██╗       |"
 echo "| ██╔════╝██╔══██╗██╔══██╗██║       |"