Impl Mixin para gerar arquivos de pesq em diversos formatos (#3710)

Impl Mixin para gerar relatórios de pesquisas

MultiFormatOutputMixin sobrescreve render_to_response e, com base no
parametro format=[xlsx, csv, json], gerar versão não paginada dos
resultados de FilterView. O mixin foi aplicado na pesquisa de matérias e
de normas e pode ser extendido para outras filterview's ou mesmo
listviews, internas ao Crud.

requirements/requirements.txt

@@ -36,6 +36,7 @@ kazoo==2.8.0
 django-prometheus==2.2.0
 
 asn1crypto==1.5.1
+XlsxWriter==3.2.0
 
 git+https://github.com/interlegis/trml2pdf
 git+https://github.com/interlegis/django-admin-bootstrapped

sapl/materia/views.py

@@ -24,6 +24,7 @@ from django.shortcuts import render
 from django.template import loader
 from django.urls import reverse
 from django.utils import formats, timezone
+from django.utils.encoding import force_text
 from django.utils.translation import ugettext_lazy as _
 from django.views.generic import CreateView, ListView, TemplateView, UpdateView
 from django.views.generic.base import RedirectView
@@ -53,7 +54,7 @@ from sapl.utils import (autor_label, autor_modal, gerar_hash_arquivo, get_base_u
                         get_client_ip, get_mime_type_from_file_extension, lista_anexados,
                         mail_service_configured, montar_row_autor, SEPARADOR_HASH_PROPOSICAO,
                         show_results_filter_set, get_tempfile_dir,
-                        google_recaptcha_configured)
+                        google_recaptcha_configured, MultiFormatOutputMixin)
 
 from .forms import (AcessorioEmLoteFilterSet, AcompanhamentoMateriaForm,
                     AnexadaEmLoteFilterSet, AdicionarVariasAutoriasFilterSet,
@@ -2039,11 +2040,27 @@ class AcompanhamentoExcluirView(TemplateView):
         return HttpResponseRedirect(self.get_success_url())
 
 
-class MateriaLegislativaPesquisaView(FilterView):
+class MateriaLegislativaPesquisaView(MultiFormatOutputMixin, FilterView):
     model = MateriaLegislativa
     filterset_class = MateriaLegislativaFilterSet
     paginate_by = 50
 
+    fields_base_report = [
+        'id', 'ano', 'numero', 'tipo__sigla', 'tipo__descricao', 'autoria__autor__nome', 'texto_original', 'ementa'
+    ]
+    fields_report = {
+        'csv': fields_base_report,
+        'xlsx': fields_base_report,
+        'json': fields_base_report,
+    }
+
+    def hook_texto_original(self, obj):
+        url = self.request.build_absolute_uri('/')[:-1]
+        texto_original = obj.texto_original if not isinstance(
+            obj, dict) else obj["texto_original"]
+
+        return f'{url}/{texto_original}'
+
     def get_filterset_kwargs(self, filterset_class):
         super().get_filterset_kwargs(filterset_class)
 
@@ -2099,7 +2116,8 @@ class MateriaLegislativaPesquisaView(FilterView):
             qs = qs.filter(materiaassunto__isnull=True)
 
         if 'o' in self.request.GET and not self.request.GET['o']:
-            args = ['-ano', 'tipo__sequencia_regimental', '-numero'] if BaseAppConfig.attr('ordenacao_pesquisa_materia') == 'R' else ['-ano', 'tipo__sigla', '-numero']
+            args = ['-ano', 'tipo__sequencia_regimental', '-numero'] if BaseAppConfig.attr(
+                'ordenacao_pesquisa_materia') == 'R' else ['-ano', 'tipo__sigla', '-numero']
 
             qs = qs.order_by(*args)
 

sapl/norma/views.py

@@ -28,7 +28,7 @@ from sapl.crud.base import (RP_DETAIL, RP_LIST, Crud, CrudAux,
                             MasterDetailCrud, make_pagination)
 from sapl.materia.models import Orgao
 from sapl.utils import show_results_filter_set, get_client_ip,\
-    sapn_is_enabled
+    sapn_is_enabled, MultiFormatOutputMixin
 
 from .forms import (AnexoNormaJuridicaForm, NormaFilterSet, NormaJuridicaForm,
                     NormaPesquisaSimplesForm, NormaRelacionadaForm,
@@ -147,11 +147,27 @@ class NormaRelacionadaCrud(MasterDetailCrud):
         layout_key = 'NormaRelacionadaDetail'
 
 
-class NormaPesquisaView(FilterView):
+class NormaPesquisaView(MultiFormatOutputMixin, FilterView):
     model = NormaJuridica
     filterset_class = NormaFilterSet
     paginate_by = 50
 
+    fields_base_report = [
+        'id', 'ano', 'numero', 'tipo__sigla', 'tipo__descricao', 'texto_integral', 'ementa'
+    ]
+    fields_report = {
+        'csv': fields_base_report,
+        'xlsx': fields_base_report,
+        'json': fields_base_report,
+    }
+
+    def hook_texto_integral(self, obj):
+        url = self.request.build_absolute_uri('/')[:-1]
+        texto_integral = obj.texto_integral if not isinstance(
+            obj, dict) else obj["texto_integral"]
+
+        return f'{url}/{texto_integral}'
+
     def get_queryset(self):
         qs = super().get_queryset()
 

sapl/templates/crud/format_options.html

@@ -0,0 +1,10 @@
+{% load i18n %}
+
+
+<div class=" d-flex justify-content-center">
+  <div class="actions btn-group" role="group">
+    <a class="btn btn-outline-secondary" href="{% url url_reverse %}?format=csv{{ filter_url }}" title="Download do resultado da pesquisa em CSV."><i class="fas fa-file-csv"></i></a>
+    <a class="btn btn-outline-secondary" href="{% url url_reverse %}?format=xlsx{{ filter_url }}" title="Download do resultado da pesquisa em XLSX."><i class="fas fa-file-excel"></i></a>
+    <a class="btn btn-outline-secondary" href="{% url url_reverse %}?format=json{{ filter_url }}" title="Download do resultado da pesquisa em JSON."><i class="far fa-file-code"></i></a>
+  </div>
+</div>

sapl/templates/materia/materialegislativa_filter.html

@@ -7,7 +7,15 @@
 
   {% block actions %}
 
-    <div class="actions btn-group" role="group">
+  {% if show_results %}
+    <div class="float-left">
+      {% with 'sapl.materia:pesquisar_materia' as url_reverse %}
+        {% include "crud/format_options.html" %}
+      {% endwith %}
+    </div>
+  {% endif %}
+
+    <div class="actions btn-group float-right pb-4" role="group">
       {% switch "SOLR_SWITCH" %}
       <a href="{% url 'sapl.base:haystack_search' %}" class="btn btn-outline-primary">
           Pesquisa Textual

sapl/templates/norma/normajuridica_filter.html

@@ -4,6 +4,15 @@
 {% load crispy_forms_tags common_tags %}
 
 {% block actions %}
+
+  {% if show_results %}
+    <div class="float-left">
+      {% with 'sapl.norma:norma_pesquisa' as url_reverse %}
+        {% include "crud/format_options.html" %}
+      {% endwith %}
+    </div>
+  {% endif %}
+
   <div class="actions btn-group float-right" role="group">
     {% switch "SOLR_SWITCH" %}
         <a href="{% url 'sapl.base:haystack_search' %}" class="btn btn-outline-primary">
@@ -19,7 +28,6 @@
       <a href="{% url 'sapl.norma:norma_pesquisa' %}" class="btn btn-outline-primary">{% trans 'Fazer nova pesquisa' %}</a>
   {% endif %}
   </div>
-  <br /><br />
 {% endblock %}
 
 {% block detail_content %}

sapl/utils.py

@@ -1,5 +1,7 @@
+import csv
 from functools import wraps
 import hashlib
+import io
 from itertools import groupby, chain
 import logging
 from operator import itemgetter
@@ -30,6 +32,7 @@ from django.db.models import Q
 from django.db.models.fields.related import ForeignKey
 from django.forms import BaseForm
 from django.forms.widgets import SplitDateTimeWidget
+from django.http.response import JsonResponse, HttpResponse
 from django.utils import six, timezone
 from django.utils.safestring import mark_safe
 from django.utils.translation import ugettext_lazy as _
@@ -39,6 +42,7 @@ from floppyforms import ClearableFileInput
 import magic
 import requests
 from unipath.path import Path
+from xlsxwriter.workbook import Workbook
 
 from sapl.crispy_layout_mixin import (form_actions, SaplFormHelper,
                                       SaplFormLayout, to_row)
@@ -1242,7 +1246,7 @@ def get_report_urls_map():
         dst_url = reverse(f"{NAMESPACE}{url.name}")
         url_map[dst_url] = {"name": url.name,
                             "public": True,
-                            "internal": True}  #TODO: get permissions from AppConfig and fine grained permissions
+                            "internal": True}  # TODO: get permissions from AppConfig and fine grained permissions
     return url_map
 
 
@@ -1290,3 +1294,217 @@ def get_path_to_name_report_map():
             '/sistema/relatorios/documentos_acessorios': 'Documentos Acessórios de Matérias Legislativas',
             '/sistema/relatorios/normas-por-autor': 'Normas Por Autor'
             }
+
+
+class MultiFormatOutputMixin:
+
+    formats_impl = 'csv', 'xlsx', 'json'
+
+    queryset_values_for_formats = True
+
+    def render_to_response(self, context, **response_kwargs):
+
+        format_result = getattr(self.request, self.request.method).get(
+            'format', None)
+
+        if format_result:
+            if format_result not in self.formats_impl:
+                raise ValidationError(
+                    'Formato Inválido e/ou não implementado!')
+
+            object_list = context['object_list']
+            object_list.query.low_mark = 0
+            object_list.query.high_mark = 0
+
+            return getattr(self, f'render_to_{format_result}')(context)
+
+        return super().render_to_response(context, **response_kwargs)
+
+    def render_to_json(self, context):
+
+        object_list = context['object_list']
+
+        if self.queryset_values_for_formats:
+            object_list = object_list.values(
+                *self.fields_report['json'])
+
+        data = []
+        for obj in object_list:
+            wr = list(self._write_row(obj, 'json'))
+
+            if not data:
+                data.append([wr])
+                continue
+
+            if wr[0] != data[-1][0][0]:
+                data.append([wr])
+            else:
+                data[-1].append(wr)
+
+        for mri, multirows in enumerate(data):
+            if len(multirows) == 1:
+                v = multirows[0]
+            else:
+                v = multirows[0]
+                for ri, cols in enumerate(multirows[1:]):
+                    for rc, cell in enumerate(cols):
+                        if v[rc] != cell:
+                            v[rc] = f'{v[rc]}\r\n{cell}'
+
+            data[mri] = dict(
+                map(lambda i, j: (i, j), self.fields_report['json'], v))
+
+        json_metadata = {
+            'headers': dict(
+                map(lambda i, j: (i, j), self.fields_report['json'], self._headers('json'))),
+            'results': data
+        }
+        response = JsonResponse(json_metadata)
+        response['Content-Disposition'] = f'attachment; filename="sapl_{self.request.resolver_match.url_name}.json"'
+        response['Cache-Control'] = 'no-cache'
+        response['Pragma'] = 'no-cache'
+        response['Expires'] = 0
+
+        return response
+
+    def render_to_csv(self, context):
+        response = HttpResponse(content_type='text/csv')
+        response['Content-Disposition'] = f'attachment; filename="sapl_{self.request.resolver_match.url_name}.csv"'
+        response['Cache-Control'] = 'no-cache'
+        response['Pragma'] = 'no-cache'
+        response['Expires'] = 0
+        writer = csv.writer(response, delimiter=";",
+                            quoting=csv.QUOTE_NONNUMERIC)
+
+        object_list = context['object_list']
+
+        if self.queryset_values_for_formats:
+            object_list = object_list.values(
+                *self.fields_report['csv'])
+
+        data = [[list(self._headers('csv'))], ]
+        for obj in object_list:
+            wr = list(self._write_row(obj, 'csv'))
+            if wr[0] != data[-1][0][0]:
+                data.append([wr])
+            else:
+                data[-1].append(wr)
+
+        for mri, multirows in enumerate(data):
+            if len(multirows) == 1:
+                writer.writerow(multirows[0])
+            else:
+                v = multirows[0]
+                for ri, cols in enumerate(multirows[1:]):
+                    for rc, cell in enumerate(cols):
+                        if v[rc] != cell:
+                            v[rc] = f'{v[rc]}\r\n{cell}'
+
+                writer.writerow(v)
+
+        return response
+
+    def render_to_xlsx(self, context):
+
+        object_list = context['object_list']
+
+        if self.queryset_values_for_formats:
+            object_list = object_list.values(
+                *self.fields_report['xlsx'])
+
+        data = [[list(self._headers('xlsx'))], ]
+        for obj in object_list:
+            wr = list(self._write_row(obj, 'xlsx'))
+            if wr[0] != data[-1][0][0]:
+                data.append([wr])
+            else:
+                data[-1].append(wr)
+
+        output = io.BytesIO()
+        wb = Workbook(output, {'in_memory': True})
+
+        ws = wb.add_worksheet()
+
+        for mri, multirows in enumerate(data):
+            if len(multirows) == 1:
+                for rc, cell in enumerate(multirows[0]):
+                    ws.write(mri, rc, cell)
+            else:
+                v = multirows[0]
+                for ri, cols in enumerate(multirows[1:]):
+                    for rc, cell in enumerate(cols):
+                        if v[rc] != cell:
+                            v[rc] = f'{v[rc]}\r\n{cell}'
+
+                for rc, cell in enumerate(v):
+                    ws.write(mri, rc, cell)
+        ws.autofit()
+        wb.close()
+
+        output.seek(0)
+
+        response = HttpResponse(output.read(
+        ), content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet")
+        response['Content-Disposition'] = f'attachment; filename="sapl_{self.request.resolver_match.url_name}.xlsx"'
+        response['Cache-Control'] = 'no-cache'
+        response['Pragma'] = 'no-cache'
+        response['Expires'] = 0
+
+        output.close()
+
+        return response
+
+    def _write_row(self, obj, format_result):
+
+        for fname in self.fields_report[format_result]:
+
+            if hasattr(self, f'hook_{fname}'):
+                v = getattr(self, f'hook_{fname}')(obj)
+                yield v
+                continue
+
+            if isinstance(obj, dict):
+                yield obj[fname]
+                continue
+
+            fname = fname.split('__')
+
+            v = obj
+            for fp in fname:
+                v = getattr(v, fp)
+
+            if hasattr(v, 'all'):
+                v = ' - '.join(map(lambda x: str(x), v.all()))
+
+            yield v
+
+    def _headers(self, format_result):
+
+        for fname in self.fields_report[format_result]:
+
+            verbose_name = []
+
+            if hasattr(self, f'hook_header_{fname}'):
+                h = getattr(self, f'hook_header_{fname}')()
+                yield h
+                continue
+
+            fname = fname.split('__')
+
+            m = self.model
+            for fp in fname:
+
+                f = m._meta.get_field(fp)
+
+                vn = str(f.verbose_name) if hasattr(f, 'verbose_name') else fp
+                if f.is_relation:
+                    m = f.related_model
+                    if m == self.model:
+                        m = f.field.model
+
+                    if vn == fp:
+                        vn = str(m._meta.verbose_name_plural)
+                verbose_name.append(vn.strip())
+
+            verbose_name = '/'.join(verbose_name).strip()
+            yield f'{verbose_name}'