From 52e1c795218a346b1629d5e54f125e804b5575d8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?LeandroJata=C3=AD?= Date: Mon, 29 Apr 2024 08:58:00 -0300 Subject: [PATCH] Impl Mixin para gerar arquivos de pesq em diversos formatos (#3710) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Impl Mixin para gerar relatórios de pesquisas MultiFormatOutputMixin sobrescreve render_to_response e, com base no parametro format=[xlsx, csv, json], gerar versão não paginada dos resultados de FilterView. O mixin foi aplicado na pesquisa de matérias e de normas e pode ser extendido para outras filterview's ou mesmo listviews, internas ao Crud. --- requirements/requirements.txt | 1 + sapl/materia/views.py | 42 +++- sapl/norma/views.py | 22 +- sapl/templates/crud/format_options.html | 10 + .../materia/materialegislativa_filter.html | 10 +- .../templates/norma/normajuridica_filter.html | 10 +- sapl/utils.py | 224 +++++++++++++++++- 7 files changed, 299 insertions(+), 20 deletions(-) create mode 100644 sapl/templates/crud/format_options.html diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 5d2d578eb..841e48f9c 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -36,6 +36,7 @@ kazoo==2.8.0 django-prometheus==2.2.0 asn1crypto==1.5.1 +XlsxWriter==3.2.0 git+https://github.com/interlegis/trml2pdf git+https://github.com/interlegis/django-admin-bootstrapped diff --git a/sapl/materia/views.py b/sapl/materia/views.py index 5492ddfb6..d6742c00a 100644 --- a/sapl/materia/views.py +++ b/sapl/materia/views.py @@ -24,6 +24,7 @@ from django.shortcuts import render from django.template import loader from django.urls import reverse from django.utils import formats, timezone +from django.utils.encoding import force_text from django.utils.translation import ugettext_lazy as _ from django.views.generic import CreateView, ListView, TemplateView, UpdateView from django.views.generic.base import RedirectView @@ -53,7 +54,7 @@ from sapl.utils import (autor_label, autor_modal, gerar_hash_arquivo, get_base_u get_client_ip, get_mime_type_from_file_extension, lista_anexados, mail_service_configured, montar_row_autor, SEPARADOR_HASH_PROPOSICAO, show_results_filter_set, get_tempfile_dir, - google_recaptcha_configured) + google_recaptcha_configured, MultiFormatOutputMixin) from .forms import (AcessorioEmLoteFilterSet, AcompanhamentoMateriaForm, AnexadaEmLoteFilterSet, AdicionarVariasAutoriasFilterSet, @@ -104,10 +105,10 @@ def proposicao_texto(request, pk): if proposicao.texto_original: if (not proposicao.data_recebimento and - not proposicao.autor.operadores.filter( - id=request.user.id + not proposicao.autor.operadores.filter( + id=request.user.id ).exists() - ): + ): logger.error("user=" + username + ". Usuário ({}) não tem permissão para acessar o texto original." .format(request.user.id)) messages.error(request, _( @@ -1261,11 +1262,11 @@ class HistoricoProposicaoView(PermissionRequiredMixin, ListView): paginate_by = 10 model = HistoricoProposicao permission_required = ( - 'materia.list_historicoproposicao', - 'materia.add_historicoproposicao', - 'materia.change_historicoproposicao', - 'materia.delete_historicoproposicao', - 'materia.detail_historicoproposicao', + 'materia.list_historicoproposicao', + 'materia.add_historicoproposicao', + 'materia.change_historicoproposicao', + 'materia.delete_historicoproposicao', + 'materia.detail_historicoproposicao', ) def get_queryset(self): @@ -2039,11 +2040,27 @@ class AcompanhamentoExcluirView(TemplateView): return HttpResponseRedirect(self.get_success_url()) -class MateriaLegislativaPesquisaView(FilterView): +class MateriaLegislativaPesquisaView(MultiFormatOutputMixin, FilterView): model = MateriaLegislativa filterset_class = MateriaLegislativaFilterSet paginate_by = 50 + fields_base_report = [ + 'id', 'ano', 'numero', 'tipo__sigla', 'tipo__descricao', 'autoria__autor__nome', 'texto_original', 'ementa' + ] + fields_report = { + 'csv': fields_base_report, + 'xlsx': fields_base_report, + 'json': fields_base_report, + } + + def hook_texto_original(self, obj): + url = self.request.build_absolute_uri('/')[:-1] + texto_original = obj.texto_original if not isinstance( + obj, dict) else obj["texto_original"] + + return f'{url}/{texto_original}' + def get_filterset_kwargs(self, filterset_class): super().get_filterset_kwargs(filterset_class) @@ -2099,8 +2116,9 @@ class MateriaLegislativaPesquisaView(FilterView): qs = qs.filter(materiaassunto__isnull=True) if 'o' in self.request.GET and not self.request.GET['o']: - args = ['-ano', 'tipo__sequencia_regimental', '-numero'] if BaseAppConfig.attr('ordenacao_pesquisa_materia') == 'R' else ['-ano', 'tipo__sigla', '-numero'] - + args = ['-ano', 'tipo__sequencia_regimental', '-numero'] if BaseAppConfig.attr( + 'ordenacao_pesquisa_materia') == 'R' else ['-ano', 'tipo__sigla', '-numero'] + qs = qs.order_by(*args) kwargs.update({ diff --git a/sapl/norma/views.py b/sapl/norma/views.py index fd12ebaf6..ce5c1a365 100644 --- a/sapl/norma/views.py +++ b/sapl/norma/views.py @@ -28,7 +28,7 @@ from sapl.crud.base import (RP_DETAIL, RP_LIST, Crud, CrudAux, MasterDetailCrud, make_pagination) from sapl.materia.models import Orgao from sapl.utils import show_results_filter_set, get_client_ip,\ - sapn_is_enabled + sapn_is_enabled, MultiFormatOutputMixin from .forms import (AnexoNormaJuridicaForm, NormaFilterSet, NormaJuridicaForm, NormaPesquisaSimplesForm, NormaRelacionadaForm, @@ -104,7 +104,7 @@ class PesquisarAssuntoNormaView(FilterView): if 'assunto' in self.request.META['QUERY_STRING'] or\ 'page' in self.request.META['QUERY_STRING']: - resultados = self.object_list + resultados = self.object_list else: resultados = [] @@ -147,11 +147,27 @@ class NormaRelacionadaCrud(MasterDetailCrud): layout_key = 'NormaRelacionadaDetail' -class NormaPesquisaView(FilterView): +class NormaPesquisaView(MultiFormatOutputMixin, FilterView): model = NormaJuridica filterset_class = NormaFilterSet paginate_by = 50 + fields_base_report = [ + 'id', 'ano', 'numero', 'tipo__sigla', 'tipo__descricao', 'texto_integral', 'ementa' + ] + fields_report = { + 'csv': fields_base_report, + 'xlsx': fields_base_report, + 'json': fields_base_report, + } + + def hook_texto_integral(self, obj): + url = self.request.build_absolute_uri('/')[:-1] + texto_integral = obj.texto_integral if not isinstance( + obj, dict) else obj["texto_integral"] + + return f'{url}/{texto_integral}' + def get_queryset(self): qs = super().get_queryset() diff --git a/sapl/templates/crud/format_options.html b/sapl/templates/crud/format_options.html new file mode 100644 index 000000000..eb3bdd18d --- /dev/null +++ b/sapl/templates/crud/format_options.html @@ -0,0 +1,10 @@ +{% load i18n %} + + +
+
+ + + +
+
diff --git a/sapl/templates/materia/materialegislativa_filter.html b/sapl/templates/materia/materialegislativa_filter.html index 3a51dc973..bf4a731b8 100644 --- a/sapl/templates/materia/materialegislativa_filter.html +++ b/sapl/templates/materia/materialegislativa_filter.html @@ -7,7 +7,15 @@ {% block actions %} -
+ {% if show_results %} +
+ {% with 'sapl.materia:pesquisar_materia' as url_reverse %} + {% include "crud/format_options.html" %} + {% endwith %} +
+ {% endif %} + +
{% switch "SOLR_SWITCH" %} Pesquisa Textual diff --git a/sapl/templates/norma/normajuridica_filter.html b/sapl/templates/norma/normajuridica_filter.html index ce10aec10..d317f7bd6 100644 --- a/sapl/templates/norma/normajuridica_filter.html +++ b/sapl/templates/norma/normajuridica_filter.html @@ -4,6 +4,15 @@ {% load crispy_forms_tags common_tags %} {% block actions %} + + {% if show_results %} +
+ {% with 'sapl.norma:norma_pesquisa' as url_reverse %} + {% include "crud/format_options.html" %} + {% endwith %} +
+ {% endif %} +
{% switch "SOLR_SWITCH" %} @@ -19,7 +28,6 @@ {% trans 'Fazer nova pesquisa' %} {% endif %}
-

{% endblock %} {% block detail_content %} diff --git a/sapl/utils.py b/sapl/utils.py index de31e05b0..5ceeecfa5 100644 --- a/sapl/utils.py +++ b/sapl/utils.py @@ -1,5 +1,7 @@ +import csv from functools import wraps import hashlib +import io from itertools import groupby, chain import logging from operator import itemgetter @@ -30,6 +32,7 @@ from django.db.models import Q from django.db.models.fields.related import ForeignKey from django.forms import BaseForm from django.forms.widgets import SplitDateTimeWidget +from django.http.response import JsonResponse, HttpResponse from django.utils import six, timezone from django.utils.safestring import mark_safe from django.utils.translation import ugettext_lazy as _ @@ -39,6 +42,7 @@ from floppyforms import ClearableFileInput import magic import requests from unipath.path import Path +from xlsxwriter.workbook import Workbook from sapl.crispy_layout_mixin import (form_actions, SaplFormHelper, SaplFormLayout, to_row) @@ -1242,12 +1246,12 @@ def get_report_urls_map(): dst_url = reverse(f"{NAMESPACE}{url.name}") url_map[dst_url] = {"name": url.name, "public": True, - "internal": True} #TODO: get permissions from AppConfig and fine grained permissions + "internal": True} # TODO: get permissions from AppConfig and fine grained permissions return url_map def is_report_allowed(request, url_path=None): - from sapl.utils import get_report_urls_map # TODO: import global + from sapl.utils import get_report_urls_map # TODO: import global url_map = get_report_urls_map() # TODO: cache this!!! Globally path = url_path if url_path else request.path @@ -1289,4 +1293,218 @@ def get_path_to_name_report_map(): '/sistema/relatorios/historico-tramitacoesadm': 'Histórico de tramitações de documentos', '/sistema/relatorios/documentos_acessorios': 'Documentos Acessórios de Matérias Legislativas', '/sistema/relatorios/normas-por-autor': 'Normas Por Autor' - } \ No newline at end of file + } + + +class MultiFormatOutputMixin: + + formats_impl = 'csv', 'xlsx', 'json' + + queryset_values_for_formats = True + + def render_to_response(self, context, **response_kwargs): + + format_result = getattr(self.request, self.request.method).get( + 'format', None) + + if format_result: + if format_result not in self.formats_impl: + raise ValidationError( + 'Formato Inválido e/ou não implementado!') + + object_list = context['object_list'] + object_list.query.low_mark = 0 + object_list.query.high_mark = 0 + + return getattr(self, f'render_to_{format_result}')(context) + + return super().render_to_response(context, **response_kwargs) + + def render_to_json(self, context): + + object_list = context['object_list'] + + if self.queryset_values_for_formats: + object_list = object_list.values( + *self.fields_report['json']) + + data = [] + for obj in object_list: + wr = list(self._write_row(obj, 'json')) + + if not data: + data.append([wr]) + continue + + if wr[0] != data[-1][0][0]: + data.append([wr]) + else: + data[-1].append(wr) + + for mri, multirows in enumerate(data): + if len(multirows) == 1: + v = multirows[0] + else: + v = multirows[0] + for ri, cols in enumerate(multirows[1:]): + for rc, cell in enumerate(cols): + if v[rc] != cell: + v[rc] = f'{v[rc]}\r\n{cell}' + + data[mri] = dict( + map(lambda i, j: (i, j), self.fields_report['json'], v)) + + json_metadata = { + 'headers': dict( + map(lambda i, j: (i, j), self.fields_report['json'], self._headers('json'))), + 'results': data + } + response = JsonResponse(json_metadata) + response['Content-Disposition'] = f'attachment; filename="sapl_{self.request.resolver_match.url_name}.json"' + response['Cache-Control'] = 'no-cache' + response['Pragma'] = 'no-cache' + response['Expires'] = 0 + + return response + + def render_to_csv(self, context): + response = HttpResponse(content_type='text/csv') + response['Content-Disposition'] = f'attachment; filename="sapl_{self.request.resolver_match.url_name}.csv"' + response['Cache-Control'] = 'no-cache' + response['Pragma'] = 'no-cache' + response['Expires'] = 0 + writer = csv.writer(response, delimiter=";", + quoting=csv.QUOTE_NONNUMERIC) + + object_list = context['object_list'] + + if self.queryset_values_for_formats: + object_list = object_list.values( + *self.fields_report['csv']) + + data = [[list(self._headers('csv'))], ] + for obj in object_list: + wr = list(self._write_row(obj, 'csv')) + if wr[0] != data[-1][0][0]: + data.append([wr]) + else: + data[-1].append(wr) + + for mri, multirows in enumerate(data): + if len(multirows) == 1: + writer.writerow(multirows[0]) + else: + v = multirows[0] + for ri, cols in enumerate(multirows[1:]): + for rc, cell in enumerate(cols): + if v[rc] != cell: + v[rc] = f'{v[rc]}\r\n{cell}' + + writer.writerow(v) + + return response + + def render_to_xlsx(self, context): + + object_list = context['object_list'] + + if self.queryset_values_for_formats: + object_list = object_list.values( + *self.fields_report['xlsx']) + + data = [[list(self._headers('xlsx'))], ] + for obj in object_list: + wr = list(self._write_row(obj, 'xlsx')) + if wr[0] != data[-1][0][0]: + data.append([wr]) + else: + data[-1].append(wr) + + output = io.BytesIO() + wb = Workbook(output, {'in_memory': True}) + + ws = wb.add_worksheet() + + for mri, multirows in enumerate(data): + if len(multirows) == 1: + for rc, cell in enumerate(multirows[0]): + ws.write(mri, rc, cell) + else: + v = multirows[0] + for ri, cols in enumerate(multirows[1:]): + for rc, cell in enumerate(cols): + if v[rc] != cell: + v[rc] = f'{v[rc]}\r\n{cell}' + + for rc, cell in enumerate(v): + ws.write(mri, rc, cell) + ws.autofit() + wb.close() + + output.seek(0) + + response = HttpResponse(output.read( + ), content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") + response['Content-Disposition'] = f'attachment; filename="sapl_{self.request.resolver_match.url_name}.xlsx"' + response['Cache-Control'] = 'no-cache' + response['Pragma'] = 'no-cache' + response['Expires'] = 0 + + output.close() + + return response + + def _write_row(self, obj, format_result): + + for fname in self.fields_report[format_result]: + + if hasattr(self, f'hook_{fname}'): + v = getattr(self, f'hook_{fname}')(obj) + yield v + continue + + if isinstance(obj, dict): + yield obj[fname] + continue + + fname = fname.split('__') + + v = obj + for fp in fname: + v = getattr(v, fp) + + if hasattr(v, 'all'): + v = ' - '.join(map(lambda x: str(x), v.all())) + + yield v + + def _headers(self, format_result): + + for fname in self.fields_report[format_result]: + + verbose_name = [] + + if hasattr(self, f'hook_header_{fname}'): + h = getattr(self, f'hook_header_{fname}')() + yield h + continue + + fname = fname.split('__') + + m = self.model + for fp in fname: + + f = m._meta.get_field(fp) + + vn = str(f.verbose_name) if hasattr(f, 'verbose_name') else fp + if f.is_relation: + m = f.related_model + if m == self.model: + m = f.field.model + + if vn == fp: + vn = str(m._meta.verbose_name_plural) + verbose_name.append(vn.strip()) + + verbose_name = '/'.join(verbose_name).strip() + yield f'{verbose_name}'