From 450c3ace3b72e9d36681450560daea4e0459baf1 Mon Sep 17 00:00:00 2001 From: Edward Oliveira Date: Tue, 26 Aug 2025 13:12:10 -0300 Subject: [PATCH] CSV/xlsx/json refactoring --- sapl/base/apps.py | 1 - sapl/materia/views.py | 23 +- sapl/norma/views.py | 7 +- sapl/protocoloadm/views.py | 8 +- sapl/sessao/views.py | 42 ++-- sapl/utils.py | 441 ++++++++++++++----------------------- 6 files changed, 199 insertions(+), 323 deletions(-) diff --git a/sapl/base/apps.py b/sapl/base/apps.py index 403d74bd1..7e4305ea8 100644 --- a/sapl/base/apps.py +++ b/sapl/base/apps.py @@ -1,4 +1,3 @@ - import django from django.utils.translation import ugettext_lazy as _ diff --git a/sapl/materia/views.py b/sapl/materia/views.py index a4f557d79..f64d8134b 100644 --- a/sapl/materia/views.py +++ b/sapl/materia/views.py @@ -2045,22 +2045,29 @@ class MateriaLegislativaPesquisaView(MultiFormatOutputMixin, FilterView): filterset_class = MateriaLegislativaFilterSet paginate_by = 50 - fields_base_report = [ - 'id', 'ano', 'numero', 'tipo__sigla', 'tipo__descricao', 'autoria__autor__nome', 'texto_original', 'ementa' + export_fields = [ + 'id', 'ano', 'numero', 'tipo__sigla', 'tipo__descricao', 'autoria', 'texto_original', 'ementa' ] - fields_report = { - 'csv': fields_base_report, - 'xlsx': fields_base_report, - 'json': fields_base_report, - } def hook_texto_original(self, obj): url = self.request.build_absolute_uri('/')[:-1] texto_original = obj.texto_original if not isinstance( obj, dict) else obj["texto_original"] - return f'{url}/media/{texto_original}' + def hook_autoria(self, obj): + """ + Hook específico para pegar nomes dos autores (reverse query) + """ + try: + autores = [ + str(autoria.autor.nome) + for autoria in obj.autoria_set.select_related('autor').all() + ] + return ', '.join(autores) + except AttributeError: + return '' + def get_filterset_kwargs(self, filterset_class): super().get_filterset_kwargs(filterset_class) diff --git a/sapl/norma/views.py b/sapl/norma/views.py index d746483e9..238650291 100644 --- a/sapl/norma/views.py +++ b/sapl/norma/views.py @@ -152,14 +152,9 @@ class NormaPesquisaView(MultiFormatOutputMixin, FilterView): filterset_class = NormaFilterSet paginate_by = 50 - fields_base_report = [ + export_fields = [ 'id', 'ano', 'numero', 'tipo__sigla', 'tipo__descricao', 'texto_integral', 'ementa' ] - fields_report = { - 'csv': fields_base_report, - 'xlsx': fields_base_report, - 'json': fields_base_report, - } def hook_texto_integral(self, obj): url = self.request.build_absolute_uri('/')[:-1] diff --git a/sapl/protocoloadm/views.py b/sapl/protocoloadm/views.py index 1b572d792..fb6f48055 100755 --- a/sapl/protocoloadm/views.py +++ b/sapl/protocoloadm/views.py @@ -992,7 +992,6 @@ class ProtocoloMateriaView(PermissionRequiredMixin, CreateView): def get_context_data(self, **kwargs): context = super(CreateView, self).get_context_data(**kwargs) autores_ativos = self.autores_ativos() - autores = [] autores.append(['0', '------']) for a in autores_ativos: @@ -1042,14 +1041,9 @@ class PesquisarDocumentoAdministrativoView(DocumentoAdministrativoMixin, paginate_by = 10 permission_required = ('protocoloadm.list_documentoadministrativo', ) - fields_base_report = [ + export_fields = [ 'id', 'ano', 'numero', 'tipo__sigla', 'tipo__descricao', 'assunto' ] - fields_report = { - 'csv': fields_base_report, - 'xlsx': fields_base_report, - 'json': fields_base_report, - } def get_filterset_kwargs(self, filterset_class): super(PesquisarDocumentoAdministrativoView, diff --git a/sapl/sessao/views.py b/sapl/sessao/views.py index 84cb755e5..4835a17b6 100755 --- a/sapl/sessao/views.py +++ b/sapl/sessao/views.py @@ -3813,22 +3813,23 @@ class PautaSessaoDetailView(PautaMultiFormatOutputMixin, DetailView): template_name = "sessao/pauta_sessao_detail.html" model = SessaoPlenaria - queryset_values_for_formats = False + export_fields = ( + ('id', 'ID'), + ('periodo', 'Período'), + ('titulo', 'Matéria'), + ('autor', 'Autor'), + ('ementa', 'Ementa'), + ('situacao', 'Situação') + ) - fields_base_report = [ - [('id', 'ID'), ('titulo', 'Matéria'), ('autor', 'Autor'), ('ementa', 'Ementa'), ('situacao', 'Situação')], - [('id', 'ID'), ('titulo', 'Matéria'), ('autor', 'Autor'), ('ementa', 'Ementa'), ('situacao', 'Situação')] - ] - fields_report = { - 'csv': fields_base_report, - 'xlsx': fields_base_report, - 'json': fields_base_report, - } + def hook_autor(self, obj): + return ','.join(obj['autor']) + + def hook_titulo(self, obj): + return str(obj['titulo']) - item_context = [ - ('materia_expediente', 'Matérias do Expediente'), - ('materias_ordem', 'Matérias da Ordem do Dia') - ] + def hook_situacao(self, obj): + return str(obj['situacao']) def get(self, request, *args, **kwargs): from sapl.relatorios.views import relatorio_pauta_sessao_weasy # Evitar import ciclico @@ -3888,7 +3889,8 @@ class PautaSessaoDetailView(PautaMultiFormatOutputMixin, DetailView): 'situacao': ultima_tramitacao.status if ultima_tramitacao else _("Não informada"), 'processo': f'{str(numeracao.numero_materia)}/{str(numeracao.ano_materia)}' if numeracao else '-', 'autor': [str(x.autor) for x in m.materia.autoria_set.select_related('autor').all()], - 'turno': get_turno(ultima_tramitacao.turno) if ultima_tramitacao else '' + 'turno': get_turno(ultima_tramitacao.turno) if ultima_tramitacao else '', + 'periodo': 'expediente', }) context.update({'materia_expediente': materias_expediente}) @@ -3972,7 +3974,8 @@ class PautaSessaoDetailView(PautaMultiFormatOutputMixin, DetailView): 'situacao': ultima_tramitacao.status if ultima_tramitacao else _("Não informada"), 'processo': f'{str(numeracao.numero_materia)}/{str(numeracao.ano_materia)}' if numeracao else '-', 'autor': [str(x.autor) for x in Autoria.objects.select_related("autor").filter(materia_id=o.materia_id)], - 'turno': get_turno(ultima_tramitacao.turno) if ultima_tramitacao else '' + 'turno': get_turno(ultima_tramitacao.turno) if ultima_tramitacao else '', + 'periodo': 'ordem dia', }) context.update({ @@ -3998,14 +4001,9 @@ class PesquisarSessaoPlenariaView(MultiFormatOutputMixin, FilterView): queryset_values_for_formats = False - fields_base_report = [ + export_fields = [ 'id', 'data_inicio', 'hora_inicio', 'data_fim', 'hora_fim', '', ] - fields_report = { - 'csv': fields_base_report, - 'xlsx': fields_base_report, - 'json': fields_base_report, - } def get_filterset_kwargs(self, filterset_class): super().get_filterset_kwargs(filterset_class) diff --git a/sapl/utils.py b/sapl/utils.py index cddeb1f49..757e698d0 100644 --- a/sapl/utils.py +++ b/sapl/utils.py @@ -78,7 +78,6 @@ def is_weak_password(password): return len(password) < MIN_PASSWORD_LENGTH or not (pwd_has_lowercase and pwd_has_uppercase and pwd_has_number and pwd_has_special_char) - def groups_remove_user(user, groups_name): from django.contrib.auth.models import Group @@ -1169,12 +1168,12 @@ def from_date_to_datetime_utc(data): class OverwriteStorage(FileSystemStorage): - ''' + """ Solução derivada do gist: https://gist.github.com/fabiomontefuscolo/1584462 Muda o comportamento padrão do Django e o faz sobrescrever arquivos de mesmo nome que foram carregados pelo usuário ao invés de renomeá-los. - ''' + """ def get_available_name(self, name, max_length=None): if self.exists(name): @@ -1331,13 +1330,62 @@ def get_path_to_name_report_map(): } +class Row: + def __init__(self, cols, is_header = False): + self.cols = cols + self.is_header = is_header + + def __repr__(self): + return f"Row(columns={self.cols}, is_header={self.is_header})" + + +class Table: + def __init__(self, header = [], rows = []): + self.header = header + self.rows = rows + + def add_header(self, header): + if header.is_header: + self.header = header + else: + raise Exception(f"Row {header} is not header!") + + def append(self, row): + if not row.is_header: + self.rows.append(row) + else: + raise Exception(f"Row {row} is header!") + + def to_list(self): + return [self.header.cols] + [r.cols for r in self.rows] + + def size(self): + return len(self.rows) + + def __repr__(self): + return f"Table(Header={self.header}, Rows={self.rows})" + + class MultiFormatOutputMixin: - formats_impl = 'csv', 'xlsx', 'json' + formats_impl = ['csv', 'xlsx', 'json'] queryset_values_for_formats = True - def render_to_response(self, context, **response_kwargs): + export_fields = () + fields_by_format = None + def get_export_fields(self): + names = getattr(self, "export_fields", []) or [] + return names + + def get_fields_by_format(self): + provided = getattr(self, "fields_by_format", None) + if provided: + return {fmt: v for fmt, v in provided.items()} + fields = self.get_export_fields() + return {fmt: fields for fmt in self.formats_impl} + + def render_to_response(self, context, **response_kwargs): format_result = getattr(self.request, self.request.method).get( 'format', None) @@ -1356,183 +1404,157 @@ class MultiFormatOutputMixin: return super().render_to_response(context, **response_kwargs) def render_to_json(self, context): - object_list = context['object_list'] - if self.queryset_values_for_formats: - object_list = object_list.values( - *self.fields_report['json']) - - data = [] - for obj in object_list: - wr = list(self._write_row(obj, self.fields_report['json'])) - - if not data: - data.append([wr]) - continue - - if wr[0] != data[-1][0][0]: - data.append([wr]) - else: - data[-1].append(wr) - - for mri, multirows in enumerate(data): - if len(multirows) == 1: - v = multirows[0] - else: - v = multirows[0] - for ri, cols in enumerate(multirows[1:]): - for rc, cell in enumerate(cols): - if v[rc] != cell: - v[rc] = f'{v[rc]}\r\n{cell}' + fmt_map = self.get_fields_by_format() + export_fields = fmt_map['json'] + rows = [Row(cols=self._extract_row(obj, export_fields)) for obj in object_list] + table = Table(header=Row(export_fields, is_header=True), rows=rows) - data[mri] = dict( - map(lambda i, j: (i, j), self.fields_report['json'], v)) + headers = self._headers(export_fields) + if isinstance(export_fields, tuple): + export_fields = [f[0] for f in export_fields] json_metadata = { - 'headers': dict( - map(lambda i, j: (i, j), self.fields_report['json'], self._headers(self.fields_report['json']))), - 'results': data + 'headers': dict(zip(export_fields, headers)), + 'results': [dict(zip(export_fields, row.cols)) for row in table.rows], } - response = JsonResponse(json_metadata) - response['Content-Disposition'] = f'attachment; filename="sapl_{self.request.resolver_match.url_name}.json"' - response['Cache-Control'] = 'no-cache' - response['Pragma'] = 'no-cache' - response['Expires'] = 0 - + response = self._set_response_params(JsonResponse(json_metadata), 'json') return response def render_to_csv(self, context): - response = HttpResponse(content_type='text/csv') - response['Content-Disposition'] = f'attachment; filename="sapl_{self.request.resolver_match.url_name}.csv"' - response['Cache-Control'] = 'no-cache' - response['Pragma'] = 'no-cache' - response['Expires'] = 0 - writer = csv.writer(response, delimiter=";", - quoting=csv.QUOTE_NONNUMERIC) - object_list = context['object_list'] - if self.queryset_values_for_formats: - object_list = object_list.values( - *self.fields_report['csv']) - - data = [[list(self._headers(self.fields_report['csv']))], ] - for obj in object_list: - wr = list(self._write_row(obj, self.fields_report['csv'])) - if wr[0] != data[-1][0][0]: - data.append([wr]) - else: - data[-1].append(wr) - - for mri, multirows in enumerate(data): - if len(multirows) == 1: - writer.writerow(multirows[0]) - else: - v = multirows[0] - for ri, cols in enumerate(multirows[1:]): - for rc, cell in enumerate(cols): - if v[rc] != cell: - v[rc] = f'{v[rc]}\r\n{cell}' - - writer.writerow(v) + fmt_map = self.get_fields_by_format() + export_fields = fmt_map['csv'] + rows = [Row(self._extract_row(obj, export_fields)) for obj in object_list] + table = Table(header=Row(self._headers(export_fields), is_header=True), rows=rows) + response = self._set_response_params(HttpResponse(content_type='text/csv'), 'csv') + writer = csv.writer(response, + delimiter=";", + quoting=csv.QUOTE_NONNUMERIC) + writer.writerows(table.to_list()) return response def render_to_xlsx(self, context): - object_list = context['object_list'] - if self.queryset_values_for_formats: - object_list = object_list.values( - *self.fields_report['xlsx']) - - data = [[list(self._headers(self.fields_report['xlsx']))], ] - for obj in object_list: - wr = list(self._write_row(obj, self.fields_report['xlsx'])) - if wr[0] != data[-1][0][0]: - data.append([wr]) - else: - data[-1].append(wr) + fmt_map = self.get_fields_by_format() + field_names = fmt_map['xlsx'] + rows = [Row(self._extract_row(obj, field_names)) for obj in object_list] + table = Table(header=Row(self._headers(field_names), is_header=True), rows=rows) output = io.BytesIO() wb = Workbook(output, {'in_memory': True}) - ws = wb.add_worksheet() - for mri, multirows in enumerate(data): - if len(multirows) == 1: - for rc, cell in enumerate(multirows[0]): - ws.write(mri, rc, cell) - else: - v = multirows[0] - for ri, cols in enumerate(multirows[1:]): - for rc, cell in enumerate(cols): - if v[rc] != cell: - v[rc] = f'{v[rc]}\r\n{cell}' - - for rc, cell in enumerate(v): - ws.write(mri, rc, cell) + for row_idx, row in enumerate(table.to_list()): + for cell_idx, cell in enumerate(row): + ws.write(row_idx, cell_idx, cell) + ws.autofit() wb.close() - output.seek(0) - response = HttpResponse(output.read( - ), content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") - response['Content-Disposition'] = f'attachment; filename="sapl_{self.request.resolver_match.url_name}.xlsx"' - response['Cache-Control'] = 'no-cache' - response['Pragma'] = 'no-cache' - response['Expires'] = 0 - + response = self._set_response_params( + HttpResponse(output.read(), + content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"), + 'xlsx' + ) output.close() - return response - def _write_row(self, obj, fields_report): - - for fname in fields_report: + def _extract_row(self, obj, columns): + """Versão mais simples que não chama métodos""" + for col_name in columns: + if isinstance(col_name, tuple): + col_name = col_name[0] - if type(fname) is tuple: - fname = fname[0] - - if hasattr(self, f'hook_{fname}'): - v = getattr(self, f'hook_{fname}')(obj) - yield v + # Hook personalizado + if hasattr(self, f'hook_{col_name}'): + value = getattr(self, f'hook_{col_name}')(obj) + yield value continue + # Dicionário if isinstance(obj, dict): - yield obj[fname] + yield obj.get(col_name, '') continue - fname = fname.split('__') - - v = obj - for fp in fname: - v = getattr(v, fp) + # Navegação SEM chamar callables + try: + field_parts = col_name.split('__') + value = obj + for part in field_parts: + if value is None: + value = '' + break + value = getattr(value, part) + # NÃO chamamos callables - apenas pegamos o atributo + + # Tratamento para relacionamentos + if hasattr(value, 'all'): + items = value.all() + value = ' - '.join(str(item) for item in items) if items.exists() else '' + + yield str(value) if value is not None else '' + + except AttributeError: + # Variações para relacionamentos reversos + base_field = col_name.split('__')[0] + variations = [f'{base_field}_set', f'{base_field}s'] + + found = False + for variation in variations: + try: + new_field = col_name.replace(base_field, variation, 1) + field_parts = new_field.split('__') + value = obj + for part in field_parts: + value = getattr(value, part) + + if hasattr(value, 'all'): + items = value.all() + value = ' - '.join(str(item) for item in items) if items.exists() else '' + + yield str(value) if value is not None else '' + found = True + break + except AttributeError: + continue - if hasattr(v, 'all'): - v = ' - '.join(map(lambda x: str(x), v.all())) + if not found: + yield '' - yield v + def _set_response_params(self, response, extension): + response[ + 'Content-Disposition'] = f'attachment; filename="sapl_{self.request.resolver_match.url_name}.{extension}"' + response['Cache-Control'] = 'no-cache' + response['Pragma'] = 'no-cache' + response['Expires'] = 0 + return response - def _headers(self, fields_report): + def _headers(self, field_names): - for fname in fields_report: + for fname in field_names: verbose_name = [] - if hasattr(self, f'hook_header_{fname}'): + if hasattr(self, f'hook_header_{fname}'): # suporta extensao de funcionalidade h = getattr(self, f'hook_header_{fname}')() yield h continue if type(fname) is tuple: - verbose_name.append(fname[1]) + verbose_name.append(fname[1]) # suporta (field_name, alias) else: fname = fname.split('__') + # nem sempre isso vai funcionar, pois o model base + # pode ser diferente. Exemplo: pauta usa SessaoPlenaria, + # mas retornamos campos de MateriaLegislativa m = self.model for fp in fname: @@ -1554,156 +1576,17 @@ class MultiFormatOutputMixin: class PautaMultiFormatOutputMixin(MultiFormatOutputMixin): - def render_to_csv(self, context): - response = HttpResponse(content_type='text/csv') - response['Content-Disposition'] = f'attachment; filename="sapl_{self.request.resolver_match.url_name}.csv"' - response['Cache-Control'] = 'no-cache' - response['Pragma'] = 'no-cache' - response['Expires'] = 0 - writer = csv.writer(response, delimiter=";", - quoting=csv.QUOTE_NONNUMERIC) - - writer.writerow(['Pauta da ' + str(context['sessaoplenaria'])]) - writer.writerow('') - - for item in self.item_context: - if item[0] in context: - - index = self.item_context.index(item) - writer.writerow([self.item_context[index][1]]) - - data = [[list(self._headers(self.fields_report['csv'][index]))], ] - for obj in context.get(item[0]): - wr = list(self._write_row(obj, self.fields_report['csv'][index])) - if wr[0] != data[-1][0][0]: - data.append([wr]) - else: - data[-1].append(wr) - - for mri, multirows in enumerate(data): - if len(multirows) == 1: - writer.writerow(multirows[0]) - else: - v = multirows[0] - for ri, cols in enumerate(multirows[1:]): - for rc, cell in enumerate(cols): - if v[rc] != cell: - v[rc] = f'{v[rc]}\r\n{cell}' - - writer.writerow(v) - writer.writerow('') + def __mutate_context(self, context): + context_materias = { + 'object_list': context.get('materias_ordem', []) + context.get('materia_expediente', []) + } + return context_materias - return response + def render_to_csv(self, context): + return super().render_to_csv(self.__mutate_context(context)) def render_to_json(self, context): - - json_metadata = {'sessaoplenaria': str(context['sessaoplenaria'])} - for item in self.item_context: - if item[0] in context: - index = self.item_context.index(item) - json_metadata.update({item[0]: {}}) - data = [] - - for obj in context.get(item[0]): - wr = list(self._write_row(obj, self.fields_report['json'][index])) - - if not data: - data.append([wr]) - continue - - if wr[0] != data[-1][0][0]: - data.append([wr]) - else: - data[-1].append(wr) - - for mri, multirows in enumerate(data): - if len(multirows) == 1: - try: - v = multirows[0] - except TypeError: - v = str(multirows[0]) - else: - try: - v = str(multirows[0]) - except TypeError: - v = multirows[0] - for ri, cols in enumerate(multirows[1:]): - for rc, cell in enumerate(cols): - if v[rc] != cell: - v[rc] = f'{v[rc]}\r\n{cell}' - - data[mri] = dict( - map(lambda i, j: (i[0], j if type(j) in [str, int, list] else str(j)), - self.fields_report['json'][index], v)) - - json_metadata.update({item[0]: { - 'headers': dict( - map(lambda i, j: (i[0], j), self.fields_report['json'][index], - self._headers(self.fields_report['json'][index]))), - 'results': data} - }) - response = JsonResponse(json_metadata) - response['Content-Disposition'] = f'attachment; filename="sapl_{self.request.resolver_match.url_name}.json"' - response['Cache-Control'] = 'no-cache' - response['Pragma'] = 'no-cache' - response['Expires'] = 0 - - return response + return super().render_to_json(self.__mutate_context(context)) def render_to_xlsx(self, context): - - output = io.BytesIO() - wb = Workbook(output, {'in_memory': True}) - - ws = wb.add_worksheet() - ws.write('A1', 'Pauta da ' + str(context['sessaoplenaria'])) - row = 2 - - for item in self.item_context: - if item[0] in context: - index = self.item_context.index(item) - ws.write(row, 0, self.item_context[index][1]) - row += 1 - data = [[list(self._headers(self.fields_report['xlsx'][index]))], ] - - for obj in context.get(item[0]): - wr = list(self._write_row(obj, self.fields_report['xlsx'][index])) - if wr[0] != data[-1][0][0]: - data.append([wr]) - else: - data[-1].append(wr) - - for mri, multirows in enumerate(data): - if len(multirows) == 1: - for rc, cell in enumerate(multirows[0]): - try: - ws.write(row, rc, cell) - except TypeError: - ws.write(row, rc, str(cell)) - row += 1 - else: - v = multirows[0] - for ri, cols in enumerate(multirows[1:]): - for rc, cell in enumerate(cols): - if v[rc] != cell: - v[rc] = f'{v[rc]}\r\n{cell}' - - for rc, cell in enumerate(v): - ws.write(row, rc, cell) - row += 1 - row += 1 - ws.autofit() - wb.close() - - output.seek(0) - - response = HttpResponse(output.read( - ), content_type="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet") - response['Content-Disposition'] = f'attachment; filename="sapl_{self.request.resolver_match.url_name}.xlsx"' - response['Cache-Control'] = 'no-cache' - response['Pragma'] = 'no-cache' - response['Expires'] = 0 - - output.close() - - return response + return super().render_to_xlsx(self.__mutate_context(context))