From 94c0a7190fc8ee43b4a34dbf09bafe858bcd4e46 Mon Sep 17 00:00:00 2001 From: Talitha Pumar Date: Fri, 13 Apr 2018 12:28:51 -0300 Subject: [PATCH 001/102] Fix #1856 (#1866) --- sapl/comissoes/forms.py | 8 +++++++- sapl/comissoes/views.py | 4 ++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/sapl/comissoes/forms.py b/sapl/comissoes/forms.py index 0a3718b1d..3576873b7 100644 --- a/sapl/comissoes/forms.py +++ b/sapl/comissoes/forms.py @@ -13,10 +13,16 @@ from sapl.parlamentares.models import Legislatura, Mandato, Parlamentar class ComposicaoForm(forms.ModelForm): + comissao = forms.CharField(required=False, label='Comissao', widget=forms.HiddenInput()) + class Meta: model = Composicao exclude = [] + def __init__(self, user=None, **kwargs): + super(ComposicaoForm, self).__init__(**kwargs) + self.fields['comissao'].widget.attrs['disabled'] = 'disabled' + def clean(self): cleaned_data = super(ComposicaoForm, self).clean() @@ -24,7 +30,7 @@ class ComposicaoForm(forms.ModelForm): return cleaned_data periodo = cleaned_data['periodo'] - comissao_pk = cleaned_data['comissao'].id + comissao_pk = self.initial['comissao'].id intersecao_periodo = Composicao.objects.filter( Q(periodo__data_inicio__lte=periodo.data_fim, periodo__data_fim__gte=periodo.data_fim) | diff --git a/sapl/comissoes/views.py b/sapl/comissoes/views.py index 6c47ac6e1..82c3a79c1 100644 --- a/sapl/comissoes/views.py +++ b/sapl/comissoes/views.py @@ -90,6 +90,10 @@ class ComposicaoCrud(MasterDetailCrud): class CreateView(MasterDetailCrud.CreateView): form_class = ComposicaoForm + + def get_initial(self): + comissao = Comissao.objects.get(id=self.kwargs['pk']) + return {'comissao': comissao} class ListView(MasterDetailCrud.ListView): From 90237c2a794490fb7ca4173834253ec20f93f42c Mon Sep 17 00:00:00 2001 From: Victor Fabre Date: Fri, 13 Apr 2018 12:39:18 -0300 Subject: [PATCH 002/102] Fix #1862 (#1867) --- sapl/protocoloadm/forms.py | 31 ++++++++++++++++++++++++++++--- sapl/protocoloadm/views.py | 18 +++++++++++++----- 2 files changed, 41 insertions(+), 8 deletions(-) diff --git a/sapl/protocoloadm/forms.py b/sapl/protocoloadm/forms.py index 9c9485ae4..eff6d09a1 100644 --- a/sapl/protocoloadm/forms.py +++ b/sapl/protocoloadm/forms.py @@ -308,6 +308,8 @@ class ProtocoloDocumentForm(ModelForm): observacao = forms.CharField(required=False, widget=forms.Textarea, label='Observação') + numero = forms.IntegerField(required=False, label='Número de Protocolo (opcional)') + class Meta: model = Protocolo fields = ['tipo_protocolo', @@ -316,6 +318,7 @@ class ProtocoloDocumentForm(ModelForm): 'assunto', 'interessado', 'observacao', + 'numero' ] def __init__(self, *args, **kwargs): @@ -331,6 +334,8 @@ class ProtocoloDocumentForm(ModelForm): [('interessado', 12)]) row5 = to_row( [('observacao', 12)]) + row6 = to_row( + [('numero', 12)]) self.helper = FormHelper() self.helper.layout = Layout( @@ -341,6 +346,11 @@ class ProtocoloDocumentForm(ModelForm): row4, row5, HTML(" "), + ), + Fieldset(_('Número do Protocolo (Apenas se quiser que a numeração comece ' + 'a partir do número a ser informado)'), + row6, + HTML(" "), form_actions(label=_('Protocolar Documento')) ) ) @@ -385,6 +395,8 @@ class ProtocoloMateriaForm(ModelForm): assunto_ementa = forms.CharField(required=True, widget=forms.Textarea, label='Ementa') + numero = forms.IntegerField(required=False, label='Número de Protocolo (opcional)') + class Meta: model = Protocolo fields = ['tipo_materia', @@ -395,7 +407,8 @@ class ProtocoloMateriaForm(ModelForm): 'observacao', 'numero_materia', 'ano_materia', - 'vincular_materia' + 'vincular_materia', + 'numero' ] def clean_autor(self): @@ -447,12 +460,24 @@ class ProtocoloMateriaForm(ModelForm): [('assunto_ementa', 12)]) row4 = to_row( [('observacao', 12)]) + row5 = to_row( + [('numero', 12)]) self.helper = FormHelper() self.helper.layout = Layout( Fieldset(_('Identificação da Matéria'), - row1, row2, row3, - row4, form_actions(label='Protocolar Matéria'))) + row1, + row2, + row3, + row4, + HTML(" "), + ), + Fieldset(_('Número do Protocolo (Apenas se quiser que a numeração comece' + ' a partir do número a ser informado)'), + row5, + HTML(" "), + form_actions(label=_('Protocolar Matéria'))) + ) super(ProtocoloMateriaForm, self).__init__( *args, **kwargs) diff --git a/sapl/protocoloadm/views.py b/sapl/protocoloadm/views.py index ee03daaff..2474bb919 100644 --- a/sapl/protocoloadm/views.py +++ b/sapl/protocoloadm/views.py @@ -304,7 +304,12 @@ class ProtocoloDocumentoView(PermissionRequiredMixin, protocolo.tipo_processo = '0' # TODO validar o significado protocolo.anulado = False - protocolo.numero = (numero['numero__max'] + 1) if numero['numero__max'] else 1 + if not protocolo.numero: + protocolo.numero = (numero['numero__max'] + 1) if numero['numero__max'] else 1 + if protocolo.numero < (numero['numero__max'] + 1): + msg = _('Número de protocolo deve ser maior que {}').format(numero['numero__max']) + messages.add_message(self.request, messages.ERROR, msg) + return self.render_to_response(self.get_context_data()) protocolo.ano = timezone.now().year protocolo.data = timezone.now() protocolo.hora = timezone.now().time() @@ -415,6 +420,7 @@ class ProtocoloMateriaView(PermissionRequiredMixin, CreateView): 'pk': protocolo.pk}) def form_valid(self, form): + protocolo = form.save(commit=False) try: numeracao = sapl.base.models.AppConfig.objects.last( ).sequencia_numeracao @@ -443,10 +449,12 @@ class ProtocoloMateriaView(PermissionRequiredMixin, CreateView): if numeracao is None: numero['numero__max'] = 0 - protocolo = Protocolo() - - protocolo.numero = ( - numero['numero__max'] + 1) if numero['numero__max'] else 1 + if not protocolo.numero: + protocolo.numero = (numero['numero__max'] + 1) if numero['numero__max'] else 1 + if protocolo.numero < (numero['numero__max'] + 1): + msg = _('Número de protocolo deve ser maior que {}').format(numero['numero__max']) + messages.add_message(self.request, messages.ERROR, msg) + return self.render_to_response(self.get_context_data()) protocolo.ano = timezone.now().year protocolo.data = timezone.now().date() protocolo.hora = timezone.now().time() From 365ca0f3ddb40fc327bb9866c5e0c6b590ef8d23 Mon Sep 17 00:00:00 2001 From: Talitha Pumar Date: Fri, 13 Apr 2018 12:42:08 -0300 Subject: [PATCH 003/102] #1858 - Trazer Sessao Legislativa Atual (#1860) * Fix #1858 * Fix #1858 * Update views.py --- sapl/sessao/views.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/sapl/sessao/views.py b/sapl/sessao/views.py index 01993f130..54d1fa21e 100644 --- a/sapl/sessao/views.py +++ b/sapl/sessao/views.py @@ -653,11 +653,15 @@ class SessaoCrud(Crud): def get_initial(self): legislatura = Legislatura.objects.order_by('-data_inicio').first() + if legislatura: return { 'legislatura': legislatura, - 'sessao_legislativa': legislatura.sessaoplenaria_set.first( - )} + 'sessao_legislativa': legislatura.sessaolegislativa_set.filter( + legislatura_id=legislatura.id, + data_inicio__year=timezone.now().year + ).first() + } else: msg = _('Cadastre alguma legislatura antes de adicionar ' + 'uma sessão plenária!') From ecabc8eb1312ddfe1ca4522e0b37160a75fdfa98 Mon Sep 17 00:00:00 2001 From: LeandroRoberto Date: Mon, 16 Apr 2018 10:27:56 -0300 Subject: [PATCH 004/102] Fix #1854 --- sapl/parlamentares/forms.py | 7 +++++++ sapl/static/styles/app.scss | 4 ++++ sapl/templates/parlamentares/layouts.yaml | 2 +- 3 files changed, 12 insertions(+), 1 deletion(-) diff --git a/sapl/parlamentares/forms.py b/sapl/parlamentares/forms.py index acee7d7e1..e5b8d7d58 100644 --- a/sapl/parlamentares/forms.py +++ b/sapl/parlamentares/forms.py @@ -163,6 +163,13 @@ class ParlamentarCreateForm(ParlamentarForm): required=True, ) + class Meta(ParlamentarForm.Meta): + widgets = { + 'fotografia': forms.ClearableFileInput(), + 'biografia': forms.Textarea( + attrs={'id': 'texto-rico'}) + } + @transaction.atomic def save(self, commit=True): parlamentar = super(ParlamentarCreateForm, self).save(commit) diff --git a/sapl/static/styles/app.scss b/sapl/static/styles/app.scss index 262b3f45e..6a578d102 100644 --- a/sapl/static/styles/app.scss +++ b/sapl/static/styles/app.scss @@ -229,8 +229,12 @@ label { .form-control-static { padding-top: 0; min-height: auto; + img { + max-width: 100%; + } } + // #### pagination ######################################## .pagination { padding-top: 25px; diff --git a/sapl/templates/parlamentares/layouts.yaml b/sapl/templates/parlamentares/layouts.yaml index d31076100..ba7767cd2 100644 --- a/sapl/templates/parlamentares/layouts.yaml +++ b/sapl/templates/parlamentares/layouts.yaml @@ -42,7 +42,7 @@ Parlamentar: - municipio_residencia uf_residencia - telefone_residencia fax_residencia - locais_atuacao - - fotografia + - fotografia:5 - biografia ParlamentarUpdate: From 5593435d73a33e1c0f35cc308b9c6055383f9912 Mon Sep 17 00:00:00 2001 From: Talitha Pumar Date: Mon, 16 Apr 2018 11:30:59 -0300 Subject: [PATCH 005/102] Fix #1838 (#1847) --- sapl/relatorios/templates/pdf_etiqueta_protocolo_gerar.py | 6 +++++- sapl/relatorios/views.py | 5 +---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/sapl/relatorios/templates/pdf_etiqueta_protocolo_gerar.py b/sapl/relatorios/templates/pdf_etiqueta_protocolo_gerar.py index 142065450..38057791f 100755 --- a/sapl/relatorios/templates/pdf_etiqueta_protocolo_gerar.py +++ b/sapl/relatorios/templates/pdf_etiqueta_protocolo_gerar.py @@ -96,7 +96,11 @@ def protocolos(lst_protocolos, dic_cabecalho): tmp_data += '\t\t' + \ dic['data'] + '\n' tmp_data += '\t\t' + \ - dic['natureza'] + '\n' + dic['natureza'] + if dic['ident_processo']: + tmp_data += ' - ' + dic['ident_processo'] + '\n' + else: + tmp_data += '\n' tmp_data += '\t\n' return tmp_data diff --git a/sapl/relatorios/views.py b/sapl/relatorios/views.py index 00f33e44e..517e23229 100644 --- a/sapl/relatorios/views.py +++ b/sapl/relatorios/views.py @@ -955,10 +955,7 @@ def get_etiqueta_protocolos(prots): if p.tipo_processo == 0: dic['natureza'] = 'Administrativo' if p.tipo_processo == 1: - if dic['num_materia']: - dic['natureza'] = dic['num_materia'] - else: - dic['natureza'] = 'Legislativo' + dic['natureza'] = 'Legislativo' dic['num_documento'] = '' for documento in DocumentoAdministrativo.objects.filter( From 222c301803f9ef6d6797b17de1182c2e779faf06 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Tue, 17 Apr 2018 11:53:59 -0300 Subject: [PATCH 006/102] Release: 3.1.72 --- docker-compose.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 36df3e501..a3173316a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ sapldb: ports: - "5432:5432" sapl: - image: interlegis/sapl:3.1.71 + image: interlegis/sapl:3.1.72 restart: always environment: ADMIN_PASSWORD: interlegis diff --git a/setup.py b/setup.py index 7b3240fbf..33c6bf7e0 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ install_requires = [ ] setup( name='interlegis-sapl', - version='3.1.71', + version='3.1.72', packages=find_packages(), include_package_data=True, license='GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007', From 68bb0596018942ebc8d0114b1cd5a38c5e365ced Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Tue, 17 Apr 2018 12:10:13 -0300 Subject: [PATCH 007/102] =?UTF-8?q?HOT-FIX:=20apaga=20ordem=20dia=20e=20ex?= =?UTF-8?q?pediente=20ao=20apagar=20sess=C3=A3o=20plen=C3=A1ria?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../migrations/0021_auto_20180417_1209.py | 26 +++++++++++++++++++ sapl/sessao/models.py | 2 +- 2 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 sapl/sessao/migrations/0021_auto_20180417_1209.py diff --git a/sapl/sessao/migrations/0021_auto_20180417_1209.py b/sapl/sessao/migrations/0021_auto_20180417_1209.py new file mode 100644 index 000000000..51993cfdc --- /dev/null +++ b/sapl/sessao/migrations/0021_auto_20180417_1209.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.11 on 2018-04-17 15:09 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('sessao', '0020_auto_20180416_1424'), + ] + + operations = [ + migrations.AlterField( + model_name='expedientemateria', + name='sessao_plenaria', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sessao.SessaoPlenaria'), + ), + migrations.AlterField( + model_name='ordemdia', + name='sessao_plenaria', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sessao.SessaoPlenaria'), + ), + ] diff --git a/sapl/sessao/models.py b/sapl/sessao/models.py index c10d3e03d..13e55aea7 100644 --- a/sapl/sessao/models.py +++ b/sapl/sessao/models.py @@ -231,7 +231,7 @@ class AbstractOrdemDia(models.Model): ) sessao_plenaria = models.ForeignKey(SessaoPlenaria, - on_delete=models.PROTECT) + on_delete=models.CASCADE) materia = models.ForeignKey(MateriaLegislativa, on_delete=models.PROTECT, verbose_name=_('Matéria')) From cb2eb54ba82375548e4bb1fb768d84eb76341fc9 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Tue, 17 Apr 2018 12:13:22 -0300 Subject: [PATCH 008/102] HOT-FIX: adiciona uma mensagem mais elucidativa para AbstractOrdemDia --- sapl/sessao/models.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sapl/sessao/models.py b/sapl/sessao/models.py index 13e55aea7..4c5e9e187 100644 --- a/sapl/sessao/models.py +++ b/sapl/sessao/models.py @@ -255,7 +255,8 @@ class AbstractOrdemDia(models.Model): abstract = True def __str__(self): - return '%s - %s' % (self.numero_ordem, self.sessao_plenaria) + return 'Ordem do Dia/Expediente: %s - %s em %s' % ( + self.numero_ordem, self.materia, self.sessao_plenaria) @reversion.register() From d6e20cf5ef791c29912a719b8eca2da585bbd52a Mon Sep 17 00:00:00 2001 From: Talitha Pumar Date: Tue, 17 Apr 2018 14:13:35 -0300 Subject: [PATCH 009/102] Fix #1893 (#1894) --- .../templates/pdf_sessao_plenaria_gerar.py | 7 +++++-- sapl/relatorios/views.py | 2 ++ sapl/sessao/views.py | 4 +++- .../sessao/blocos_resumo/oradores_expediente.html | 12 +++++++----- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/sapl/relatorios/templates/pdf_sessao_plenaria_gerar.py b/sapl/relatorios/templates/pdf_sessao_plenaria_gerar.py index d9c8ed76e..1c01bfb08 100644 --- a/sapl/relatorios/templates/pdf_sessao_plenaria_gerar.py +++ b/sapl/relatorios/templates/pdf_sessao_plenaria_gerar.py @@ -215,8 +215,11 @@ def oradores_expediente(lst_oradores_expediente): tmp += '\t\t\t
\n' tmp += '\t\t\n' for orador_expediente in lst_oradores_expediente: - tmp += '\t\t' + str(orador_expediente['num_ordem']) + ' - ' + orador_expediente[ - 'nom_parlamentar'] + '/' + str(orador_expediente['sgl_partido']) + '\n' + tmp += '\t\t' + str(orador_expediente[ + 'num_ordem']) + ' - ' + orador_expediente[ + 'nom_parlamentar'] + '/' + str(orador_expediente[ + 'sgl_partido']) + ' - ' + str(orador_expediente[ + 'observacao']) + '\n' return tmp diff --git a/sapl/relatorios/views.py b/sapl/relatorios/views.py index adfdaec37..9f46450b8 100644 --- a/sapl/relatorios/views.py +++ b/sapl/relatorios/views.py @@ -617,6 +617,8 @@ def get_sessao_plenaria(sessao, casa): orador_expediente.numero_ordem) dic_oradores_expediente["nom_parlamentar"] = ( parlamentar.nome_parlamentar) + dic_oradores_expediente["observacao"] = ( + orador_expediente.observacao) partido_sigla = Filiacao.objects.filter( parlamentar=parlamentar).first() if not partido_sigla: diff --git a/sapl/sessao/views.py b/sapl/sessao/views.py index b876953fe..b256db879 100644 --- a/sapl/sessao/views.py +++ b/sapl/sessao/views.py @@ -1306,11 +1306,13 @@ class ResumoView(DetailView): sessao_plenaria_id=self.object.id).order_by('numero_ordem'): numero_ordem = orador.numero_ordem url_discurso = orador.url_discurso + observacao = orador.observacao parlamentar = Parlamentar.objects.get( id=orador.parlamentar_id) ora = {'numero_ordem': numero_ordem, 'url_discurso': url_discurso, - 'parlamentar': parlamentar + 'parlamentar': parlamentar, + 'observacao' : observacao } oradores.append(ora) diff --git a/sapl/templates/sessao/blocos_resumo/oradores_expediente.html b/sapl/templates/sessao/blocos_resumo/oradores_expediente.html index 064b7d033..7d4b77390 100644 --- a/sapl/templates/sessao/blocos_resumo/oradores_expediente.html +++ b/sapl/templates/sessao/blocos_resumo/oradores_expediente.html @@ -1,14 +1,16 @@
Oradores do Expediente
-
Parlamentar
-
Discurso
+
Parlamentar
+
Discurso
+
Observação
{% for o in oradores %} -
{{o.numero_ordem}} - {{o.parlamentar}}
-
{{o.url_discurso}}
+
{{o.numero_ordem}} - {{o.parlamentar}}
+
{{o.url_discurso}}
+
{{o.observacao}}

{% endfor %}
-
\ No newline at end of file + From 372690f56c226323ac0b99675bcbdb3ec06a7f89 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Wed, 18 Apr 2018 12:00:01 -0300 Subject: [PATCH 010/102] HOT-FIX: Fix #1872 --- sapl/sessao/forms.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/sapl/sessao/forms.py b/sapl/sessao/forms.py index f05209229..aea3c5dbb 100644 --- a/sapl/sessao/forms.py +++ b/sapl/sessao/forms.py @@ -464,14 +464,13 @@ class OradorExpedienteForm(ModelForm): def __init__(self, *args, **kwargs): super(OradorExpedienteForm, self).__init__(*args, **kwargs) - legislatura_atual = [l for l in Legislatura.objects.all() if l.atual] + legislatura_vigente = SessaoPlenaria.objects.get(pk=kwargs['initial']['id_sessao']).legislatura - if legislatura_atual: - legislatura_atual = legislatura_atual[0] + if legislatura_vigente: self.fields['parlamentar'].queryset = \ Parlamentar.objects.filter(ativo=True, - mandato__legislatura=legislatura_atual - ).order_by('nome_parlamentar') + mandato__legislatura=legislatura_vigente + ).order_by('nome_parlamentar') class Meta: model = OradorExpediente From 997f87261af8f2d28c3ee77d6c749a9ebb8736cd Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Wed, 18 Apr 2018 12:13:33 -0300 Subject: [PATCH 011/102] Fixes #1897 --- sapl/sessao/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sapl/sessao/views.py b/sapl/sessao/views.py index b256db879..9f8f8aaa2 100644 --- a/sapl/sessao/views.py +++ b/sapl/sessao/views.py @@ -2596,7 +2596,7 @@ class PesquisarSessaoPlenariaView(FilterView): 'tipo', 'sessao_legislativa', 'legislatura') qs = qs.distinct().order_by( - '-legislatura__numero', '-data_inicio', '-numero') + '-legislatura__numero', '-data_inicio', '-hora_inicio') kwargs.update({ 'queryset': qs, From 7937b5480c3f099319ea92e2fd4cf516e2c262bd Mon Sep 17 00:00:00 2001 From: Talitha Pumar Date: Wed, 18 Apr 2018 12:15:40 -0300 Subject: [PATCH 012/102] Fix #1895 (#1901) --- sapl/sessao/forms.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/sapl/sessao/forms.py b/sapl/sessao/forms.py index aea3c5dbb..219e5b69b 100644 --- a/sapl/sessao/forms.py +++ b/sapl/sessao/forms.py @@ -472,6 +472,25 @@ class OradorExpedienteForm(ModelForm): mandato__legislatura=legislatura_vigente ).order_by('nome_parlamentar') + def clean(self): + super(OradorExpedienteForm, self).clean() + cleaned_data = self.cleaned_data + + if not self.is_valid(): + return self.cleaned_data + + sessao_id = self.initial['id_sessao'] + ordem = OradorExpediente.objects.filter( + sessao_plenaria_id=sessao_id, + numero_ordem=cleaned_data['numero_ordem'] + ).exists() + if ordem: + raise ValidationError(_( + 'Já existe orador nesta posição da ordem de pronunciamento')) + + return self.cleaned_data + + class Meta: model = OradorExpediente exclude = ['sessao_plenaria'] From 825e27c1dd44a041da729920a65a8ffde839d20e Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 5 Apr 2018 17:42:59 -0300 Subject: [PATCH 013/102] =?UTF-8?q?Adiciona=20mais=20propaga=C3=A7=C3=B5es?= =?UTF-8?q?=20de=20dele=C3=A7=C3=B5es=20pr=C3=A9-migra=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_dados.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 9b2846fa7..6b10e51ee 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -201,7 +201,7 @@ class ForeignKeyFaltando(ObjectDoesNotExist): campo = campos_novos_para_antigos[self.field] _, tabela, campos_pk = get_estrutura_legado(self.field.model) pk = {c: getattr(self.old, c) for c in campos_pk} - sql = 'select * from {} where {}'.format( + sql = 'select * from {} where {};'.format( tabela, ' and '.join(['{} = {}'.format(k, v) for k, v in pk.items()])) return OrderedDict((('campo', campo), @@ -494,6 +494,8 @@ PROPAGACOES_DE_EXCLUSAO = [ ('parlamentar', 'dependente', 'cod_parlamentar'), ('parlamentar', 'filiacao', 'cod_parlamentar'), ('parlamentar', 'mandato', 'cod_parlamentar'), + ('parlamentar', 'composicao_mesa', 'cod_parlamentar'), + ('parlamentar', 'composicao_comissao', 'cod_parlamentar'), # comissao ('comissao', 'composicao_comissao', 'cod_comissao'), @@ -518,6 +520,11 @@ PROPAGACOES_DE_EXCLUSAO = [ ('materia_legislativa', 'anexada', 'cod_materia_principal'), ('materia_legislativa', 'anexada', 'cod_materia_anexada'), ('materia_legislativa', 'documento_acessorio', 'cod_materia'), + ('materia_legislativa', 'numeracao', 'cod_materia'), + + # norma + ('norma_juridica', 'vinculo_norma_juridica', 'cod_norma_referente'), + ('norma_juridica', 'vinculo_norma_juridica', 'cod_norma_referida'), # documento administrativo ('documento_administrativo', 'tramitacao_administrativo', 'cod_documento'), @@ -800,7 +807,7 @@ def migrar_dados(interativo=True): arq_ocorrencias = dir_ocorrencias.child( nome_banco_legado + '.yaml') with open(arq_ocorrencias, 'w') as arq: - dump = yaml.dump(dict(ocorrencias), allow_unicode=True) + dump = yaml.dump(dict(ocorrencias), allow_unicode=True, width=1000) arq.write(dump.replace('\n- ', '\n\n- ')) info('Ocorrências salvas em\n {}'.format(arq_ocorrencias)) From 09085b88a2780fc76ff9194d1298792812a65740 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 6 Apr 2018 12:55:08 -0300 Subject: [PATCH 014/102] Corrige rehash de senhas importadas do zope --- sapl/hashers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sapl/hashers.py b/sapl/hashers.py index e80642def..3e9cf6c5f 100644 --- a/sapl/hashers.py +++ b/sapl/hashers.py @@ -46,11 +46,12 @@ ZOPE_SHA1_PREFIX = '{SSHA}' def zope_encoded_password_to_django(encoded): "Migra um hash de senha do zope para uso com o ZopeSHA1PasswordHasher" - if encoded.startswith(ZOPE_SHA1_PREFIX): + if encoded and encoded.startswith(ZOPE_SHA1_PREFIX): data = encoded[len(ZOPE_SHA1_PREFIX):] salt = get_salt_from_zope_sha1(data) hasher = ZopeSHA1PasswordHasher() return super(ZopeSHA1PasswordHasher, hasher).encode(data, salt) else: # assume it's a plain password and use the default hashing + # a None password blocks login, forcing a password reset return make_password(encoded) From 1cf2f35def5025bcf9b3bd3ee9e1c184a5e317c5 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 6 Apr 2018 19:08:54 -0300 Subject: [PATCH 015/102] =?UTF-8?q?Corrige=20caminhos=20de=20arquivos=20na?= =?UTF-8?q?=20migra=C3=A7=C3=A3o=20de=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao.py | 6 -- sapl/legacy/migracao_documentos.py | 134 ++++++++--------------------- 2 files changed, 36 insertions(+), 104 deletions(-) diff --git a/sapl/legacy/migracao.py b/sapl/legacy/migracao.py index 2690e9a53..be8830aa8 100644 --- a/sapl/legacy/migracao.py +++ b/sapl/legacy/migracao.py @@ -14,12 +14,6 @@ def migrar(interativo=False): migrar_documentos() -# fonte: https://stackoverflow.com/a/17081026/1877490 -def make_tarfile(output_filename, source_dir): - with tarfile.open(output_filename, "w:gz") as tar: - tar.add(source_dir, arcname=os.path.basename(source_dir)) - - def gerar_pacote(): banco = settings.DATABASES['legacy']['NAME'] diff --git a/sapl/legacy/migracao_documentos.py b/sapl/legacy/migracao_documentos.py index 9ae46ef5d..a94b9160a 100644 --- a/sapl/legacy/migracao_documentos.py +++ b/sapl/legacy/migracao_documentos.py @@ -1,12 +1,12 @@ -import mimetypes import os import re from glob import glob import yaml +from django.db import transaction from sapl.base.models import CasaLegislativa -from sapl.legacy.migracao_dados import exec_legado, warn +from sapl.legacy.migracao_dados import exec_legado from sapl.materia.models import (DocumentoAcessorio, MateriaLegislativa, Proposicao) from sapl.norma.models import NormaJuridica @@ -16,88 +16,26 @@ from sapl.protocoloadm.models import (DocumentoAcessorioAdministrativo, from sapl.sessao.models import SessaoPlenaria from sapl.settings import MEDIA_ROOT - # MIGRAÇÃO DE DOCUMENTOS ################################################### -def get_ano(obj): - return [obj.ano] - - -def ___(obj): - return [] - - DOCS = { - CasaLegislativa: [ - ('logotipo', - 'props_sapl/{}.*', - 'public/casa/logotipo/', - ___) - ], - Parlamentar: [ - ('fotografia', - 'parlamentar/fotos/{}_foto_parlamentar', - 'public/parlamentar/{0}/', - ___) - ], - MateriaLegislativa: [ - ('texto_original', - 'materia/{}_texto_integral', - 'public/materialegislativa/{1}/{0}/', - get_ano) - ], - DocumentoAcessorio: [ - ('arquivo', - 'materia/{}', - 'public/documentoacessorio/{1}/{0}/', - lambda obj: [obj.materia.ano]) - ], - NormaJuridica: [ - ('texto_integral', - 'norma_juridica/{}_texto_integral', - 'public/normajuridica/{1}/{0}/', - get_ano) - ], - SessaoPlenaria: [ - ('upload_pauta', - 'pauta_sessao/{}_pauta_sessao', - 'public/sessaoplenaria/{0}/pauta/', - ___), - ('upload_ata', - 'ata_sessao/{}_ata_sessao', - 'public/sessaoplenaria/{0}/ata/', - ___), - ('upload_anexo', - 'anexo_sessao/{}_texto_anexado', - 'public/sessaoplenaria/{0}/anexo/', - ___) - ], - Proposicao: [ - ('texto_original', - 'proposicao/{}', - 'private/proposicao/{0}/', - get_ano) - ], - DocumentoAdministrativo: [ - ('texto_integral', - 'administrativo/{}_texto_integral', - 'private/documentoadministrativo/{0}/', - get_ano) - ], - DocumentoAcessorioAdministrativo: [ - ('arquivo', - 'administrativo/{}', - 'private/documentoacessorioadministrativo/{0}/', - ___) - ], + CasaLegislativa: [('logotipo', 'props_sapl/{}.*')], + Parlamentar: [('fotografia', 'parlamentar/fotos/{}_foto_parlamentar')], + MateriaLegislativa: [('texto_original', 'materia/{}_texto_integral')], + DocumentoAcessorio: [('arquivo', 'materia/{}')], + NormaJuridica: [('texto_integral', 'norma_juridica/{}_texto_integral')], + SessaoPlenaria: [('upload_pauta', 'pauta_sessao/{}_pauta_sessao'), + ('upload_ata', 'ata_sessao/{}_ata_sessao'), + ('upload_anexo', 'anexo_sessao/{}_texto_anexado')], + Proposicao: [('texto_original', 'proposicao/{}')], + DocumentoAdministrativo: [('texto_integral', + 'administrativo/{}_texto_integral')], + DocumentoAcessorioAdministrativo: [('arquivo', 'administrativo/{}')], } -DOCS = {model: [(campo, - os.path.join('sapl_documentos', origem), - os.path.join('sapl', destino), - get_extra_args) - for campo, origem, destino, get_extra_args in campos] +DOCS = {model: [(campo, os.path.join('sapl_documentos', origem)) + for campo, origem, in campos] for model, campos in DOCS.items()} @@ -141,11 +79,13 @@ def migrar_propriedades_da_casa(): [(casa.municipio, casa.uf)] = exec_legado(sql_localidade) print('.... Migrando logotipo da casa ....') - [(_, origem, destino, __)] = DOCS[CasaLegislativa] + [(campo, origem)] = DOCS[CasaLegislativa] # a extensão do logo pode ter sido ajustada pelo tipo real do arquivo id_logo = os.path.splitext(propriedades['id_logo'])[0] [origem] = glob(em_media(origem.format(id_logo))) - destino = os.path.join(destino, os.path.basename(origem)) + destino = os.path.join( + CasaLegislativa._meta.get_field(campo).upload_to, + os.path.basename(origem)) mover_documento(origem, destino) casa.logotipo = destino casa.save() @@ -153,36 +93,36 @@ def migrar_propriedades_da_casa(): def migrar_docs_por_ids(model): - for campo, base_origem, base_destino, get_extra_args in DOCS[model]: + for campo, base_origem in DOCS[model]: print('#### Migrando {} de {} ####'.format(campo, model.__name__)) dir_origem, nome_origem = os.path.split(em_media(base_origem)) nome_origem = nome_origem.format('(\d+)') pat = re.compile('^{}\.\w+$'.format(nome_origem)) - if not os.path.isdir(dir_origem): print(' >>> O diretório {} não existe! Abortado.'.format( dir_origem)) continue - for arq in os.listdir(dir_origem): - match = pat.match(arq) - if match: + matches = [pat.match(arq) for arq in os.listdir(dir_origem)] + ids_origens = [(int(m.group(1)), + os.path.join(dir_origem, m.group(0))) + for m in matches if m] + objetos = {obj.id: obj for obj in model.objects.all()} + upload_to = model._meta.get_field(campo).upload_to + + with transaction.atomic(): + for id, origem in ids_origens: # associa documento ao objeto - origem = os.path.join(dir_origem, match.group(0)) - id = match.group(1) - try: - obj = model.objects.get(pk=id) - except model.DoesNotExist: - msg = ' {} (pk={}) não encontrado para documento em [{}]' - print(msg.format(model.__name__, id, origem)) - else: - destino = os.path.join( - base_destino.format(id, *get_extra_args(obj)), - os.path.basename(origem)) + obj = objetos.get(id) + if obj: + destino = upload_to(obj, os.path.basename(origem)) mover_documento(origem, destino) setattr(obj, campo, destino) obj.save() + else: + msg = ' {} (pk={}) não encontrado para documento em [{}]' + print(msg.format(model.__name__, id, origem)) def migrar_documentos(): @@ -215,5 +155,3 @@ def migrar_documentos(): print('\n#### Encerrado ####\n\n' '{} documentos sobraram sem ser migrados!!!'.format( len(sobrando))) - for doc in sobrando: - print(' {}'. format(doc)) From 500574a82044fcf410178d874c6206837e50e02c Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Mon, 9 Apr 2018 14:42:35 -0300 Subject: [PATCH 016/102] =?UTF-8?q?Usa=20pretty-yaml=20p=20registro=20de?= =?UTF-8?q?=20ocorr=C3=AAncias=20de=20migra=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- requirements/migration-requirements.txt | 1 + sapl/legacy/migracao_dados.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/requirements/migration-requirements.txt b/requirements/migration-requirements.txt index d5793dad8..fbe9543e2 100644 --- a/requirements/migration-requirements.txt +++ b/requirements/migration-requirements.txt @@ -1,2 +1,3 @@ -r dev-requirements.txt mysqlclient==1.3.12 +pyaml diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 6b10e51ee..dd6e4f8d1 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -8,6 +8,7 @@ from operator import xor from subprocess import PIPE, call import pkg_resources +import pyaml import pytz import reversion import yaml @@ -807,8 +808,7 @@ def migrar_dados(interativo=True): arq_ocorrencias = dir_ocorrencias.child( nome_banco_legado + '.yaml') with open(arq_ocorrencias, 'w') as arq: - dump = yaml.dump(dict(ocorrencias), allow_unicode=True, width=1000) - arq.write(dump.replace('\n- ', '\n\n- ')) + pyaml.dump(ocorrencias, arq, vspacing=1) info('Ocorrências salvas em\n {}'.format(arq_ocorrencias)) # recria tipos de autor padrão que não foram criados pela migração From 70bc3148885f1c98fc8ec4beda7505b312733625 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Mon, 9 Apr 2018 14:57:02 -0300 Subject: [PATCH 017/102] Ignora app legacy no shell_plus --- sapl/legacy_migration_settings.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sapl/legacy_migration_settings.py b/sapl/legacy_migration_settings.py index 7d911bf7e..0d8844147 100644 --- a/sapl/legacy_migration_settings.py +++ b/sapl/legacy_migration_settings.py @@ -33,3 +33,5 @@ DEBUG = True # delisga indexação fulltext em tempo real HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.BaseSignalProcessor' + +SHELL_PLUS_DONT_LOAD = ['legacy'] From 5ef440685eda05e1052acd9d585b2b337fc76977 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Mon, 9 Apr 2018 18:05:11 -0300 Subject: [PATCH 018/102] Adiciona gravacao de marco da migracao --- sapl/legacy/migracao_dados.py | 44 ++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index dd6e4f8d1..ceaea8633 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -1,3 +1,4 @@ +import datetime import re import traceback from collections import OrderedDict, defaultdict, namedtuple @@ -19,6 +20,7 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist from django.db import connections, transaction from django.db.models import Max, Q +from pyaml import UnsafePrettyYAMLDumper from unipath import Path from sapl.base.models import AppConfig as AppConf @@ -823,7 +825,7 @@ def move_para_depois_de(lista, movido, referencias): return lista -def migrar_todos_os_models(): +def get_models_a_migrar(): models = [model for app in appconfs for model in app.models.values() if model in field_renames] # Devido à referência TipoProposicao.tipo_conteudo_related @@ -836,7 +838,11 @@ def migrar_todos_os_models(): move_para_depois_de(models, Proposicao, [MateriaLegislativa, DocumentoAdministrativo]) - for model in models: + return models + + +def migrar_todos_os_models(): + for model in get_models_a_migrar(): migrar_model(model) @@ -1233,4 +1239,36 @@ AJUSTE_DEPOIS_SALVAR = { NormaJuridica: adjust_normajuridica_depois_salvar, } -# CHECKS #################################################################### + +# MARCO ###################################################################### + +TIME_FORMAT = '%H:%M:%S' + + +def time_representer(dumper, data): + return dumper.represent_scalar('!time', data.strftime(TIME_FORMAT)) +UnsafePrettyYAMLDumper.add_representer(datetime.time, time_representer) + + +def time_constructor(loader, node): + value = loader.construct_scalar(node) + return datetime.datetime.strptime(value, TIME_FORMAT).time() +yaml.add_constructor(u'!time', time_constructor) + + +DIR_MARCO = Path(DIR_DADOS_MIGRACAO, 'marcos', nome_banco_legado) + + +def grava_marco_base(): + user_model = get_user_model() + models = get_models_a_migrar() + [ + Composicao, user_model, Group, ContentType] + for model in models: + info('Gravando marco de [{}]'.format(model.__name__)) + dir_model = Path( + DIR_MARCO, 'dados', model._meta.app_label, model.__name__) + dir_model.mkdir(parents=True) + for data in model.objects.all().values(): + nome_arq = Path(dir_model, data['id']) + with open(nome_arq, 'w') as arq: + pyaml.dump(data, arq) From acc9cbe2626cd2469d94fd80531e2af310e369ac Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Mon, 9 Apr 2018 18:12:24 -0300 Subject: [PATCH 019/102] =?UTF-8?q?Move=20migrate=20p=20script=20de=20recr?= =?UTF-8?q?ia=C3=A7=C3=A3o=20de=20bancos=20postgres?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/migra_um_db.sh | 5 ----- sapl/legacy/scripts/recria_um_db_postgres.sh | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sapl/legacy/scripts/migra_um_db.sh b/sapl/legacy/scripts/migra_um_db.sh index ed4db9662..b5b336cd2 100755 --- a/sapl/legacy/scripts/migra_um_db.sh +++ b/sapl/legacy/scripts/migra_um_db.sh @@ -32,11 +32,6 @@ if [ $# -ge 2 ]; then echo "O banco legado foi restaurado" |& tee -a $LOG echo >> $LOG - echo "--- DJANGO MIGRATE ---" | tee -a $LOG - echo >> $LOG - DATABASE_NAME=$1 ./manage.py migrate --settings sapl.legacy_migration_settings - echo >> $LOG - echo "--- MIGRACAO ---" | tee -a $LOG echo >> $LOG DATABASE_NAME=$1 ./manage.py migracao_25_31 --force --dados --settings sapl.legacy_migration_settings 2>&1 | tee -a $LOG diff --git a/sapl/legacy/scripts/recria_um_db_postgres.sh b/sapl/legacy/scripts/recria_um_db_postgres.sh index 3ff66e8f3..98defaaa6 100755 --- a/sapl/legacy/scripts/recria_um_db_postgres.sh +++ b/sapl/legacy/scripts/recria_um_db_postgres.sh @@ -4,3 +4,8 @@ echo "Database $1" sudo -u postgres psql -c "drop DATABASE if exists $1" sudo -u postgres psql -c "CREATE DATABASE $1 WITH OWNER = sapl ENCODING = 'UTF8' TABLESPACE = pg_default LC_COLLATE = 'pt_BR.UTF-8' LC_CTYPE = 'pt_BR.UTF-8' CONNECTION LIMIT = -1 TEMPLATE template0;" + + +echo "--- DJANGO MIGRATE ---" | tee -a $LOG +DATABASE_NAME=$1 ./manage.py migrate --settings sapl.legacy_migration_settings + From 444e9dacad43b672975bc75e392dc6f0366b451c Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Mon, 9 Apr 2018 18:27:31 -0300 Subject: [PATCH 020/102] Atualiza assinatura de LegacyRouter.allow_migrate --- sapl/legacy/router.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sapl/legacy/router.py b/sapl/legacy/router.py index 68b235059..1aa51a46b 100644 --- a/sapl/legacy/router.py +++ b/sapl/legacy/router.py @@ -16,7 +16,7 @@ class LegacyRouter: return True return None - def allow_migrate(self, db, model): - if model._meta.app_label == 'legacy': + def allow_migrate(self, db, app_label, model_name=None, **hints): + if app_label == 'legacy': return False return None From 27b4096f46c6b5df9be66d0e39eabbdbfef38777 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 10 Apr 2018 13:15:02 -0300 Subject: [PATCH 021/102] Corrige ajuste de tipo de proposicao antes de salvar --- sapl/legacy/migracao_dados.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index ceaea8633..f0bf710f0 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -146,6 +146,15 @@ CAMPOS_VIRTUAIS_PROPOSICAO = { for campo_virtual in CAMPOS_VIRTUAIS_PROPOSICAO.values(): campos_novos_para_antigos[campo_virtual] = 'cod_mat_ou_doc' + +CAMPOS_VIRTUAIS_TIPO_PROPOSICAO = { + 'M': CampoVirtual(TipoProposicao, TipoMateriaLegislativa), + 'D': CampoVirtual(TipoProposicao, TipoDocumento) +} +for campo_virtual in CAMPOS_VIRTUAIS_TIPO_PROPOSICAO.values(): + campos_novos_para_antigos[campo_virtual] = 'tip_mat_ou_doc' + + # campos virtuais de Autor para funcionar com get_fk_related CAMPOS_VIRTUAIS_AUTOR = {related: CampoVirtual(Autor, related) for related in (Parlamentar, Comissao, Partido)} @@ -1074,22 +1083,16 @@ def adjust_tipoafastamento(new, old): new.indicador = 'F' -TIPO_MATERIA_OU_TIPO_DOCUMENTO = {'M': TipoMateriaLegislativa, - 'D': TipoDocumento} +def set_generic_fk(new, campo_virtual, old): + new.content_type = content_types[campo_virtual.related_model] + new.object_id = get_fk_related(campo_virtual, old) def adjust_tipoproposicao(new, old): "Aponta para o tipo relacionado de matéria ou documento" - value = old.tip_mat_ou_doc - model_tipo = TIPO_MATERIA_OU_TIPO_DOCUMENTO[old.ind_mat_ou_doc] - tipo = model_tipo.objects.filter(pk=value) - if tipo: - new.tipo_conteudo_related = tipo[0] - else: - raise ForeignKeyFaltando( - field=TipoProposicao.tipo_conteudo_related, - value=(model_tipo.__name__, value), - label={'ind_mat_ou_doc': old.ind_mat_ou_doc}) + if old.tip_mat_ou_doc: + campo_virtual = CAMPOS_VIRTUAIS_TIPO_PROPOSICAO[old.ind_mat_ou_doc] + set_generic_fk(new, campo_virtual, old) def adjust_proposicao_antes_salvar(new, old): @@ -1098,8 +1101,7 @@ def adjust_proposicao_antes_salvar(new, old): if old.cod_mat_ou_doc: tipo_mat_ou_doc = type(new.tipo.tipo_conteudo_related) campo_virtual = CAMPOS_VIRTUAIS_PROPOSICAO[tipo_mat_ou_doc] - new.content_type = content_types[campo_virtual.related_model] - new.object_id = get_fk_related(campo_virtual, old) + set_generic_fk(new, campo_virtual, old) def adjust_statustramitacao(new, old): From dd8ac9203b728b20e12c0a3300ef97ea6f99b76f Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 10 Apr 2018 13:15:41 -0300 Subject: [PATCH 022/102] =?UTF-8?q?Executa=20ajustes=20pr=C3=A9-migra?= =?UTF-8?q?=C3=A7=C3=A3o=20se=20existirem?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_dados.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index f0bf710f0..9363e7464 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -782,6 +782,12 @@ def populate_renamed_fields(new, old): def migrar_dados(interativo=True): + # executa ajustes pré-migração, se existirem + arq_ajustes_pre_migracao = DIR_DADOS_MIGRACAO.child( + 'ajustes_pre_migracao', '{}.sql'.format(sigla_casa)) + if arq_ajustes_pre_migracao.exists(): + exec_legado(arq_ajustes_pre_migracao.read_file()) + uniformiza_banco() # excluindo database antigo. From d7af0598eebaa8a6678f153ff1a5566738cccb6f Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 10 Apr 2018 14:25:21 -0300 Subject: [PATCH 023/102] Garante coluna no legado materia_legislativa.txt_resultado --- sapl/legacy/migracao_dados.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 9363e7464..52adfd2f1 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -567,6 +567,9 @@ def uniformiza_banco(): garante_coluna_no_legado('tipo_materia_legislativa', 'quorum_minimo_votacao int(11) NULL') + garante_coluna_no_legado('materia_legislativa', + 'txt_resultado TEXT NULL') + # Cria campos cod_presenca_sessao (sendo a nova PK da tabela) # e dat_sessao em sessao_plenaria_presenca if not existe_coluna_no_legado('sessao_plenaria_presenca', From 5370641ceb1cf5809ba7fa3df41079c9e7be0d4d Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 10 Apr 2018 15:06:39 -0300 Subject: [PATCH 024/102] Aumenta tamanho de numero_origem_externa em MateriaLegislativa MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Algumas bases do sapl 2.5 têm esse campo com tamanho 9 --- sapl/legacy/models.py | 2 +- .../migrations/0028_auto_20180418_1629.py | 20 +++++++++++++++++++ sapl/materia/models.py | 2 +- 3 files changed, 22 insertions(+), 2 deletions(-) create mode 100644 sapl/materia/migrations/0028_auto_20180418_1629.py diff --git a/sapl/legacy/models.py b/sapl/legacy/models.py index 4c459c9dd..7d341624d 100644 --- a/sapl/legacy/models.py +++ b/sapl/legacy/models.py @@ -433,7 +433,7 @@ class MateriaLegislativa(models.Model): cod_regime_tramitacao = models.IntegerField() dat_publicacao = models.DateField(blank=True, null=True) tip_origem_externa = models.IntegerField(blank=True, null=True) - num_origem_externa = models.CharField(max_length=5, blank=True, null=True) + num_origem_externa = models.CharField(max_length=10, blank=True, null=True) ano_origem_externa = models.SmallIntegerField(blank=True, null=True) dat_origem_externa = models.DateField(blank=True, null=True) cod_local_origem_externa = models.IntegerField(blank=True, null=True) diff --git a/sapl/materia/migrations/0028_auto_20180418_1629.py b/sapl/materia/migrations/0028_auto_20180418_1629.py new file mode 100644 index 000000000..c082ce1e6 --- /dev/null +++ b/sapl/materia/migrations/0028_auto_20180418_1629.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.13 on 2018-04-18 19:29 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('materia', '0027_auto_20180409_1443'), + ] + + operations = [ + migrations.AlterField( + model_name='materialegislativa', + name='numero_origem_externa', + field=models.CharField(blank=True, max_length=10, verbose_name='Número'), + ), + ] diff --git a/sapl/materia/models.py b/sapl/materia/models.py index ce4690c39..a1571072a 100644 --- a/sapl/materia/models.py +++ b/sapl/materia/models.py @@ -167,7 +167,7 @@ class MateriaLegislativa(models.Model): on_delete=models.PROTECT, verbose_name=_('Tipo')) numero_origem_externa = models.CharField( - max_length=5, blank=True, verbose_name=_('Número')) + max_length=10, blank=True, verbose_name=_('Número')) ano_origem_externa = models.PositiveSmallIntegerField( blank=True, null=True, verbose_name=_('Ano'), choices=RANGE_ANOS) data_origem_externa = models.DateField( From 0e8ff2e9871d8c855b53a0478c05cf7b3a0bc6a8 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Wed, 11 Apr 2018 15:43:52 -0300 Subject: [PATCH 025/102] Exporta docs do zope como repo git --- .../scripts/exporta_zope/exporta_zope.py | 29 ++++++++++++++++--- .../scripts/exporta_zope/requirements.txt | 2 ++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 11ff1ecbb..b3a4622b2 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -12,10 +12,12 @@ from collections import defaultdict from contextlib import contextmanager from functools import partial +import git import magic import yaml import ZODB.DB import ZODB.FileStorage +from unipath import Path from ZODB.broken import Broken EXTENSOES = { @@ -270,7 +272,7 @@ def dump_usuarios(sapl, path): save_as_yaml(path, 'usuarios.yaml', users) -def dump_sapl(data_fs_path, destino='../../../../media'): +def _dump_sapl(data_fs_path, destino='../../../../media'): app, close_db = get_app(data_fs_path) try: sapl = find_sapl(app) @@ -288,9 +290,28 @@ def dump_sapl(data_fs_path, destino='../../../../media'): close_db() +DIR_DADOS_MIGRACAO = Path('~/migracao_sapl/').expand() + + +def dump_sapl(sigla): + data_fs_path = DIR_DADOS_MIGRACAO.child('datafs', + 'Data_cm_{}.fs'.format(sigla)) + nome_banco_legado = 'sapl_cm_{}'.format(sigla) + destino = DIR_DADOS_MIGRACAO.child('repos', nome_banco_legado) + destino.mkdir(parents=True) + repo = git.Repo.init(destino) + assert not repo.index.diff(None) # o repo não tem mudanças pendentes + _dump_sapl(data_fs_path, destino) + # grava mundaças + repo.git.add(A=True) + if 'master' not in repo.heads or repo.index.diff('HEAD'): + # se de fato existe mudança + repo.index.commit('Exporta documentos do zope') + + if __name__ == "__main__": if len(sys.argv) == 2: - data_fs_path = sys.argv[1] - dump_sapl(data_fs_path) + sigla = sys.argv[1] + dump_sapl(sigla) else: - print('Uso: python exporta_zope ') + print('Uso: python exporta_zope ') diff --git a/sapl/legacy/scripts/exporta_zope/requirements.txt b/sapl/legacy/scripts/exporta_zope/requirements.txt index 4794267ae..421005181 100644 --- a/sapl/legacy/scripts/exporta_zope/requirements.txt +++ b/sapl/legacy/scripts/exporta_zope/requirements.txt @@ -1,3 +1,5 @@ # ZODB version 3.7.4 PyYAML==3.12 ZODB==5.3.0 +Unipath==1.1 +GitPython==2.1.9 From a9b8d4fbd42d283045b479ed0e2c28d3983bc0b8 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Wed, 11 Apr 2018 16:47:45 -0300 Subject: [PATCH 026/102] =?UTF-8?q?Adiciona=20tipo=20mp4=20=C3=A0=20export?= =?UTF-8?q?a=C3=A7=C3=A3o=20de=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index b3a4622b2..18a5bc3ef 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -40,6 +40,7 @@ EXTENSOES = { 'image/tiff': '.tiff', 'application/tiff': '.tiff', 'audio/x-wav': '.wav', + 'video/mp4': '.mp4', # TODO rever... 'text/richtext': '.rtf', @@ -273,6 +274,7 @@ def dump_usuarios(sapl, path): def _dump_sapl(data_fs_path, destino='../../../../media'): + assert Path(data_fs_path).exists() app, close_db = get_app(data_fs_path) try: sapl = find_sapl(app) @@ -296,6 +298,7 @@ DIR_DADOS_MIGRACAO = Path('~/migracao_sapl/').expand() def dump_sapl(sigla): data_fs_path = DIR_DADOS_MIGRACAO.child('datafs', 'Data_cm_{}.fs'.format(sigla)) + assert data_fs_path.exists() nome_banco_legado = 'sapl_cm_{}'.format(sigla) destino = DIR_DADOS_MIGRACAO.child('repos', nome_banco_legado) destino.mkdir(parents=True) From a1a5dda83b7b73ac75ca9e8a9c67f37f2999178e Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 12 Apr 2018 15:23:12 -0300 Subject: [PATCH 027/102] Grava nomes dos docs exportados --- .../scripts/exporta_zope/exporta_zope.py | 22 ++++++++++++++++++- .../scripts/exporta_zope/requirements.txt | 7 +++--- 2 files changed, 25 insertions(+), 4 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 18a5bc3ef..19186a170 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -14,6 +14,7 @@ from functools import partial import git import magic +import pyaml import yaml import ZODB.DB import ZODB.FileStorage @@ -78,6 +79,9 @@ def guess_extension(caminho): raise Exception(msg, e) +nomes_arquivos_exportados = {} + + def dump_file(doc, path): name = doc['__name__'] fullname = os.path.join(path, name) @@ -112,6 +116,7 @@ def dump_file(doc, path): # trocamos a extensão pela adivinhada final_name = base + extension os.rename(fullname, final_name) + nomes_arquivos_exportados[fullname] = final_name print(final_name) return name @@ -273,9 +278,23 @@ def dump_usuarios(sapl, path): save_as_yaml(path, 'usuarios.yaml', users) +def grava_nomes_arquivos_exportados(destino): + """Grava nomes dos arquivos exportados (originais -> definitivos) + """ + destino = Path(destino) + + def rel(caminho): + return str(destino.rel_path_to(caminho)) + + with open(destino.child('arquivos_exportados.yaml'), 'w') as arq: + nomes = {rel(k): rel(v) for k, v in nomes_arquivos_exportados.items()} + pyaml.dump(nomes, arq) + + def _dump_sapl(data_fs_path, destino='../../../../media'): assert Path(data_fs_path).exists() app, close_db = get_app(data_fs_path) + nomes_arquivos_exportados.clear() # apaga nomes dos arquivos migrados try: sapl = find_sapl(app) # extrai folhas XSLT @@ -288,6 +307,7 @@ def _dump_sapl(data_fs_path, destino='../../../../media'): with logando_nao_identificados(): dump_folder(docs, destino) dump_propriedades(docs, destino) + grava_nomes_arquivos_exportados(destino) finally: close_db() @@ -298,7 +318,7 @@ DIR_DADOS_MIGRACAO = Path('~/migracao_sapl/').expand() def dump_sapl(sigla): data_fs_path = DIR_DADOS_MIGRACAO.child('datafs', 'Data_cm_{}.fs'.format(sigla)) - assert data_fs_path.exists() + assert data_fs_path.exists(), 'Origem não existe: {}'.format(data_fs_path) nome_banco_legado = 'sapl_cm_{}'.format(sigla) destino = DIR_DADOS_MIGRACAO.child('repos', nome_banco_legado) destino.mkdir(parents=True) diff --git a/sapl/legacy/scripts/exporta_zope/requirements.txt b/sapl/legacy/scripts/exporta_zope/requirements.txt index 421005181..c7aa86b4f 100644 --- a/sapl/legacy/scripts/exporta_zope/requirements.txt +++ b/sapl/legacy/scripts/exporta_zope/requirements.txt @@ -1,5 +1,6 @@ # ZODB version 3.7.4 -PyYAML==3.12 ZODB==5.3.0 -Unipath==1.1 -GitPython==2.1.9 +PyYAML +Unipath +GitPython +pyaml From d7e61137e0d72fe3d430369c2ffafa5b043b7c29 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 12 Apr 2018 19:31:48 -0300 Subject: [PATCH 028/102] =?UTF-8?q?Adiciona=20mais=20tipos=20de=20arquivo?= =?UTF-8?q?=20=C3=A0=20exporta=C3=A7=C3=A3o=20de=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 19186a170..3e7f20613 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -42,6 +42,8 @@ EXTENSOES = { 'application/tiff': '.tiff', 'audio/x-wav': '.wav', 'video/mp4': '.mp4', + 'image/x-icon': '.ico', + 'application/vnd.oasis.opendocument.text-template': '.ott', # TODO rever... 'text/richtext': '.rtf', From 1b26489baa191a59ef20f33401bfb17a88acf5ac Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 13 Apr 2018 10:55:28 -0300 Subject: [PATCH 029/102] =?UTF-8?q?Garante=20grava=C3=A7=C3=A3o=20de=20nom?= =?UTF-8?q?es=20de=20arqs=20exportados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 3e7f20613..e27ea974d 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -309,9 +309,9 @@ def _dump_sapl(data_fs_path, destino='../../../../media'): with logando_nao_identificados(): dump_folder(docs, destino) dump_propriedades(docs, destino) - grava_nomes_arquivos_exportados(destino) finally: close_db() + grava_nomes_arquivos_exportados(destino) DIR_DADOS_MIGRACAO = Path('~/migracao_sapl/').expand() From d5bcea3ed9cfb588b24d19c3ef3d92e6433fc0e9 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Mon, 16 Apr 2018 16:17:26 -0300 Subject: [PATCH 030/102] Usa git annex --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index e27ea974d..ffcbc2d8c 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -317,6 +317,10 @@ def _dump_sapl(data_fs_path, destino='../../../../media'): DIR_DADOS_MIGRACAO = Path('~/migracao_sapl/').expand() +def repo_execute(repo, cmd): + return repo.git.execute(cmd.split()) + + def dump_sapl(sigla): data_fs_path = DIR_DADOS_MIGRACAO.child('datafs', 'Data_cm_{}.fs'.format(sigla)) @@ -325,9 +329,10 @@ def dump_sapl(sigla): destino = DIR_DADOS_MIGRACAO.child('repos', nome_banco_legado) destino.mkdir(parents=True) repo = git.Repo.init(destino) - assert not repo.index.diff(None) # o repo não tem mudanças pendentes _dump_sapl(data_fs_path, destino) # grava mundaças + repo_execute(repo, 'git annex init') + repo_execute(repo, 'git annex add sapl_documentos') repo.git.add(A=True) if 'master' not in repo.heads or repo.index.diff('HEAD'): # se de fato existe mudança From 87c7e82fd2ac434ea0f4031538cf45d4114770d9 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Wed, 18 Apr 2018 13:52:34 -0300 Subject: [PATCH 031/102] Salva apenas arquivos diferentes para o annex --- .../scripts/exporta_zope/exporta_zope.py | 150 +++++++++--------- 1 file changed, 79 insertions(+), 71 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index ffcbc2d8c..2c687bc07 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -5,6 +5,8 @@ # Esse script precisa rodar em python 2 # e depende apenas do descrito no arquivo requiments.txt +import cStringIO +import hashlib import mimetypes import os import sys @@ -16,9 +18,10 @@ import git import magic import pyaml import yaml +from unipath import Path + import ZODB.DB import ZODB.FileStorage -from unipath import Path from ZODB.broken import Broken EXTENSOES = { @@ -62,29 +65,25 @@ def br(obj): return obj -def guess_extension(caminho): - mime = magic.from_file(caminho, mime=True) +def guess_extension(fullname, buffer): + mime = magic.from_buffer(buffer, mime=True) try: return EXTENSOES[mime] except KeyError as e: - msg = '\n'.join([ - 'Extensão não conhecida para o arquivo:', - caminho, - 'E mimetype:', - mime, - ' Algumas possibilidades são:', ] + + possibilidades = '\n'.join( [" '{}': '{}',".format(mime, ext) - for ext in mimetypes.guess_all_extensions(mime)] + - ['Atualize o código do dicionário EXTENSOES!'] - ) + for ext in mimetypes.guess_all_extensions(mime)]) + msg = '''Extensão não conhecida para o arquivo: {} + e mimetype: {} + Algumas possibilidades são: + {} + Atualize o código do dicionário EXTENSOES! + '''.format(fullname, mime, possibilidades) print(msg) raise Exception(msg, e) -nomes_arquivos_exportados = {} - - -def dump_file(doc, path): +def dump_file(doc, path, salvar): name = doc['__name__'] fullname = os.path.join(path, name) @@ -104,23 +103,11 @@ def dump_file(doc, path): doc['data'] = pdata pdata = doc - with open(fullname, 'w') as arq: - while pdata: - arq.write(pdata.pop('data')) - pdata = br(pdata.pop('next', None)) - - base, original_extension = os.path.splitext(fullname) - extension = guess_extension(fullname) - if extension == '.xml' and original_extension in ['.xsl', '.xslt']: - # não trocamos as extensões XSL e XSLT - final_name = fullname - else: - # trocamos a extensão pela adivinhada - final_name = base + extension - os.rename(fullname, final_name) - nomes_arquivos_exportados[fullname] = final_name - print(final_name) - + output = cStringIO.StringIO() + while pdata: + output.write(pdata.pop('data')) + pdata = br(pdata.pop('next', None)) + salvar(fullname, output.getvalue()) return name @@ -165,7 +152,7 @@ def logando_nao_identificados(): print('#' * 80) -def dump_folder(folder, path='', enum=enumerate_folder): +def dump_folder(folder, path, salvar, enum=enumerate_folder): name = folder['id'] path = os.path.join(path, name) if not os.path.exists(path): @@ -175,7 +162,7 @@ def dump_folder(folder, path='', enum=enumerate_folder): if dump == '?': nao_identificados[meta_type].append(path + '/' + id) elif dump: - id_interno = dump(obj, path) + id_interno = dump(obj, path, salvar) assert id == id_interno return name @@ -211,18 +198,16 @@ def read_sde(element): return data -def save_as_yaml(path, name, obj): +def save_as_yaml(path, name, obj, salvar): fullname = os.path.join(path, name) - with open(fullname, 'w') as arquivo: - yaml.safe_dump(obj, arquivo, allow_unicode=True) - print(fullname) - return fullname + conteudo = yaml.safe_dump(obj, allow_unicode=True) + salvar(fullname, conteudo) -def dump_sde(strdoc, path, tipo): +def dump_sde(strdoc, path, salvar, tipo): id = strdoc['id'] sde = read_sde(strdoc) - save_as_yaml(path, '{}.{}.yaml'.format(id, tipo), sde) + save_as_yaml(path, '{}.{}.yaml'.format(id, tipo), sde, salvar) return id @@ -243,7 +228,7 @@ DUMP_FUNCTIONS = { def get_app(data_fs_path): - storage = ZODB.FileStorage.FileStorage(data_fs_path) + storage = ZODB.FileStorage.FileStorage(data_fs_path, read_only=True) db = ZODB.DB(storage) connection = db.open() root = connection.root() @@ -265,60 +250,79 @@ def find_sapl(app): return sapl -def dump_propriedades(docs, path, encoding='iso-8859-1'): +def dump_propriedades(docs, path, salvar, encoding='iso-8859-1'): props_sapl = br(docs['props_sapl']) ids = [p['id'] for p in props_sapl['_properties']] props = {id: props_sapl[id] for id in ids} props = {id: p.decode(encoding) if isinstance(p, str) else p for id, p in props.items()} - save_as_yaml(path, 'sapl_documentos/propriedades.yaml', props) + save_as_yaml(path, 'sapl_documentos/propriedades.yaml', props, salvar) -def dump_usuarios(sapl, path): +def dump_usuarios(sapl, path, salvar): users = br(br(sapl['acl_users'])['data']) users = {k: br(v) for k, v in users['data'].items()} - save_as_yaml(path, 'usuarios.yaml', users) - - -def grava_nomes_arquivos_exportados(destino): - """Grava nomes dos arquivos exportados (originais -> definitivos) - """ - destino = Path(destino) - - def rel(caminho): - return str(destino.rel_path_to(caminho)) - - with open(destino.child('arquivos_exportados.yaml'), 'w') as arq: - nomes = {rel(k): rel(v) for k, v in nomes_arquivos_exportados.items()} - pyaml.dump(nomes, arq) + save_as_yaml(path, 'usuarios.yaml', users, salvar) -def _dump_sapl(data_fs_path, destino='../../../../media'): +def _dump_sapl(data_fs_path, destino, salvar): assert Path(data_fs_path).exists() app, close_db = get_app(data_fs_path) - nomes_arquivos_exportados.clear() # apaga nomes dos arquivos migrados try: sapl = find_sapl(app) # extrai folhas XSLT - dump_folder(br(sapl['XSLT']), destino) + dump_folder(br(sapl['XSLT']), destino, salvar) # extrai usuários com suas senhas e perfis - dump_usuarios(sapl, destino) + dump_usuarios(sapl, destino, salvar) # extrai documentos docs = br(sapl['sapl_documentos']) with logando_nao_identificados(): - dump_folder(docs, destino) - dump_propriedades(docs, destino) + dump_folder(docs, destino, salvar) + dump_propriedades(docs, destino, salvar) finally: close_db() - grava_nomes_arquivos_exportados(destino) DIR_DADOS_MIGRACAO = Path('~/migracao_sapl/').expand() -def repo_execute(repo, cmd): - return repo.git.execute(cmd.split()) +def repo_execute(repo, cmd, *args): + return repo.git.execute(cmd.split() + list(args)) + + +def get_annex_hashes(repo): + hashes = repo_execute( + repo, 'git annex find', '--format=${keyname}\n', '--include=*') + return {os.path.splitext(h)[0] for h in hashes.splitlines()} + + +def ajusta_extensao(fullname, conteudo): + base, extensao = os.path.splitext(fullname) + if extensao not in ['.xsl', '.xslt', '.yaml']: + # não trocamos as extensões XSL, XSLT e YAML + extensao = guess_extension(fullname, conteudo) + return base + extensao + + +def build_salvar(repo): + """Constroi função salvar que pula arquivos que já estão no annex + """ + hashes = get_annex_hashes(repo) + + def salvar(fullname, conteudo): + sha = hashlib.sha256() + sha.update(conteudo) + if sha.hexdigest() not in hashes: + fullname = ajusta_extensao(fullname, conteudo) + if os.path.exists(fullname): + # destrava arquivo pré-existente (o conteúdo mudou) + repo_execute(repo, 'git annex unlock', fullname) + with open(fullname, 'w') as arq: + arq.write(conteudo) + print(fullname) + + return salvar def dump_sapl(sigla): @@ -329,9 +333,13 @@ def dump_sapl(sigla): destino = DIR_DADOS_MIGRACAO.child('repos', nome_banco_legado) destino.mkdir(parents=True) repo = git.Repo.init(destino) - _dump_sapl(data_fs_path, destino) - # grava mundaças repo_execute(repo, 'git annex init') + repo_execute(repo, 'git config annex.thin true') + + salvar = build_salvar(repo) + _dump_sapl(data_fs_path, destino, salvar) + + # grava mundaças repo_execute(repo, 'git annex add sapl_documentos') repo.git.add(A=True) if 'master' not in repo.heads or repo.index.diff('HEAD'): From 0b49930da0dfef6bc25d0a2399fc36967ed30096 Mon Sep 17 00:00:00 2001 From: Victor Fabre Date: Wed, 18 Apr 2018 12:15:58 -0300 Subject: [PATCH 032/102] Fix #1899 (#1900) --- sapl/sessao/views.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/sapl/sessao/views.py b/sapl/sessao/views.py index 9f8f8aaa2..889f0ab8c 100644 --- a/sapl/sessao/views.py +++ b/sapl/sessao/views.py @@ -978,13 +978,12 @@ class MesaView(FormMixin, DetailView): return self.render_to_response(context) mesa = sessao.integrantemesa_set.all() if sessao else [] - cargos_ocupados = [m.cargo for m in mesa] cargos = CargoMesa.objects.all() cargos_vagos = list(set(cargos) - set(cargos_ocupados)) # FIX-ME: tem formas melhores de fazer isso, poupando linhas. - parlamentares = Legislatura.objects.first().mandato_set.all() + parlamentares = Legislatura.objects.get(id=sessao.legislatura_id).mandato_set.all() parlamentares_ocupados = [m.parlamentar for m in mesa] parlamentares_vagos = list( set( From 2806bf4d9bb8ad2f01d47d8b57b18c33541af0b5 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Wed, 18 Apr 2018 16:58:05 -0300 Subject: [PATCH 033/102] Fixes #1898 --- sapl/sessao/views.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sapl/sessao/views.py b/sapl/sessao/views.py index 889f0ab8c..95785b6fb 100644 --- a/sapl/sessao/views.py +++ b/sapl/sessao/views.py @@ -6,7 +6,7 @@ from django.contrib.auth.decorators import permission_required from django.contrib.auth.mixins import PermissionRequiredMixin from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from django.core.urlresolvers import reverse -from django.db.models import Max, Q +from django.db.models import Max, Q, Func from django.forms.utils import ErrorList from django.http import JsonResponse from django.http.response import Http404, HttpResponseRedirect @@ -393,8 +393,13 @@ def get_presencas_generic(model, sessao, legislatura): presentes = [p.parlamentar for p in presencas] + # COLLATION pt_BR.utf8 + nome_c = Func( + 'parlamentar__nome_parlamentar', + function='pt_BR.utf8', + template='(%(expressions)s) COLLATE "%(function)s"') mandato = Mandato.objects.filter( - legislatura=legislatura).order_by('parlamentar__nome_parlamentar') + legislatura=legislatura).order_by(nome_c.asc()) for m in mandato: if m.parlamentar in presentes: From 44150ef549a35f7ea9f58c0289328b79fa2075ec Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Thu, 19 Apr 2018 11:09:59 -0300 Subject: [PATCH 034/102] Fixes #1904 --- sapl/protocoloadm/forms.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/sapl/protocoloadm/forms.py b/sapl/protocoloadm/forms.py index dc3d5280c..36baa8f3e 100644 --- a/sapl/protocoloadm/forms.py +++ b/sapl/protocoloadm/forms.py @@ -668,11 +668,12 @@ class DocumentoAdministrativoForm(ModelForm): numero_documento = self.cleaned_data['numero'] tipo_documento = self.data['tipo'] - documento = DocumentoAdministrativo.objects.filter(numero=numero_documento, - tipo=tipo_documento, ano=ano_protocolo) - - if documento: - raise ValidationError('Documento já existente') + if not self.instance.pk: + documento = DocumentoAdministrativo.objects.filter(numero=numero_documento, + tipo=tipo_documento, + ano=ano_protocolo) + if documento: + raise ValidationError('Documento já existente') # campos opcionais, mas que se informados devem ser válidos if numero_protocolo and ano_protocolo: From a78b1adf821941de823bbc695b743db87fad30d5 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Thu, 19 Apr 2018 11:17:59 -0300 Subject: [PATCH 035/102] Release: 3.1.73 --- docker-compose.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a3173316a..1c0fe8fd4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ sapldb: ports: - "5432:5432" sapl: - image: interlegis/sapl:3.1.72 + image: interlegis/sapl:3.1.73 restart: always environment: ADMIN_PASSWORD: interlegis diff --git a/setup.py b/setup.py index 33c6bf7e0..732fe70bd 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ install_requires = [ ] setup( name='interlegis-sapl', - version='3.1.72', + version='3.1.73', packages=find_packages(), include_package_data=True, license='GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007', From f2f00cb5b1e687f93063e5d76464fef091d0f494 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Thu, 19 Apr 2018 11:23:36 -0300 Subject: [PATCH 036/102] HOT-FIX: retira checagem duplicada --- sapl/protocoloadm/forms.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sapl/protocoloadm/forms.py b/sapl/protocoloadm/forms.py index 36baa8f3e..b51b5facd 100644 --- a/sapl/protocoloadm/forms.py +++ b/sapl/protocoloadm/forms.py @@ -655,9 +655,6 @@ class DocumentoAdministrativoForm(ModelForm): def clean(self): super(DocumentoAdministrativoForm, self).clean() - if not self.is_valid(): - return self.cleaned_data - cleaned_data = self.cleaned_data if not self.is_valid(): From 4aad96c15f3739d732a6dbc337051be11e523b70 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Thu, 19 Apr 2018 11:38:54 -0300 Subject: [PATCH 037/102] =?UTF-8?q?N=C3=A3o=20permite=20atualizar=20doc=20?= =?UTF-8?q?adm=20para=20tipo-n=C3=BAmero-ano=20existente?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/protocoloadm/forms.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/sapl/protocoloadm/forms.py b/sapl/protocoloadm/forms.py index b51b5facd..1d5e84133 100644 --- a/sapl/protocoloadm/forms.py +++ b/sapl/protocoloadm/forms.py @@ -662,10 +662,17 @@ class DocumentoAdministrativoForm(ModelForm): numero_protocolo = self.data['numero_protocolo'] ano_protocolo = self.data['ano_protocolo'] - numero_documento = self.cleaned_data['numero'] - tipo_documento = self.data['tipo'] + numero_documento = int(self.cleaned_data['numero']) + tipo_documento = int(self.data['tipo']) + ano_documento = int(self.data['ano']) - if not self.instance.pk: + # não permite atualizar para numero/ano/tipo existente + if self.instance.pk: + mudanca_doc = numero_documento != self.instance.numero \ + or ano_documento != self.instance.ano \ + or tipo_documento != self.instance.tipo.pk + + if not self.instance.pk or mudanca_doc: documento = DocumentoAdministrativo.objects.filter(numero=numero_documento, tipo=tipo_documento, ano=ano_protocolo) From 1dfcd5462796be2d55bedc315d216b1753ebddf3 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Fri, 20 Apr 2018 15:21:41 -0300 Subject: [PATCH 038/102] Revert "Fixes #1898" This reverts commit 2806bf4d9bb8ad2f01d47d8b57b18c33541af0b5. --- sapl/sessao/views.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/sapl/sessao/views.py b/sapl/sessao/views.py index 95785b6fb..889f0ab8c 100644 --- a/sapl/sessao/views.py +++ b/sapl/sessao/views.py @@ -6,7 +6,7 @@ from django.contrib.auth.decorators import permission_required from django.contrib.auth.mixins import PermissionRequiredMixin from django.core.exceptions import MultipleObjectsReturned, ObjectDoesNotExist from django.core.urlresolvers import reverse -from django.db.models import Max, Q, Func +from django.db.models import Max, Q from django.forms.utils import ErrorList from django.http import JsonResponse from django.http.response import Http404, HttpResponseRedirect @@ -393,13 +393,8 @@ def get_presencas_generic(model, sessao, legislatura): presentes = [p.parlamentar for p in presencas] - # COLLATION pt_BR.utf8 - nome_c = Func( - 'parlamentar__nome_parlamentar', - function='pt_BR.utf8', - template='(%(expressions)s) COLLATE "%(function)s"') mandato = Mandato.objects.filter( - legislatura=legislatura).order_by(nome_c.asc()) + legislatura=legislatura).order_by('parlamentar__nome_parlamentar') for m in mandato: if m.parlamentar in presentes: From 8fc46b7260db3f8799f3347ab6a97ca04550f6c4 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Fri, 20 Apr 2018 15:22:23 -0300 Subject: [PATCH 039/102] Release: 3.1.74 --- docker-compose.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1c0fe8fd4..9a369d466 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ sapldb: ports: - "5432:5432" sapl: - image: interlegis/sapl:3.1.73 + image: interlegis/sapl:3.1.74 restart: always environment: ADMIN_PASSWORD: interlegis diff --git a/setup.py b/setup.py index 732fe70bd..33d1bd47e 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ install_requires = [ ] setup( name='interlegis-sapl', - version='3.1.73', + version='3.1.74', packages=find_packages(), include_package_data=True, license='GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007', From edd132196790f5551ce6e343a0ff78efcccbcb76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro=20Sconetto?= Date: Fri, 20 Apr 2018 15:27:18 -0300 Subject: [PATCH 040/102] Fix #1785 --- sapl/materia/models.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sapl/materia/models.py b/sapl/materia/models.py index ce4690c39..4a38a3813 100644 --- a/sapl/materia/models.py +++ b/sapl/materia/models.py @@ -325,9 +325,11 @@ class AcompanhamentoMateria(models.Model): verbose_name_plural = _('Acompanhamentos de Matéria') def __str__(self): - # FIXME str should be human readable, using hash is very strange - return _('%(materia)s - #%(hash)s') % { - 'materia': self.materia, 'hash': self.hash} + return _('%(materia)s - %(email)s - Registrado em: %(data)s') % { + 'materia': self.materia, + 'email': self.email, + 'data': str(self.data_cadastro.strftime('%d/%m/%Y')) + } @reversion.register() From e7795002b14f23a53d4485565aa580b8953466e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro=20Sconetto?= Date: Fri, 20 Apr 2018 15:38:07 -0300 Subject: [PATCH 041/102] Fixing Travis Error --- sapl/materia/models.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/sapl/materia/models.py b/sapl/materia/models.py index 4a38a3813..98a045d01 100644 --- a/sapl/materia/models.py +++ b/sapl/materia/models.py @@ -325,11 +325,17 @@ class AcompanhamentoMateria(models.Model): verbose_name_plural = _('Acompanhamentos de Matéria') def __str__(self): - return _('%(materia)s - %(email)s - Registrado em: %(data)s') % { - 'materia': self.materia, - 'email': self.email, - 'data': str(self.data_cadastro.strftime('%d/%m/%Y')) - } + if self.data_cadastro is None: + return _('%(materia)s - %(email)s') % { + 'materia': self.materia, + 'email': self.email + } + else: + return _('%(materia)s - %(email)s - Registrado em: %(data)s') % { + 'materia': self.materia, + 'email': self.email, + 'data': str(self.data_cadastro.strftime('%d/%m/%Y')) + } @reversion.register() From e4866c09f0749fdd02e5cff73b4dd16cd3e7cf80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro=20Sconetto?= Date: Mon, 23 Apr 2018 09:57:51 -0300 Subject: [PATCH 042/102] Fix #1908 (#1909) --- sapl/sessao/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sapl/sessao/forms.py b/sapl/sessao/forms.py index 219e5b69b..c35abf306 100644 --- a/sapl/sessao/forms.py +++ b/sapl/sessao/forms.py @@ -452,7 +452,7 @@ class OradorForm(ModelForm): sessao_plenaria_id=id_sessao)] self.fields['parlamentar'].queryset = Parlamentar.objects.filter( - id__in=ids).order_by('nome_completo') + id__in=ids).order_by('nome_parlamentar') class Meta: model = Orador From 0707de627b13e8231a97972797cc5dd595c3947b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro=20Sconetto?= Date: Mon, 23 Apr 2018 09:57:51 -0300 Subject: [PATCH 043/102] Fix #1908 (#1909) --- sapl/sessao/forms.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sapl/sessao/forms.py b/sapl/sessao/forms.py index 219e5b69b..c35abf306 100644 --- a/sapl/sessao/forms.py +++ b/sapl/sessao/forms.py @@ -452,7 +452,7 @@ class OradorForm(ModelForm): sessao_plenaria_id=id_sessao)] self.fields['parlamentar'].queryset = Parlamentar.objects.filter( - id__in=ids).order_by('nome_completo') + id__in=ids).order_by('nome_parlamentar') class Meta: model = Orador From 42dd49ba2c960254e6e9506e4c8970153c80af98 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Thu, 19 Apr 2018 11:23:36 -0300 Subject: [PATCH 044/102] HOT-FIX: retira checagem duplicada --- sapl/protocoloadm/forms.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sapl/protocoloadm/forms.py b/sapl/protocoloadm/forms.py index 36baa8f3e..b51b5facd 100644 --- a/sapl/protocoloadm/forms.py +++ b/sapl/protocoloadm/forms.py @@ -655,9 +655,6 @@ class DocumentoAdministrativoForm(ModelForm): def clean(self): super(DocumentoAdministrativoForm, self).clean() - if not self.is_valid(): - return self.cleaned_data - cleaned_data = self.cleaned_data if not self.is_valid(): From 005131282d733aa3f64f8ed943ba323a6cafa83f Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Thu, 19 Apr 2018 11:38:54 -0300 Subject: [PATCH 045/102] =?UTF-8?q?N=C3=A3o=20permite=20atualizar=20doc=20?= =?UTF-8?q?adm=20para=20tipo-n=C3=BAmero-ano=20existente?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/protocoloadm/forms.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/sapl/protocoloadm/forms.py b/sapl/protocoloadm/forms.py index b51b5facd..1d5e84133 100644 --- a/sapl/protocoloadm/forms.py +++ b/sapl/protocoloadm/forms.py @@ -662,10 +662,17 @@ class DocumentoAdministrativoForm(ModelForm): numero_protocolo = self.data['numero_protocolo'] ano_protocolo = self.data['ano_protocolo'] - numero_documento = self.cleaned_data['numero'] - tipo_documento = self.data['tipo'] + numero_documento = int(self.cleaned_data['numero']) + tipo_documento = int(self.data['tipo']) + ano_documento = int(self.data['ano']) - if not self.instance.pk: + # não permite atualizar para numero/ano/tipo existente + if self.instance.pk: + mudanca_doc = numero_documento != self.instance.numero \ + or ano_documento != self.instance.ano \ + or tipo_documento != self.instance.tipo.pk + + if not self.instance.pk or mudanca_doc: documento = DocumentoAdministrativo.objects.filter(numero=numero_documento, tipo=tipo_documento, ano=ano_protocolo) From 959005532aa22eedc0af3adbf55f747341da8d19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jo=C3=A3o=20Pedro=20Sconetto?= Date: Fri, 20 Apr 2018 15:38:07 -0300 Subject: [PATCH 046/102] Fix #Fix #1785 --- sapl/materia/models.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sapl/materia/models.py b/sapl/materia/models.py index a1571072a..606fabbd7 100644 --- a/sapl/materia/models.py +++ b/sapl/materia/models.py @@ -325,9 +325,17 @@ class AcompanhamentoMateria(models.Model): verbose_name_plural = _('Acompanhamentos de Matéria') def __str__(self): - # FIXME str should be human readable, using hash is very strange - return _('%(materia)s - #%(hash)s') % { - 'materia': self.materia, 'hash': self.hash} + if self.data_cadastro is None: + return _('%(materia)s - %(email)s') % { + 'materia': self.materia, + 'email': self.email + } + else: + return _('%(materia)s - %(email)s - Registrado em: %(data)s') % { + 'materia': self.materia, + 'email': self.email, + 'data': str(self.data_cadastro.strftime('%d/%m/%Y')) + } @reversion.register() From a03ced746ed2fb8c7ec023571f95cbe551ff65d5 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Mon, 23 Apr 2018 13:08:08 -0300 Subject: [PATCH 047/102] Remove volumes pendentes --- scripts_docker/remove-all-containers.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts_docker/remove-all-containers.sh b/scripts_docker/remove-all-containers.sh index 64d41cd5c..a3d8fc624 100755 --- a/scripts_docker/remove-all-containers.sh +++ b/scripts_docker/remove-all-containers.sh @@ -1,4 +1,5 @@ #!/bin/bash -sudo docker stop $(docker ps -a -q) # Parar containers -sudo docker rm $(sudo docker ps -a -q) # Remover containers -sudo docker rmi -f $( sudo docker images -q ) # Remover imagens +sudo docker stop $(docker ps -a -q) # Para containers +sudo docker rm $(sudo docker ps -a -q) # Remove containers +sudo docker rmi -f $( sudo docker images -q ) # Remove imagens +sudo docker volume rm $(sudo docker volume ls -q -f dangling=true) # Remove volumes From ce32268154e8c268139b90025e7bfd07ce5d1de2 Mon Sep 17 00:00:00 2001 From: Edward Date: Mon, 23 Apr 2018 13:32:35 -0300 Subject: [PATCH 048/102] Fix #1898 (#1910) --- sapl/sessao/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sapl/sessao/views.py b/sapl/sessao/views.py index 889f0ab8c..4ad6178ab 100644 --- a/sapl/sessao/views.py +++ b/sapl/sessao/views.py @@ -393,6 +393,8 @@ def get_presencas_generic(model, sessao, legislatura): presentes = [p.parlamentar for p in presencas] + presentes = sorted(presentes, key=lambda x: x.nome_parlamentar) + mandato = Mandato.objects.filter( legislatura=legislatura).order_by('parlamentar__nome_parlamentar') From 790cfd2aa99807105ca66f872e794ede0aaa124e Mon Sep 17 00:00:00 2001 From: Edward Date: Mon, 23 Apr 2018 13:32:35 -0300 Subject: [PATCH 049/102] Fix #1898 (#1910) --- sapl/sessao/views.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sapl/sessao/views.py b/sapl/sessao/views.py index 889f0ab8c..4ad6178ab 100644 --- a/sapl/sessao/views.py +++ b/sapl/sessao/views.py @@ -393,6 +393,8 @@ def get_presencas_generic(model, sessao, legislatura): presentes = [p.parlamentar for p in presencas] + presentes = sorted(presentes, key=lambda x: x.nome_parlamentar) + mandato = Mandato.objects.filter( legislatura=legislatura).order_by('parlamentar__nome_parlamentar') From 6201c7f09284d4f94c9beca14f1892cc117118ea Mon Sep 17 00:00:00 2001 From: Edward Date: Wed, 25 Apr 2018 14:16:32 -0300 Subject: [PATCH 050/102] Fix #1913 (#1914) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix #1913 * Remove import não utilizado --- sapl/sessao/views.py | 73 +++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 45 deletions(-) diff --git a/sapl/sessao/views.py b/sapl/sessao/views.py index 4ad6178ab..d7844b3ce 100644 --- a/sapl/sessao/views.py +++ b/sapl/sessao/views.py @@ -30,7 +30,6 @@ from sapl.materia.forms import filtra_tramitacao_status from sapl.materia.models import (Autoria, DocumentoAcessorio, TipoMateriaLegislativa, Tramitacao) from sapl.materia.views import MateriaLegislativaPesquisaView -from sapl.norma.models import NormaJuridica from sapl.parlamentares.models import (Filiacao, Legislatura, Mandato, Parlamentar, SessaoLegislativa) from sapl.sessao.apps import AppConfig @@ -1114,15 +1113,13 @@ def remove_parlamentar_composicao(request): if 'composicao_mesa' in request.POST: try: - composicao = IntegranteMesa.objects.get( - id=int(request.POST['composicao_mesa'])) + IntegranteMesa.objects.get( + id=int(request.POST['composicao_mesa'])).delete() except ObjectDoesNotExist: return JsonResponse( {'msg': ( 'Composição da Mesa não pôde ser removida!', 0)}) - composicao.delete() - return JsonResponse( {'msg': ( 'Parlamentar excluido com sucesso!', 1)}) @@ -1526,8 +1523,7 @@ class VotacaoEditView(SessaoPermissionMixin): ordem_id = kwargs['oid'] if(int(request.POST['anular_votacao']) == 1): - for r in RegistroVotacao.objects.filter(ordem_id=ordem_id): - r.delete() + RegistroVotacao.objects.filter(ordem_id=ordem_id).delete() ordem = OrdemDia.objects.get( sessao_plenaria_id=self.object.id, @@ -1558,9 +1554,8 @@ class VotacaoEditView(SessaoPermissionMixin): materia = {'materia': ordem.materia, 'ementa': ordem.materia.ementa} context.update({'materia': materia}) - votacao = RegistroVotacao.objects.filter( - materia_id=materia_id, - ordem_id=ordem_id).last() + votacao = RegistroVotacao.objects.filter(materia_id=materia_id, + ordem_id=ordem_id).last() votacao_existente = {'observacao': sub( ' ', ' ', strip_tags(votacao.observacao)), 'resultado': votacao.tipo_resultado_votacao.nome, @@ -1707,8 +1702,7 @@ def fechar_votacao_materia(materia): VotoParlamentar.objects.filter(ordem=materia).delete() elif type(materia) == ExpedienteMateria: - RegistroVotacao.objects.filter( - expediente=materia).delete() + RegistroVotacao.objects.filter(expediente=materia).delete() VotoParlamentar.objects.filter(expediente=materia).delete() if materia.resultado: @@ -1756,7 +1750,7 @@ class VotacaoNominalAbstract(SessaoPermissionMixin): elif self.expediente: expediente_id = kwargs['oid'] if (RegistroVotacao.objects.filter( - expediente_id=expediente_id).exists()): + expediente_id=expediente_id).exists()): msg = _('Esta matéria já foi votada!') messages.add_message(request, messages.ERROR, msg) return HttpResponseRedirect(reverse( @@ -1843,8 +1837,7 @@ class VotacaoNominalAbstract(SessaoPermissionMixin): return self.form_invalid(form) # Remove todas as votação desta matéria, caso existam if self.ordem: - RegistroVotacao.objects.filter( - ordem_id=ordem_id).delete() + RegistroVotacao.objects.filter(ordem_id=ordem_id).delete() elif self.expediente: RegistroVotacao.objects.filter( expediente_id=expediente_id).delete() @@ -1972,11 +1965,10 @@ class VotacaoNominalEditAbstract(SessaoPermissionMixin): if self.ordem: ordem_id = kwargs['oid'] - try: - ordem = OrdemDia.objects.get(id=ordem_id) - votacao = RegistroVotacao.objects.get( - ordem_id=ordem_id) - except ObjectDoesNotExist: + ordem = OrdemDia.objects.filter(id=ordem_id).last() + votacao = RegistroVotacao.objects.filter(ordem_id=ordem_id).last() + + if not ordem or not votacao: raise Http404() materia = ordem.materia @@ -1985,11 +1977,10 @@ class VotacaoNominalEditAbstract(SessaoPermissionMixin): elif self.expediente: expediente_id = kwargs['oid'] - try: - expediente = ExpedienteMateria.objects.get(id=expediente_id) - votacao = RegistroVotacao.objects.get( - expediente_id=expediente_id) - except ObjectDoesNotExist: + expediente = ExpedienteMateria.objects.filter(id=expediente_id).last() + votacao = RegistroVotacao.objects.filter(expediente_id=expediente_id).last() + + if not expediente or not votacao: raise Http404() materia = expediente.materia @@ -2106,9 +2097,9 @@ class VotacaoNominalTransparenciaDetailView(TemplateView): materia_votacao = self.request.GET.get('materia', None) if materia_votacao == 'ordem': - votacao = RegistroVotacao.objects.get(ordem=self.kwargs['oid']) + votacao = RegistroVotacao.objects.filter(ordem=self.kwargs['oid']).last() elif materia_votacao == 'expediente': - votacao = RegistroVotacao.objects.get(expediente=self.kwargs['oid']) + votacao = RegistroVotacao.objects.filter(expediente=self.kwargs['oid']).last() else: raise Http404() @@ -2142,10 +2133,9 @@ class VotacaoNominalExpedienteDetailView(DetailView): materia_id = kwargs['mid'] expediente_id = kwargs['oid'] - votacao = RegistroVotacao.objects.get( - materia_id=materia_id, - expediente_id=expediente_id) - expediente = ExpedienteMateria.objects.get(id=expediente_id) + votacao = RegistroVotacao.objects.filter(materia_id=materia_id, + expediente_id=expediente_id).last() + expediente = ExpedienteMateria.objects.filter(id=expediente_id).last() votos = VotoParlamentar.objects.filter(votacao_id=votacao.id) list_votos = [] @@ -2190,9 +2180,9 @@ class VotacaoSimbolicaTransparenciaDetailView(TemplateView): materia_votacao = self.request.GET.get('materia', None) if materia_votacao == 'ordem': - votacao = RegistroVotacao.objects.get(ordem=self.kwargs['oid']) + votacao = RegistroVotacao.objects.filter(ordem=self.kwargs['oid']).last() elif materia_votacao == 'expediente': - votacao = RegistroVotacao.objects.get(expediente=self.kwargs['oid']) + votacao = RegistroVotacao.objects.filter(expediente=self.kwargs['oid']).last() else: raise Http404() @@ -2378,14 +2368,9 @@ class VotacaoExpedienteEditView(SessaoPermissionMixin): 'ementa': expediente.materia.ementa} context.update({'materia': materia}) - try: - votacao = RegistroVotacao.objects.get( - materia_id=materia_id, - expediente_id=expediente_id) - except MultipleObjectsReturned: - votacao = RegistroVotacao.objects.filter( - materia_id=materia_id, - expediente_id=expediente_id).last() + votacao = RegistroVotacao.objects.filter(materia_id=materia_id, + expediente_id=expediente_id + ).last() votacao_existente = {'observacao': sub( ' ', ' ', strip_tags(votacao.observacao)), 'resultado': votacao.tipo_resultado_votacao.nome, @@ -2404,10 +2389,8 @@ class VotacaoExpedienteEditView(SessaoPermissionMixin): materia_id = kwargs['mid'] expediente_id = kwargs['oid'] - if(int(request.POST['anular_votacao']) == 1): - for r in RegistroVotacao.objects.filter( - expediente_id=expediente_id): - r.delete() + if int(request.POST['anular_votacao']) == 1: + RegistroVotacao.objects.filter(expediente_id=expediente_id).delete() expediente = ExpedienteMateria.objects.get( sessao_plenaria_id=self.object.id, From 742783379412b36941102e3a607afc38d76f5754 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Mon, 23 Apr 2018 13:08:08 -0300 Subject: [PATCH 051/102] Remove volumes pendentes --- scripts_docker/remove-all-containers.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts_docker/remove-all-containers.sh b/scripts_docker/remove-all-containers.sh index 64d41cd5c..a3d8fc624 100755 --- a/scripts_docker/remove-all-containers.sh +++ b/scripts_docker/remove-all-containers.sh @@ -1,4 +1,5 @@ #!/bin/bash -sudo docker stop $(docker ps -a -q) # Parar containers -sudo docker rm $(sudo docker ps -a -q) # Remover containers -sudo docker rmi -f $( sudo docker images -q ) # Remover imagens +sudo docker stop $(docker ps -a -q) # Para containers +sudo docker rm $(sudo docker ps -a -q) # Remove containers +sudo docker rmi -f $( sudo docker images -q ) # Remove imagens +sudo docker volume rm $(sudo docker volume ls -q -f dangling=true) # Remove volumes From ea62aeec3f493406f82007eb0b8c38f0a7666644 Mon Sep 17 00:00:00 2001 From: Edward Date: Wed, 25 Apr 2018 14:16:32 -0300 Subject: [PATCH 052/102] Fix #1913 (#1914) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix #1913 * Remove import não utilizado --- sapl/sessao/views.py | 73 +++++++++++++++++--------------------------- 1 file changed, 28 insertions(+), 45 deletions(-) diff --git a/sapl/sessao/views.py b/sapl/sessao/views.py index 4ad6178ab..d7844b3ce 100644 --- a/sapl/sessao/views.py +++ b/sapl/sessao/views.py @@ -30,7 +30,6 @@ from sapl.materia.forms import filtra_tramitacao_status from sapl.materia.models import (Autoria, DocumentoAcessorio, TipoMateriaLegislativa, Tramitacao) from sapl.materia.views import MateriaLegislativaPesquisaView -from sapl.norma.models import NormaJuridica from sapl.parlamentares.models import (Filiacao, Legislatura, Mandato, Parlamentar, SessaoLegislativa) from sapl.sessao.apps import AppConfig @@ -1114,15 +1113,13 @@ def remove_parlamentar_composicao(request): if 'composicao_mesa' in request.POST: try: - composicao = IntegranteMesa.objects.get( - id=int(request.POST['composicao_mesa'])) + IntegranteMesa.objects.get( + id=int(request.POST['composicao_mesa'])).delete() except ObjectDoesNotExist: return JsonResponse( {'msg': ( 'Composição da Mesa não pôde ser removida!', 0)}) - composicao.delete() - return JsonResponse( {'msg': ( 'Parlamentar excluido com sucesso!', 1)}) @@ -1526,8 +1523,7 @@ class VotacaoEditView(SessaoPermissionMixin): ordem_id = kwargs['oid'] if(int(request.POST['anular_votacao']) == 1): - for r in RegistroVotacao.objects.filter(ordem_id=ordem_id): - r.delete() + RegistroVotacao.objects.filter(ordem_id=ordem_id).delete() ordem = OrdemDia.objects.get( sessao_plenaria_id=self.object.id, @@ -1558,9 +1554,8 @@ class VotacaoEditView(SessaoPermissionMixin): materia = {'materia': ordem.materia, 'ementa': ordem.materia.ementa} context.update({'materia': materia}) - votacao = RegistroVotacao.objects.filter( - materia_id=materia_id, - ordem_id=ordem_id).last() + votacao = RegistroVotacao.objects.filter(materia_id=materia_id, + ordem_id=ordem_id).last() votacao_existente = {'observacao': sub( ' ', ' ', strip_tags(votacao.observacao)), 'resultado': votacao.tipo_resultado_votacao.nome, @@ -1707,8 +1702,7 @@ def fechar_votacao_materia(materia): VotoParlamentar.objects.filter(ordem=materia).delete() elif type(materia) == ExpedienteMateria: - RegistroVotacao.objects.filter( - expediente=materia).delete() + RegistroVotacao.objects.filter(expediente=materia).delete() VotoParlamentar.objects.filter(expediente=materia).delete() if materia.resultado: @@ -1756,7 +1750,7 @@ class VotacaoNominalAbstract(SessaoPermissionMixin): elif self.expediente: expediente_id = kwargs['oid'] if (RegistroVotacao.objects.filter( - expediente_id=expediente_id).exists()): + expediente_id=expediente_id).exists()): msg = _('Esta matéria já foi votada!') messages.add_message(request, messages.ERROR, msg) return HttpResponseRedirect(reverse( @@ -1843,8 +1837,7 @@ class VotacaoNominalAbstract(SessaoPermissionMixin): return self.form_invalid(form) # Remove todas as votação desta matéria, caso existam if self.ordem: - RegistroVotacao.objects.filter( - ordem_id=ordem_id).delete() + RegistroVotacao.objects.filter(ordem_id=ordem_id).delete() elif self.expediente: RegistroVotacao.objects.filter( expediente_id=expediente_id).delete() @@ -1972,11 +1965,10 @@ class VotacaoNominalEditAbstract(SessaoPermissionMixin): if self.ordem: ordem_id = kwargs['oid'] - try: - ordem = OrdemDia.objects.get(id=ordem_id) - votacao = RegistroVotacao.objects.get( - ordem_id=ordem_id) - except ObjectDoesNotExist: + ordem = OrdemDia.objects.filter(id=ordem_id).last() + votacao = RegistroVotacao.objects.filter(ordem_id=ordem_id).last() + + if not ordem or not votacao: raise Http404() materia = ordem.materia @@ -1985,11 +1977,10 @@ class VotacaoNominalEditAbstract(SessaoPermissionMixin): elif self.expediente: expediente_id = kwargs['oid'] - try: - expediente = ExpedienteMateria.objects.get(id=expediente_id) - votacao = RegistroVotacao.objects.get( - expediente_id=expediente_id) - except ObjectDoesNotExist: + expediente = ExpedienteMateria.objects.filter(id=expediente_id).last() + votacao = RegistroVotacao.objects.filter(expediente_id=expediente_id).last() + + if not expediente or not votacao: raise Http404() materia = expediente.materia @@ -2106,9 +2097,9 @@ class VotacaoNominalTransparenciaDetailView(TemplateView): materia_votacao = self.request.GET.get('materia', None) if materia_votacao == 'ordem': - votacao = RegistroVotacao.objects.get(ordem=self.kwargs['oid']) + votacao = RegistroVotacao.objects.filter(ordem=self.kwargs['oid']).last() elif materia_votacao == 'expediente': - votacao = RegistroVotacao.objects.get(expediente=self.kwargs['oid']) + votacao = RegistroVotacao.objects.filter(expediente=self.kwargs['oid']).last() else: raise Http404() @@ -2142,10 +2133,9 @@ class VotacaoNominalExpedienteDetailView(DetailView): materia_id = kwargs['mid'] expediente_id = kwargs['oid'] - votacao = RegistroVotacao.objects.get( - materia_id=materia_id, - expediente_id=expediente_id) - expediente = ExpedienteMateria.objects.get(id=expediente_id) + votacao = RegistroVotacao.objects.filter(materia_id=materia_id, + expediente_id=expediente_id).last() + expediente = ExpedienteMateria.objects.filter(id=expediente_id).last() votos = VotoParlamentar.objects.filter(votacao_id=votacao.id) list_votos = [] @@ -2190,9 +2180,9 @@ class VotacaoSimbolicaTransparenciaDetailView(TemplateView): materia_votacao = self.request.GET.get('materia', None) if materia_votacao == 'ordem': - votacao = RegistroVotacao.objects.get(ordem=self.kwargs['oid']) + votacao = RegistroVotacao.objects.filter(ordem=self.kwargs['oid']).last() elif materia_votacao == 'expediente': - votacao = RegistroVotacao.objects.get(expediente=self.kwargs['oid']) + votacao = RegistroVotacao.objects.filter(expediente=self.kwargs['oid']).last() else: raise Http404() @@ -2378,14 +2368,9 @@ class VotacaoExpedienteEditView(SessaoPermissionMixin): 'ementa': expediente.materia.ementa} context.update({'materia': materia}) - try: - votacao = RegistroVotacao.objects.get( - materia_id=materia_id, - expediente_id=expediente_id) - except MultipleObjectsReturned: - votacao = RegistroVotacao.objects.filter( - materia_id=materia_id, - expediente_id=expediente_id).last() + votacao = RegistroVotacao.objects.filter(materia_id=materia_id, + expediente_id=expediente_id + ).last() votacao_existente = {'observacao': sub( ' ', ' ', strip_tags(votacao.observacao)), 'resultado': votacao.tipo_resultado_votacao.nome, @@ -2404,10 +2389,8 @@ class VotacaoExpedienteEditView(SessaoPermissionMixin): materia_id = kwargs['mid'] expediente_id = kwargs['oid'] - if(int(request.POST['anular_votacao']) == 1): - for r in RegistroVotacao.objects.filter( - expediente_id=expediente_id): - r.delete() + if int(request.POST['anular_votacao']) == 1: + RegistroVotacao.objects.filter(expediente_id=expediente_id).delete() expediente = ExpedienteMateria.objects.get( sessao_plenaria_id=self.object.id, From 1acf71395399a3ba54235e73150c51b80c3f36ce Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Wed, 25 Apr 2018 15:53:00 -0300 Subject: [PATCH 053/102] =?UTF-8?q?Adiciona=20campo=20n=C3=BAmero=20extern?= =?UTF-8?q?o=20(opcional)=20para=20docs=20adm?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/protocoloadm/forms.py | 15 ++++++++------ ..._documentoadministrativo_numero_externo.py | 20 +++++++++++++++++++ sapl/protocoloadm/models.py | 4 ++++ sapl/templates/protocoloadm/layouts.yaml | 1 + 4 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 sapl/protocoloadm/migrations/0004_documentoadministrativo_numero_externo.py diff --git a/sapl/protocoloadm/forms.py b/sapl/protocoloadm/forms.py index 1d5e84133..06a2f7854 100644 --- a/sapl/protocoloadm/forms.py +++ b/sapl/protocoloadm/forms.py @@ -155,6 +155,7 @@ class DocumentoAdministrativoFilterSet(django_filters.FilterSet): fields = ['tipo', 'numero', 'protocolo__numero', + 'numero_externo', 'data', 'tramitacaoadministrativo__unidade_tramitacao_destino', 'tramitacaoadministrativo__status'] @@ -173,7 +174,8 @@ class DocumentoAdministrativoFilterSet(django_filters.FilterSet): row2 = to_row( [('ano', 4), - ('protocolo__numero', 4), + ('protocolo__numero', 2), + ('numero_externo', 2), ('data', 4)]) row3 = to_row( @@ -645,6 +647,7 @@ class DocumentoAdministrativoForm(ModelForm): 'tramitacao', 'dias_prazo', 'data_fim_prazo', + 'numero_externo', 'observacao', 'texto_integral', 'protocolo', @@ -673,10 +676,10 @@ class DocumentoAdministrativoForm(ModelForm): or tipo_documento != self.instance.tipo.pk if not self.instance.pk or mudanca_doc: - documento = DocumentoAdministrativo.objects.filter(numero=numero_documento, - tipo=tipo_documento, - ano=ano_protocolo) - if documento: + doc_exists = DocumentoAdministrativo.objects.filter(numero=numero_documento, + tipo=tipo_documento, + ano=ano_protocolo).exists() + if doc_exists: raise ValidationError('Documento já existente') # campos opcionais, mas que se informados devem ser válidos @@ -725,7 +728,7 @@ class DocumentoAdministrativoForm(ModelForm): [('texto_integral', 12)]) row6 = to_row( - [('dias_prazo', 6), ('data_fim_prazo', 6)]) + [('numero_externo', 4), ('dias_prazo', 6), ('data_fim_prazo', 2)]) row7 = to_row( [('observacao', 12)]) diff --git a/sapl/protocoloadm/migrations/0004_documentoadministrativo_numero_externo.py b/sapl/protocoloadm/migrations/0004_documentoadministrativo_numero_externo.py new file mode 100644 index 000000000..b3659efbb --- /dev/null +++ b/sapl/protocoloadm/migrations/0004_documentoadministrativo_numero_externo.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.11 on 2018-04-25 18:40 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('protocoloadm', '0003_auto_20180103_1343'), + ] + + operations = [ + migrations.AddField( + model_name='documentoadministrativo', + name='numero_externo', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='Número Externo'), + ), + ] diff --git a/sapl/protocoloadm/models.py b/sapl/protocoloadm/models.py index d4d45b3ad..cf7616d7a 100644 --- a/sapl/protocoloadm/models.py +++ b/sapl/protocoloadm/models.py @@ -133,6 +133,10 @@ class DocumentoAdministrativo(models.Model): verbose_name=_('Em Tramitação?'), choices=YES_NO_CHOICES) assunto = models.TextField(verbose_name=_('Assunto')) + numero_externo = models.PositiveIntegerField( + blank=True, + null=True, + verbose_name=_('Número Externo')) observacao = models.TextField( blank=True, verbose_name=_('Observação')) texto_integral = models.FileField( diff --git a/sapl/templates/protocoloadm/layouts.yaml b/sapl/templates/protocoloadm/layouts.yaml index 76b80da18..b1bb2f31b 100644 --- a/sapl/templates/protocoloadm/layouts.yaml +++ b/sapl/templates/protocoloadm/layouts.yaml @@ -11,6 +11,7 @@ DocumentoAdministrativo: - interessado tramitacao - texto_integral {% trans 'Outras Informações' %}: + - numero_externo - dias_prazo data_fim_prazo - observacao From fc471d6972204b1e2b1fc6019c6090ba15bf5093 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Wed, 25 Apr 2018 15:53:00 -0300 Subject: [PATCH 054/102] =?UTF-8?q?Adiciona=20campo=20n=C3=BAmero=20extern?= =?UTF-8?q?o=20(opcional)=20para=20docs=20adm?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/protocoloadm/forms.py | 15 ++++++++------ ..._documentoadministrativo_numero_externo.py | 20 +++++++++++++++++++ sapl/protocoloadm/models.py | 4 ++++ sapl/templates/protocoloadm/layouts.yaml | 1 + 4 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 sapl/protocoloadm/migrations/0004_documentoadministrativo_numero_externo.py diff --git a/sapl/protocoloadm/forms.py b/sapl/protocoloadm/forms.py index 1d5e84133..06a2f7854 100644 --- a/sapl/protocoloadm/forms.py +++ b/sapl/protocoloadm/forms.py @@ -155,6 +155,7 @@ class DocumentoAdministrativoFilterSet(django_filters.FilterSet): fields = ['tipo', 'numero', 'protocolo__numero', + 'numero_externo', 'data', 'tramitacaoadministrativo__unidade_tramitacao_destino', 'tramitacaoadministrativo__status'] @@ -173,7 +174,8 @@ class DocumentoAdministrativoFilterSet(django_filters.FilterSet): row2 = to_row( [('ano', 4), - ('protocolo__numero', 4), + ('protocolo__numero', 2), + ('numero_externo', 2), ('data', 4)]) row3 = to_row( @@ -645,6 +647,7 @@ class DocumentoAdministrativoForm(ModelForm): 'tramitacao', 'dias_prazo', 'data_fim_prazo', + 'numero_externo', 'observacao', 'texto_integral', 'protocolo', @@ -673,10 +676,10 @@ class DocumentoAdministrativoForm(ModelForm): or tipo_documento != self.instance.tipo.pk if not self.instance.pk or mudanca_doc: - documento = DocumentoAdministrativo.objects.filter(numero=numero_documento, - tipo=tipo_documento, - ano=ano_protocolo) - if documento: + doc_exists = DocumentoAdministrativo.objects.filter(numero=numero_documento, + tipo=tipo_documento, + ano=ano_protocolo).exists() + if doc_exists: raise ValidationError('Documento já existente') # campos opcionais, mas que se informados devem ser válidos @@ -725,7 +728,7 @@ class DocumentoAdministrativoForm(ModelForm): [('texto_integral', 12)]) row6 = to_row( - [('dias_prazo', 6), ('data_fim_prazo', 6)]) + [('numero_externo', 4), ('dias_prazo', 6), ('data_fim_prazo', 2)]) row7 = to_row( [('observacao', 12)]) diff --git a/sapl/protocoloadm/migrations/0004_documentoadministrativo_numero_externo.py b/sapl/protocoloadm/migrations/0004_documentoadministrativo_numero_externo.py new file mode 100644 index 000000000..b3659efbb --- /dev/null +++ b/sapl/protocoloadm/migrations/0004_documentoadministrativo_numero_externo.py @@ -0,0 +1,20 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.11 on 2018-04-25 18:40 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('protocoloadm', '0003_auto_20180103_1343'), + ] + + operations = [ + migrations.AddField( + model_name='documentoadministrativo', + name='numero_externo', + field=models.PositiveIntegerField(blank=True, null=True, verbose_name='Número Externo'), + ), + ] diff --git a/sapl/protocoloadm/models.py b/sapl/protocoloadm/models.py index d4d45b3ad..cf7616d7a 100644 --- a/sapl/protocoloadm/models.py +++ b/sapl/protocoloadm/models.py @@ -133,6 +133,10 @@ class DocumentoAdministrativo(models.Model): verbose_name=_('Em Tramitação?'), choices=YES_NO_CHOICES) assunto = models.TextField(verbose_name=_('Assunto')) + numero_externo = models.PositiveIntegerField( + blank=True, + null=True, + verbose_name=_('Número Externo')) observacao = models.TextField( blank=True, verbose_name=_('Observação')) texto_integral = models.FileField( diff --git a/sapl/templates/protocoloadm/layouts.yaml b/sapl/templates/protocoloadm/layouts.yaml index 76b80da18..b1bb2f31b 100644 --- a/sapl/templates/protocoloadm/layouts.yaml +++ b/sapl/templates/protocoloadm/layouts.yaml @@ -11,6 +11,7 @@ DocumentoAdministrativo: - interessado tramitacao - texto_integral {% trans 'Outras Informações' %}: + - numero_externo - dias_prazo data_fim_prazo - observacao From 7257c48fadbd0d8a818e39aa481932cbc0688b85 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Wed, 25 Apr 2018 15:54:48 -0300 Subject: [PATCH 055/102] Release: 3.1.75 --- docker-compose.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 9a369d466..48e9a3b30 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ sapldb: ports: - "5432:5432" sapl: - image: interlegis/sapl:3.1.74 + image: interlegis/sapl:3.1.75 restart: always environment: ADMIN_PASSWORD: interlegis diff --git a/setup.py b/setup.py index 33d1bd47e..edf2dc502 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ install_requires = [ ] setup( name='interlegis-sapl', - version='3.1.74', + version='3.1.75', packages=find_packages(), include_package_data=True, license='GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007', From a502b90cc60b8123931831701d8d5a72bf0e3af5 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 19 Apr 2018 10:59:37 -0300 Subject: [PATCH 056/102] =?UTF-8?q?Adiciona=20depend=C3=AAncias=20faltando?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sapl/legacy/scripts/exporta_zope/requirements.txt b/sapl/legacy/scripts/exporta_zope/requirements.txt index c7aa86b4f..69305576b 100644 --- a/sapl/legacy/scripts/exporta_zope/requirements.txt +++ b/sapl/legacy/scripts/exporta_zope/requirements.txt @@ -4,3 +4,5 @@ PyYAML Unipath GitPython pyaml +python-magic +ipython From 8997c91dfcbf252127ece602c066d00e14e34203 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 19 Apr 2018 11:58:43 -0300 Subject: [PATCH 057/102] Ignora aquivos vazios ao exportar docs --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 2c687bc07..a22c49b87 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -18,10 +18,9 @@ import git import magic import pyaml import yaml -from unipath import Path - import ZODB.DB import ZODB.FileStorage +from unipath import Path from ZODB.broken import Broken EXTENSOES = { @@ -107,7 +106,11 @@ def dump_file(doc, path, salvar): while pdata: output.write(pdata.pop('data')) pdata = br(pdata.pop('next', None)) - salvar(fullname, output.getvalue()) + + conteudo = output.getvalue() + if conteudo: + # pula arquivos vazios + salvar(fullname, conteudo) return name From b8fc7fcd2a8187704898a3cc0f84937ad204c524 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 19 Apr 2018 11:59:14 -0300 Subject: [PATCH 058/102] =?UTF-8?q?Adiciona=20mais=20tipos=20de=20arquivos?= =?UTF-8?q?=20=C3=A0=20exporta=C3=A7=C3=A3o=20de=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index a22c49b87..2d66d97c4 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -29,6 +29,9 @@ EXTENSOES = { 'application/vnd.oasis.opendocument.text': '.odt', 'application/vnd.ms-excel': '.xls', 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx', # noqa + 'application/vnd.oasis.opendocument.text-template': '.ott', + 'application/vnd.ms-powerpoint': '.ppt', + 'application/xml': '.xml', 'text/xml': '.xml', 'application/zip': '.zip', @@ -45,7 +48,9 @@ EXTENSOES = { 'audio/x-wav': '.wav', 'video/mp4': '.mp4', 'image/x-icon': '.ico', - 'application/vnd.oasis.opendocument.text-template': '.ott', + 'image/x-ms-bmp': '.bmp', + 'video/x-ms-asf': '.asf', + 'audio/mpeg': '.mp3', # TODO rever... 'text/richtext': '.rtf', @@ -53,7 +58,9 @@ EXTENSOES = { # sem extensao 'application/octet-stream': '', # binário 'inode/x-empty': '', # vazio - 'text/x-unknown-content-type': '', + 'application/x-empty': '', # vazio + 'text/x-unknown-content-type': '', # desconhecido + 'application/CDFV2-unknown': '', # desconhecido } From 8cb44f0235b1a0cd24d9803499a2b36d31fac3ad Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 19 Apr 2018 13:42:37 -0300 Subject: [PATCH 059/102] Ajusta print para uso com pipe --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 2d66d97c4..545e10462 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -14,10 +14,11 @@ from collections import defaultdict from contextlib import contextmanager from functools import partial +import yaml + import git import magic import pyaml -import yaml import ZODB.DB import ZODB.FileStorage from unipath import Path @@ -155,8 +156,8 @@ def logando_nao_identificados(): if nao_identificados: print('#' * 80) print('#' * 80) - print(u'FORAM ENCONTRADOS ARQUIVOS DE FORMATO NÃO IDENTIFICADO!!!') - print(u'REFAÇA A EXPORTAÇÃO\n') + print('FORAM ENCONTRADOS ARQUIVOS DE FORMATO NÃO IDENTIFICADO!!!') + print('REFAÇA A EXPORTAÇÃO\n') print(nao_identificados) print('#' * 80) print('#' * 80) From 31cd0c83d07b780d9417d36a41676db0156e4ac5 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 19 Apr 2018 18:07:52 -0300 Subject: [PATCH 060/102] Migra usando repo git para marcos --- requirements/migration-requirements.txt | 1 + sapl/legacy/migracao.py | 7 ++- sapl/legacy/migracao_dados.py | 25 ++++++-- sapl/legacy/migracao_documentos.py | 81 ++++++++++++++----------- sapl/legacy/migracao_usuarios.py | 8 +-- 5 files changed, 74 insertions(+), 48 deletions(-) diff --git a/requirements/migration-requirements.txt b/requirements/migration-requirements.txt index fbe9543e2..e3f887b14 100644 --- a/requirements/migration-requirements.txt +++ b/requirements/migration-requirements.txt @@ -1,3 +1,4 @@ -r dev-requirements.txt +GitPython mysqlclient==1.3.12 pyaml diff --git a/sapl/legacy/migracao.py b/sapl/legacy/migracao.py index be8830aa8..a72bd9aaa 100644 --- a/sapl/legacy/migracao.py +++ b/sapl/legacy/migracao.py @@ -3,15 +3,16 @@ import tarfile from django.conf import settings -from sapl.legacy.migracao_dados import migrar_dados +from sapl.legacy.migracao_dados import REPO, gravar_marco, migrar_dados from sapl.legacy.migracao_documentos import migrar_documentos from sapl.legacy.migracao_usuarios import migrar_usuarios def migrar(interativo=False): migrar_dados(interativo=interativo) - migrar_usuarios() - migrar_documentos() + migrar_usuarios(REPO.working_dir) + migrar_documentos(REPO) + gravar_marco() def gerar_pacote(): diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 52adfd2f1..492a8e579 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -8,6 +8,7 @@ from itertools import groupby from operator import xor from subprocess import PIPE, call +import git import pkg_resources import pyaml import pytz @@ -1256,30 +1257,44 @@ AJUSTE_DEPOIS_SALVAR = { TIME_FORMAT = '%H:%M:%S' +# permite a gravação de tempos puros pelo pretty-yaml def time_representer(dumper, data): return dumper.represent_scalar('!time', data.strftime(TIME_FORMAT)) UnsafePrettyYAMLDumper.add_representer(datetime.time, time_representer) +# permite a leitura de tempos puros pelo pyyaml (no padrão gravado acima) def time_constructor(loader, node): value = loader.construct_scalar(node) return datetime.datetime.strptime(value, TIME_FORMAT).time() yaml.add_constructor(u'!time', time_constructor) -DIR_MARCO = Path(DIR_DADOS_MIGRACAO, 'marcos', nome_banco_legado) +REPO = git.Repo.init(Path(DIR_DADOS_MIGRACAO, 'repos', nome_banco_legado)) -def grava_marco_base(): +def gravar_marco(): + """Grava um dump de todos os dados como arquivos yaml no repo de marco + """ + # prepara ou localiza repositorio + dir_dados = Path(REPO.working_dir, 'dados') + + # exporta dados como arquivos yaml user_model = get_user_model() models = get_models_a_migrar() + [ Composicao, user_model, Group, ContentType] for model in models: info('Gravando marco de [{}]'.format(model.__name__)) - dir_model = Path( - DIR_MARCO, 'dados', model._meta.app_label, model.__name__) + dir_model = dir_dados.child(model._meta.app_label, model.__name__) dir_model.mkdir(parents=True) for data in model.objects.all().values(): - nome_arq = Path(dir_model, data['id']) + nome_arq = Path(dir_model, '{}.yaml'.format(data['id'])) with open(nome_arq, 'w') as arq: pyaml.dump(data, arq) + + # salva mudanças + REPO.git.add([dir_dados.name]) + if 'master' not in REPO.heads or REPO.index.diff('HEAD'): + # se de fato existe mudança + REPO.index.commit('Grava marco') + REPO.git.execute('git tag marco'.split()) diff --git a/sapl/legacy/migracao_documentos.py b/sapl/legacy/migracao_documentos.py index a94b9160a..57446f39c 100644 --- a/sapl/legacy/migracao_documentos.py +++ b/sapl/legacy/migracao_documentos.py @@ -1,6 +1,7 @@ import os import re from glob import glob +from os.path import join import yaml from django.db import transaction @@ -14,7 +15,6 @@ from sapl.parlamentares.models import Parlamentar from sapl.protocoloadm.models import (DocumentoAcessorioAdministrativo, DocumentoAdministrativo) from sapl.sessao.models import SessaoPlenaria -from sapl.settings import MEDIA_ROOT # MIGRAÇÃO DE DOCUMENTOS ################################################### @@ -34,25 +34,41 @@ DOCS = { DocumentoAcessorioAdministrativo: [('arquivo', 'administrativo/{}')], } -DOCS = {model: [(campo, os.path.join('sapl_documentos', origem)) +DOCS = {model: [(campo, join('sapl_documentos', origem)) for campo, origem, in campos] for model, campos in DOCS.items()} -def em_media(caminho): - return os.path.join(MEDIA_ROOT, caminho) - - -def mover_documento(origem, destino): - origem, destino = [em_media(c) if not os.path.isabs(c) else c +def mover_documento(repo, origem, destino): + origem, destino = [join(repo.working_dir, c) if not os.path.isabs(c) else c for c in (origem, destino)] os.makedirs(os.path.dirname(destino), exist_ok=True) - os.rename(origem, destino) + repo.git.mv(origem, destino) + # conserta link do git annex (antes do commit) + # em geral é o mais seguro a fazer, + # mas foi especificamente necessário pois o conteúdo das imagens + # é acessado antes do commit pelo cropping de imagem + repo.git.execute('git annex fix'.split() + [destino]) -def migrar_propriedades_da_casa(): +def migrar_logotipo(repo, casa, propriedades): + print('.... Migrando logotipo da casa ....') + [(campo, origem)] = DOCS[CasaLegislativa] + # a extensão do logo pode ter sido ajustada pelo tipo real do arquivo + nome_nas_propriedades = os.path.splitext(propriedades['id_logo'])[0] + arquivos = glob(join(repo.working_dir, origem.format(nome_nas_propriedades))) + if arquivos: + assert len(arquivos) == 1, 'Há mais de um logotipo para a casa' + [logo] = arquivos + destino = join(CasaLegislativa._meta.get_field(campo).upload_to, + os.path.basename(logo)) + mover_documento(repo, logo, destino) + casa.logotipo = destino + + +def migrar_propriedades_da_casa(repo): print('#### Migrando propriedades da casa ####') - caminho = em_media('sapl_documentos/propriedades.yaml') + caminho = join(repo.working_dir, 'sapl_documentos/propriedades.yaml') with open(caminho, 'r') as arquivo: propriedades = yaml.safe_load(arquivo) casa = CasaLegislativa.objects.first() @@ -72,31 +88,24 @@ def migrar_propriedades_da_casa(): for campo, prop in campos_para_propriedades: setattr(casa, campo, propriedades[prop]) - # Localidade + # localidade sql_localidade = ''' select nom_localidade, sgl_uf from localidade where cod_localidade = {}'''.format(propriedades['cod_localidade']) [(casa.municipio, casa.uf)] = exec_legado(sql_localidade) - print('.... Migrando logotipo da casa ....') - [(campo, origem)] = DOCS[CasaLegislativa] - # a extensão do logo pode ter sido ajustada pelo tipo real do arquivo - id_logo = os.path.splitext(propriedades['id_logo'])[0] - [origem] = glob(em_media(origem.format(id_logo))) - destino = os.path.join( - CasaLegislativa._meta.get_field(campo).upload_to, - os.path.basename(origem)) - mover_documento(origem, destino) - casa.logotipo = destino + # logotipo + migrar_logotipo(repo, casa, propriedades) + casa.save() - os.remove(caminho) + repo.git.rm(caminho) -def migrar_docs_por_ids(model): +def migrar_docs_por_ids(repo, model): for campo, base_origem in DOCS[model]: print('#### Migrando {} de {} ####'.format(campo, model.__name__)) - dir_origem, nome_origem = os.path.split(em_media(base_origem)) + dir_origem, nome_origem = os.path.split(join(repo.working_dir, base_origem)) nome_origem = nome_origem.format('(\d+)') pat = re.compile('^{}\.\w+$'.format(nome_origem)) if not os.path.isdir(dir_origem): @@ -106,7 +115,7 @@ def migrar_docs_por_ids(model): matches = [pat.match(arq) for arq in os.listdir(dir_origem)] ids_origens = [(int(m.group(1)), - os.path.join(dir_origem, m.group(0))) + join(dir_origem, m.group(0))) for m in matches if m] objetos = {obj.id: obj for obj in model.objects.all()} upload_to = model._meta.get_field(campo).upload_to @@ -117,7 +126,7 @@ def migrar_docs_por_ids(model): obj = objetos.get(id) if obj: destino = upload_to(obj, os.path.basename(origem)) - mover_documento(origem, destino) + mover_documento(repo, origem, destino) setattr(obj, campo, destino) obj.save() else: @@ -125,16 +134,15 @@ def migrar_docs_por_ids(model): print(msg.format(model.__name__, id, origem)) -def migrar_documentos(): - # aqui supomos que uma pasta chamada sapl_documentos está em MEDIA_ROOT - # com o conteúdo da pasta de mesmo nome do zope - # Os arquivos da pasta serão MOVIDOS para a nova estrutura! - # A pasta, após conferência do que não foi migrado, deve ser apagada. +def migrar_documentos(repo): + # aqui supomos que uma pasta chamada sapl_documentos está em + # com o conteúdo exportado do zope + # Os arquivos da pasta serão (git) MOVIDOS para a nova estrutura! # # Isto significa que para rodar novamente esta função é preciso # restaurar a pasta sapl_documentos ao estado inicial - migrar_propriedades_da_casa() + migrar_propriedades_da_casa(repo) for model in [ Parlamentar, @@ -146,10 +154,11 @@ def migrar_documentos(): DocumentoAdministrativo, DocumentoAcessorioAdministrativo, ]: - migrar_docs_por_ids(model) + migrar_docs_por_ids(repo, model) - sobrando = [os.path.join(dir, file) - for (dir, _, files) in os.walk(em_media('sapl_documentos')) + sobrando = [join(dir, file) + for (dir, _, files) in os.walk(join(repo.working_dir, + 'sapl_documentos')) for file in files] if sobrando: print('\n#### Encerrado ####\n\n' diff --git a/sapl/legacy/migracao_usuarios.py b/sapl/legacy/migracao_usuarios.py index 106a2def6..be0478c82 100644 --- a/sapl/legacy/migracao_usuarios.py +++ b/sapl/legacy/migracao_usuarios.py @@ -1,8 +1,8 @@ import yaml from django.contrib.auth.models import Group, User +from unipath import Path from sapl.hashers import zope_encoded_password_to_django -from sapl.settings import MEDIA_ROOT PERFIL_LEGADO_PARA_NOVO = {legado: Group.objects.get(name=novo) for legado, novo in [ @@ -44,9 +44,9 @@ def decode_nome(nome): return nome -def migrar_usuarios(): +def migrar_usuarios(dir_repo): """ - Lê o arquivo media/usuarios.yaml e importa os usuários nele listados, + Lê o arquivo /usuarios.yaml e importa os usuários nele listados, com senhas e perfis. Os usuários são criados se necessário e seus perfis ajustados. @@ -68,7 +68,7 @@ def migrar_usuarios(): Também podemos assumir que essa é uma tarefa de um administrador """ - ARQUIVO_USUARIOS = MEDIA_ROOT.child('usuarios.yaml') + ARQUIVO_USUARIOS = Path(dir_repo).child('usuarios.yaml') with open(ARQUIVO_USUARIOS, 'r') as f: usuarios = yaml.load(f) # conferimos de que só há um nome de usuário From d18c606c27fdcb7efd605dc57f58f6e3b3430568 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 19 Apr 2018 18:29:35 -0300 Subject: [PATCH 061/102] =?UTF-8?q?Adiciona=20mais=20tipos=20de=20arquivos?= =?UTF-8?q?=20=C3=A0=20exporta=C3=A7=C3=A3o=20de=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 545e10462..fe95aac31 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -14,14 +14,14 @@ from collections import defaultdict from contextlib import contextmanager from functools import partial -import yaml - import git import magic import pyaml +import yaml +from unipath import Path + import ZODB.DB import ZODB.FileStorage -from unipath import Path from ZODB.broken import Broken EXTENSOES = { @@ -32,10 +32,14 @@ EXTENSOES = { 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx', # noqa 'application/vnd.oasis.opendocument.text-template': '.ott', 'application/vnd.ms-powerpoint': '.ppt', + 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx', # noqa + 'application/vnd.oasis.opendocument.spreadsheet': '.ods', 'application/xml': '.xml', 'text/xml': '.xml', 'application/zip': '.zip', + 'application/x-rar': '.rar', + 'image/jpeg': '.jpeg', 'image/png': '.png', 'image/gif': '.gif', From 38e457620b5712db2ba526f2e1160fe917ef7edd Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 20 Apr 2018 13:41:19 -0300 Subject: [PATCH 062/102] =?UTF-8?q?Grava=20ocorrencias=20de=20migra=C3=A7?= =?UTF-8?q?=C3=A3o=20no=20repo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_dados.py | 17 ++++++----------- 1 file changed, 6 insertions(+), 11 deletions(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 492a8e579..f1b6d0299 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -730,8 +730,8 @@ def reinicia_sequence(model, id): DIR_DADOS_MIGRACAO = Path('~/migracao_sapl/').expand() -PATH_TABELA_TIMEZONES = DIR_DADOS_MIGRACAO.child('tabela_timezones.yaml') -DIR_RESULTADOS = DIR_DADOS_MIGRACAO.child('resultados') +NOME_BANCO_LEGADO = DATABASES['legacy']['NAME'] +REPO = git.Repo.init(Path(DIR_DADOS_MIGRACAO, 'repos', NOME_BANCO_LEGADO)) def dict_representer(dumper, data): @@ -740,9 +740,9 @@ yaml.add_representer(OrderedDict, dict_representer) # configura timezone de migração -nome_banco_legado = DATABASES['legacy']['NAME'] -match = re.match('sapl_cm_(.*)', nome_banco_legado) +match = re.match('sapl_cm_(.*)', NOME_BANCO_LEGADO) sigla_casa = match.group(1) +PATH_TABELA_TIMEZONES = DIR_DADOS_MIGRACAO.child('tabela_timezones.yaml') with open(PATH_TABELA_TIMEZONES, 'r') as arq: tabela_timezones = yaml.load(arq) municipio, uf, nome_timezone = tabela_timezones[sigla_casa] @@ -818,18 +818,16 @@ def migrar_dados(interativo=True): info('Começando migração: ...') try: ocorrencias.clear() - dir_ocorrencias = DIR_RESULTADOS.child(date.today().isoformat()) - dir_ocorrencias.mkdir(parents=True) migrar_todos_os_models() except Exception as e: ocorrencias['traceback'] = str(traceback.format_exc()) raise e finally: # grava ocorrências - arq_ocorrencias = dir_ocorrencias.child( - nome_banco_legado + '.yaml') + arq_ocorrencias = Path(REPO.working_dir, 'ocorrencias.yaml') with open(arq_ocorrencias, 'w') as arq: pyaml.dump(ocorrencias, arq, vspacing=1) + REPO.git.add([arq_ocorrencias.name]) info('Ocorrências salvas em\n {}'.format(arq_ocorrencias)) # recria tipos de autor padrão que não foram criados pela migração @@ -1270,9 +1268,6 @@ def time_constructor(loader, node): yaml.add_constructor(u'!time', time_constructor) -REPO = git.Repo.init(Path(DIR_DADOS_MIGRACAO, 'repos', nome_banco_legado)) - - def gravar_marco(): """Grava um dump de todos os dados como arquivos yaml no repo de marco """ From 8da14380928363dbce957bfe4ba23bed5905fa30 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 20 Apr 2018 14:05:41 -0300 Subject: [PATCH 063/102] =?UTF-8?q?Restaura=20dump=20mysql=20no=20come?= =?UTF-8?q?=C3=A7o=20da=20migra=C3=A7=C3=A3o=20de=20dados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_dados.py | 20 +++++++++++++++++++- sapl/legacy/scripts/migra_um_db.sh | 19 ++----------------- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index f1b6d0299..b7ce154a5 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -1,4 +1,5 @@ import datetime +import os import re import traceback from collections import OrderedDict, defaultdict, namedtuple @@ -27,6 +28,7 @@ from unipath import Path from sapl.base.models import AppConfig as AppConf from sapl.base.models import Autor, TipoAutor, cria_models_tipo_autor from sapl.comissoes.models import Comissao, Composicao, Participacao +from sapl.legacy import scripts from sapl.legacy.models import NormaJuridica as OldNormaJuridica from sapl.legacy.models import TipoNumeracaoProtocolo from sapl.materia.models import (AcompanhamentoMateria, MateriaLegislativa, @@ -785,7 +787,23 @@ def populate_renamed_fields(new, old): setattr(new, field.name, value) +def roda_comando_shell(cmd): + res = os.system(cmd) + assert res == 0, 'O comando falhou: {}'.format(cmd) + + def migrar_dados(interativo=True): + + # restaura dump + arq_dump = Path(DIR_DADOS_MIGRACAO.child( + 'dumps_mysql', '{}.sql'.format(NOME_BANCO_LEGADO))) + assert arq_dump.exists(), 'Dump do mysql faltando: {}'.format(arq_dump) + info('Restaurando dump mysql de [{}]'.format(arq_dump)) + normaliza_dump_mysql = Path(scripts.__file__).parent.child( + 'normaliza_dump_mysql.sh') + roda_comando_shell('{} {}'.format(normaliza_dump_mysql, arq_dump)) + roda_comando_shell('mysql -uroot < {}'.format(arq_dump)) + # executa ajustes pré-migração, se existirem arq_ajustes_pre_migracao = DIR_DADOS_MIGRACAO.child( 'ajustes_pre_migracao', '{}.sql'.format(sigla_casa)) @@ -1292,4 +1310,4 @@ def gravar_marco(): if 'master' not in REPO.heads or REPO.index.diff('HEAD'): # se de fato existe mudança REPO.index.commit('Grava marco') - REPO.git.execute('git tag marco'.split()) + REPO.git.execute('git tag -f marco'.split()) diff --git a/sapl/legacy/scripts/migra_um_db.sh b/sapl/legacy/scripts/migra_um_db.sh index b5b336cd2..dad480704 100755 --- a/sapl/legacy/scripts/migra_um_db.sh +++ b/sapl/legacy/scripts/migra_um_db.sh @@ -1,10 +1,7 @@ #!/bin/bash # rodar esse script na raiz do projeto -if [ $# -ge 2 ]; then - - # proteje pasta com dumps de alterações acidentais - # chmod -R -w ~/migracao_sapl/sapl_dumps +if [ $# -eq 1 ]; then DIR_MIGRACAO=~/migracao_sapl @@ -20,23 +17,11 @@ if [ $# -ge 2 ]; then echo "########################################" | tee -a $LOG echo >> $LOG - if [ $3 ]; then - # se há senha do mysql - mysql -u$2 -p"$3" -N -s -e "DROP DATABASE IF EXISTS $1; CREATE DATABASE $1;" - mysql -u$2 -p"$3" < $DIR_MIGRACAO/dumps_mysql/$1.sql - else - # se não há senha do mysql - mysql -u$2 -N -s -e "DROP DATABASE IF EXISTS $1; CREATE DATABASE $1;" - mysql -u$2 < $DIR_MIGRACAO/dumps_mysql/$1.sql - fi; - echo "O banco legado foi restaurado" |& tee -a $LOG - echo >> $LOG - echo "--- MIGRACAO ---" | tee -a $LOG echo >> $LOG DATABASE_NAME=$1 ./manage.py migracao_25_31 --force --dados --settings sapl.legacy_migration_settings 2>&1 | tee -a $LOG echo >> $LOG else echo "USO:" - echo " $0 [senha mysql]" + echo " $0 " fi; From 3feeb34db0e17d616b48ed20c26d3b59e94e5cc7 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 20 Apr 2018 14:28:34 -0300 Subject: [PATCH 064/102] =?UTF-8?q?Salva=20exporta=C3=A7=C3=A3o=20parcial?= =?UTF-8?q?=20no=20repo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scripts/exporta_zope/exporta_zope.py | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index fe95aac31..6420517f6 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -18,10 +18,9 @@ import git import magic import pyaml import yaml -from unipath import Path - import ZODB.DB import ZODB.FileStorage +from unipath import Path from ZODB.broken import Broken EXTENSOES = { @@ -352,14 +351,20 @@ def dump_sapl(sigla): repo_execute(repo, 'git config annex.thin true') salvar = build_salvar(repo) - _dump_sapl(data_fs_path, destino, salvar) - - # grava mundaças - repo_execute(repo, 'git annex add sapl_documentos') - repo.git.add(A=True) - if 'master' not in repo.heads or repo.index.diff('HEAD'): - # se de fato existe mudança - repo.index.commit('Exporta documentos do zope') + try: + finalizado = False + _dump_sapl(data_fs_path, destino, salvar) + finalizado = True + finally: + # grava mundaças + repo_execute(repo, 'git annex add sapl_documentos') + repo.git.add(A=True) + if 'master' not in repo.heads or repo.index.diff('HEAD'): + # se de fato existe mudança + status = 'completa' if finalizado else 'parcial' + repo.index.commit(u'Exportação do zope {}'.format(status)) + if finalizado: + repo.git.execute('git tag -f zope'.split()) if __name__ == "__main__": From 3fd2ad5dab2f00fb714bd04ca54d5028119deec9 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 20 Apr 2018 14:51:22 -0300 Subject: [PATCH 065/102] =?UTF-8?q?Ignora=20prefixo=20da=20sigla=20na=20ex?= =?UTF-8?q?porta=C3=A7=C3=A3o=20do=20zope?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 6420517f6..c115c7a09 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -340,6 +340,7 @@ def build_salvar(repo): def dump_sapl(sigla): + sigla = sigla[-3:] # ignora prefixo (por ex. 'sapl_cm_') data_fs_path = DIR_DADOS_MIGRACAO.child('datafs', 'Data_cm_{}.fs'.format(sigla)) assert data_fs_path.exists(), 'Origem não existe: {}'.format(data_fs_path) From 133641f27ddcab35fd3591c014a41116ee30e9cb Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 20 Apr 2018 17:36:35 -0300 Subject: [PATCH 066/102] =?UTF-8?q?Checa=20q=20exporta=C3=A7=C3=A3o=20do?= =?UTF-8?q?=20zope=20foi=20feita=20antes=20de=20migrar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao.py | 8 ++++++++ sapl/legacy/migracao_dados.py | 6 ++++-- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 10 ++++------ sapl/legacy/scripts/exporta_zope/variaveis_comuns.py | 4 ++++ 4 files changed, 20 insertions(+), 8 deletions(-) create mode 100644 sapl/legacy/scripts/exporta_zope/variaveis_comuns.py diff --git a/sapl/legacy/migracao.py b/sapl/legacy/migracao.py index a72bd9aaa..3840b1106 100644 --- a/sapl/legacy/migracao.py +++ b/sapl/legacy/migracao.py @@ -6,9 +6,17 @@ from django.conf import settings from sapl.legacy.migracao_dados import REPO, gravar_marco, migrar_dados from sapl.legacy.migracao_documentos import migrar_documentos from sapl.legacy.migracao_usuarios import migrar_usuarios +from sapl.legacy.scripts.exporta_zope.variaveis_comuns import TAG_ZOPE + + +def adornar_msg(msg): + return '\n{1}\n{0}\n{1}'.format(msg, '#' * len(msg)) def migrar(interativo=False): + assert TAG_ZOPE in {t.name for t in REPO.tags}, adornar_msg( + 'Antes de migrar ' + 'é necessário fazer a exportação de documentos do zope') migrar_dados(interativo=interativo) migrar_usuarios(REPO.working_dir) migrar_documentos(REPO) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index b7ce154a5..922cc99a6 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -31,6 +31,8 @@ from sapl.comissoes.models import Comissao, Composicao, Participacao from sapl.legacy import scripts from sapl.legacy.models import NormaJuridica as OldNormaJuridica from sapl.legacy.models import TipoNumeracaoProtocolo +from sapl.legacy.scripts.exporta_zope.variaveis_comuns import \ + DIR_DADOS_MIGRACAO from sapl.materia.models import (AcompanhamentoMateria, MateriaLegislativa, Proposicao, StatusTramitacao, TipoDocumento, TipoMateriaLegislativa, TipoProposicao, @@ -731,9 +733,9 @@ def reinicia_sequence(model, id): sequence_name, id)) -DIR_DADOS_MIGRACAO = Path('~/migracao_sapl/').expand() NOME_BANCO_LEGADO = DATABASES['legacy']['NAME'] -REPO = git.Repo.init(Path(DIR_DADOS_MIGRACAO, 'repos', NOME_BANCO_LEGADO)) +DIR_REPO = Path(DIR_DADOS_MIGRACAO, 'repos', NOME_BANCO_LEGADO) +REPO = git.Repo.init(DIR_REPO) def dict_representer(dumper, data): diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index c115c7a09..b2a3cfe83 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -16,11 +16,12 @@ from functools import partial import git import magic -import pyaml import yaml +from unipath import Path + import ZODB.DB import ZODB.FileStorage -from unipath import Path +from variaveis_comuns import DIR_DADOS_MIGRACAO, TAG_ZOPE from ZODB.broken import Broken EXTENSOES = { @@ -298,9 +299,6 @@ def _dump_sapl(data_fs_path, destino, salvar): close_db() -DIR_DADOS_MIGRACAO = Path('~/migracao_sapl/').expand() - - def repo_execute(repo, cmd, *args): return repo.git.execute(cmd.split() + list(args)) @@ -365,7 +363,7 @@ def dump_sapl(sigla): status = 'completa' if finalizado else 'parcial' repo.index.commit(u'Exportação do zope {}'.format(status)) if finalizado: - repo.git.execute('git tag -f zope'.split()) + repo.git.execute('git tag -f'.split() + [TAG_ZOPE]) if __name__ == "__main__": diff --git a/sapl/legacy/scripts/exporta_zope/variaveis_comuns.py b/sapl/legacy/scripts/exporta_zope/variaveis_comuns.py new file mode 100644 index 000000000..e773f0717 --- /dev/null +++ b/sapl/legacy/scripts/exporta_zope/variaveis_comuns.py @@ -0,0 +1,4 @@ +from unipath import Path + +DIR_DADOS_MIGRACAO = Path('~/migracao_sapl/').expand() +TAG_ZOPE = 'zope' From b0e5162dcd439bb718bb58877b65559a7f42ad31 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 20 Apr 2018 17:42:20 -0300 Subject: [PATCH 067/102] =?UTF-8?q?Reaponta=20MEDIA=5FROOT=20para=20repo?= =?UTF-8?q?=20em=20migra=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_dados.py | 8 +++----- sapl/legacy_migration_settings.py | 8 ++++++++ 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 922cc99a6..301b11458 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -31,8 +31,9 @@ from sapl.comissoes.models import Comissao, Composicao, Participacao from sapl.legacy import scripts from sapl.legacy.models import NormaJuridica as OldNormaJuridica from sapl.legacy.models import TipoNumeracaoProtocolo -from sapl.legacy.scripts.exporta_zope.variaveis_comuns import \ - DIR_DADOS_MIGRACAO +from sapl.legacy_migration_settings import (DATABASES, DIR_DADOS_MIGRACAO, + DIR_REPO, NOME_BANCO_LEGADO, + PROJECT_DIR) from sapl.materia.models import (AcompanhamentoMateria, MateriaLegislativa, Proposicao, StatusTramitacao, TipoDocumento, TipoMateriaLegislativa, TipoProposicao, @@ -45,7 +46,6 @@ from sapl.protocoloadm.models import (DocumentoAdministrativo, Protocolo, StatusTramitacaoAdministrativo) from sapl.sessao.models import (ExpedienteMateria, OrdemDia, RegistroVotacao, TipoResultadoVotacao) -from sapl.settings import DATABASES, PROJECT_DIR from sapl.utils import normalize from .timezonesbrasil import get_timezone @@ -733,8 +733,6 @@ def reinicia_sequence(model, id): sequence_name, id)) -NOME_BANCO_LEGADO = DATABASES['legacy']['NAME'] -DIR_REPO = Path(DIR_DADOS_MIGRACAO, 'repos', NOME_BANCO_LEGADO) REPO = git.Repo.init(DIR_REPO) diff --git a/sapl/legacy_migration_settings.py b/sapl/legacy_migration_settings.py index 0d8844147..96f6a83ad 100644 --- a/sapl/legacy_migration_settings.py +++ b/sapl/legacy_migration_settings.py @@ -3,6 +3,9 @@ import os from decouple import Config, RepositoryEnv from dj_database_url import parse as db_url +from sapl.legacy.scripts.exporta_zope.variaveis_comuns import \ + DIR_DADOS_MIGRACAO + from .settings import * # flake8: noqa config = Config(RepositoryEnv(BASE_DIR.child('legacy', '.env'))) @@ -35,3 +38,8 @@ DEBUG = True HAYSTACK_SIGNAL_PROCESSOR = 'haystack.signals.BaseSignalProcessor' SHELL_PLUS_DONT_LOAD = ['legacy'] + +NOME_BANCO_LEGADO = DATABASES['legacy']['NAME'] +DIR_REPO = Path(DIR_DADOS_MIGRACAO, 'repos', NOME_BANCO_LEGADO) + +MEDIA_ROOT = DIR_REPO From 6f450820bc2c4ccbb90add6b5a59173a5bff89ed Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 20 Apr 2018 17:42:40 -0300 Subject: [PATCH 068/102] Simplifica comando de migracao --- .../management/commands/migracao_25_31.py | 24 ++----------------- sapl/legacy/scripts/migra_um_db.sh | 2 +- 2 files changed, 3 insertions(+), 23 deletions(-) diff --git a/sapl/legacy/management/commands/migracao_25_31.py b/sapl/legacy/management/commands/migracao_25_31.py index 27591e058..4541425c4 100644 --- a/sapl/legacy/management/commands/migracao_25_31.py +++ b/sapl/legacy/management/commands/migracao_25_31.py @@ -1,33 +1,13 @@ from django.core import management from django.core.management.base import BaseCommand -from sapl.legacy.migracao import migrar, migrar_dados +from sapl.legacy.migracao import migrar class Command(BaseCommand): help = 'Migração de dados do SAPL 2.5 para o SAPL 3.1' - def add_arguments(self, parser): - parser.add_argument( - '--force', - action='store_true', - default=False, - dest='force', - help='Não interativa: pula confirmação de exclusão dos dados', - ) - parser.add_argument( - '--dados', - action='store_true', - default=False, - dest='dados', - help='migra somente dados', - ) - def handle(self, *args, **options): management.call_command('migrate') - somente_dados, interativo = options['dados'], not options['force'] - if somente_dados: - migrar_dados(interativo=interativo) - else: - migrar(interativo=interativo) + migrar(interativo=False) diff --git a/sapl/legacy/scripts/migra_um_db.sh b/sapl/legacy/scripts/migra_um_db.sh index dad480704..613fd0a4d 100755 --- a/sapl/legacy/scripts/migra_um_db.sh +++ b/sapl/legacy/scripts/migra_um_db.sh @@ -19,7 +19,7 @@ if [ $# -eq 1 ]; then echo "--- MIGRACAO ---" | tee -a $LOG echo >> $LOG - DATABASE_NAME=$1 ./manage.py migracao_25_31 --force --dados --settings sapl.legacy_migration_settings 2>&1 | tee -a $LOG + DATABASE_NAME=$1 ./manage.py migracao_25_31 --settings sapl.legacy_migration_settings 2>&1 | tee -a $LOG echo >> $LOG else echo "USO:" From 2896a53c761868b7da5684016d058c18f26b9763 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 20 Apr 2018 18:21:57 -0300 Subject: [PATCH 069/102] Gera pacote de migracao no repo --- sapl/legacy/migracao.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/sapl/legacy/migracao.py b/sapl/legacy/migracao.py index 3840b1106..4446324c4 100644 --- a/sapl/legacy/migracao.py +++ b/sapl/legacy/migracao.py @@ -1,12 +1,10 @@ import subprocess -import tarfile - -from django.conf import settings from sapl.legacy.migracao_dados import REPO, gravar_marco, migrar_dados from sapl.legacy.migracao_documentos import migrar_documentos from sapl.legacy.migracao_usuarios import migrar_usuarios from sapl.legacy.scripts.exporta_zope.variaveis_comuns import TAG_ZOPE +from sapl.legacy_migration_settings import DIR_REPO, NOME_BANCO_LEGADO def adornar_msg(msg): @@ -21,25 +19,23 @@ def migrar(interativo=False): migrar_usuarios(REPO.working_dir) migrar_documentos(REPO) gravar_marco() + gerar_pacote() def gerar_pacote(): - banco = settings.DATABASES['legacy']['NAME'] # backup do banco print('Gerando backup do banco... ', end='', flush=True) - arq_backup = settings.MEDIA_ROOT.child('{}.backup'.format(banco)) + arq_backup = DIR_REPO.child('{}.backup'.format(NOME_BANCO_LEGADO)) backup_cmd = ''' pg_dump --host localhost --port 5432 --username postgres --no-password --format custom --blobs --verbose --file {} {}'''.format( - arq_backup, banco) + arq_backup, NOME_BANCO_LEGADO) subprocess.check_output(backup_cmd.split(), stderr=subprocess.DEVNULL) print('SUCESSO') # tar de media/sapl print('Criando tar de media... ', end='', flush=True) - tar_media = settings.MEDIA_ROOT.child('{}.media.tgz'.format(banco)) - dir_media = settings.MEDIA_ROOT.child('sapl') - with tarfile.open(tar_media, "w:gz") as tar: - tar.add(dir_media, arcname=dir_media.name) + arq_tar = DIR_REPO.child('{}.media.tar'.format(NOME_BANCO_LEGADO)) + subprocess.check_output(['tar', 'cfh', arq_tar, '-C', DIR_REPO, 'sapl']) print('SUCESSO') From 8e3977b1abd7d31faae0f9ef2d89cac43e732109 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Mon, 23 Apr 2018 14:50:01 -0300 Subject: [PATCH 070/102] =?UTF-8?q?Corrige=20normaliza=C3=A7=C3=A3o=20de?= =?UTF-8?q?=20dump=20mysql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_dados.py | 21 +++++++---- sapl/legacy/scripts/normaliza_dump_mysql.py | 40 +++++++++++++++++++++ sapl/legacy/scripts/normaliza_dump_mysql.sh | 28 --------------- 3 files changed, 54 insertions(+), 35 deletions(-) create mode 100755 sapl/legacy/scripts/normaliza_dump_mysql.py delete mode 100755 sapl/legacy/scripts/normaliza_dump_mysql.sh diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 301b11458..3fc2fc2d9 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -12,9 +12,12 @@ from subprocess import PIPE, call import git import pkg_resources import pyaml +import yaml +from pyaml import UnsafePrettyYAMLDumper +from unipath import Path + import pytz import reversion -import yaml from django.apps import apps from django.contrib.auth import get_user_model from django.contrib.auth.models import Group @@ -22,9 +25,6 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist from django.db import connections, transaction from django.db.models import Max, Q -from pyaml import UnsafePrettyYAMLDumper -from unipath import Path - from sapl.base.models import AppConfig as AppConf from sapl.base.models import Autor, TipoAutor, cria_models_tipo_autor from sapl.comissoes.models import Comissao, Composicao, Participacao @@ -48,6 +48,7 @@ from sapl.sessao.models import (ExpedienteMateria, OrdemDia, RegistroVotacao, TipoResultadoVotacao) from sapl.utils import normalize +from .scripts.normaliza_dump_mysql import normaliza_dump_mysql from .timezonesbrasil import get_timezone # BASE ###################################################################### @@ -144,6 +145,7 @@ for nome_novo, nome_antigo in (('comissao', 'cod_comissao'), class CampoVirtual(namedtuple('CampoVirtual', 'model related_model')): null = True + CAMPOS_VIRTUAIS_PROPOSICAO = { TipoMateriaLegislativa: CampoVirtual(Proposicao, MateriaLegislativa), TipoDocumento: CampoVirtual(Proposicao, DocumentoAdministrativo) @@ -176,6 +178,7 @@ for related, campo_antigo in [(Parlamentar, 'cod_parlamentar'), def info(msg): print('INFO: ' + msg) + ocorrencias = defaultdict(list) @@ -738,6 +741,8 @@ REPO = git.Repo.init(DIR_REPO) def dict_representer(dumper, data): return dumper.represent_dict(data.items()) + + yaml.add_representer(OrderedDict, dict_representer) @@ -799,9 +804,7 @@ def migrar_dados(interativo=True): 'dumps_mysql', '{}.sql'.format(NOME_BANCO_LEGADO))) assert arq_dump.exists(), 'Dump do mysql faltando: {}'.format(arq_dump) info('Restaurando dump mysql de [{}]'.format(arq_dump)) - normaliza_dump_mysql = Path(scripts.__file__).parent.child( - 'normaliza_dump_mysql.sh') - roda_comando_shell('{} {}'.format(normaliza_dump_mysql, arq_dump)) + normaliza_dump_mysql(arq_dump) roda_comando_shell('mysql -uroot < {}'.format(arq_dump)) # executa ajustes pré-migração, se existirem @@ -1276,6 +1279,8 @@ TIME_FORMAT = '%H:%M:%S' # permite a gravação de tempos puros pelo pretty-yaml def time_representer(dumper, data): return dumper.represent_scalar('!time', data.strftime(TIME_FORMAT)) + + UnsafePrettyYAMLDumper.add_representer(datetime.time, time_representer) @@ -1283,6 +1288,8 @@ UnsafePrettyYAMLDumper.add_representer(datetime.time, time_representer) def time_constructor(loader, node): value = loader.construct_scalar(node) return datetime.datetime.strptime(value, TIME_FORMAT).time() + + yaml.add_constructor(u'!time', time_constructor) diff --git a/sapl/legacy/scripts/normaliza_dump_mysql.py b/sapl/legacy/scripts/normaliza_dump_mysql.py new file mode 100755 index 000000000..a56d74b3a --- /dev/null +++ b/sapl/legacy/scripts/normaliza_dump_mysql.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +import re +import sys + +from unipath import Path + +cabecalho = ''' +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +/*!40000 DROP DATABASE IF EXISTS `{banco}`*/; + +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `{banco}` /*!40100 DEFAULT CHARACTER SET latin1 */; + +USE `{banco}`; + +''' + + +def normaliza_dump_mysql(nome_arquivo): + arquivo = Path(nome_arquivo).expand() + banco = arquivo.stem + conteudo = arquivo.read_file() + inicio = re.finditer('--\n-- Table structure for table .*\n--\n', conteudo) + inicio = next(inicio).start() + conteudo = cabecalho.format(banco=banco) + conteudo[inicio:] + arquivo.write_file(conteudo) + + +if __name__ == "__main__": + nome_aquivo = sys.argv[1] + normaliza_dump_mysql(nome_aquivo) diff --git a/sapl/legacy/scripts/normaliza_dump_mysql.sh b/sapl/legacy/scripts/normaliza_dump_mysql.sh deleted file mode 100755 index 75bd5435f..000000000 --- a/sapl/legacy/scripts/normaliza_dump_mysql.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -ARQUIVO=$1 -BANCO=`basename $1 | cut -f1 -d.` -TMP=__tmp.sql - -cat << EOF > $TMP - -/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; -/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; -/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8 */; -/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; -/*!40103 SET TIME_ZONE='+00:00' */; -/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; -/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; -/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; -/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; - -/*!40000 DROP DATABASE IF EXISTS \`$BANCO\`*/; - -CREATE DATABASE /*!32312 IF NOT EXISTS*/ \`$BANCO\` /*!40100 DEFAULT CHARACTER SET latin1 */; - -USE \`$BANCO\`; -EOF - -sed 1,`grep -n '^USE ' $ARQUIVO |cut -f1 -d:`d $ARQUIVO >> $TMP -mv $TMP $ARQUIVO From 13bf1113da688ac6f4387cc944105465c07f4d6d Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 24 Apr 2018 14:45:37 -0300 Subject: [PATCH 071/102] =?UTF-8?q?Reinicia=20sequence=20pela=20pk=20m?= =?UTF-8?q?=C3=A1xima=20do=20legado?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_dados.py | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 3fc2fc2d9..4b0ad3120 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -12,12 +12,9 @@ from subprocess import PIPE, call import git import pkg_resources import pyaml -import yaml -from pyaml import UnsafePrettyYAMLDumper -from unipath import Path - import pytz import reversion +import yaml from django.apps import apps from django.contrib.auth import get_user_model from django.contrib.auth.models import Group @@ -25,6 +22,9 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist from django.db import connections, transaction from django.db.models import Max, Q +from pyaml import UnsafePrettyYAMLDumper +from unipath import Path + from sapl.base.models import AppConfig as AppConf from sapl.base.models import Autor, TipoAutor, cria_models_tipo_autor from sapl.comissoes.models import Comissao, Composicao, Participacao @@ -725,11 +725,6 @@ def fill_dados_basicos(): appconf.save() -def get_last_pk(model): - last_value = model.objects.all().aggregate(Max('pk')) - return last_value['pk__max'] or 0 - - def reinicia_sequence(model, id): sequence_name = '%s_id_seq' % model._meta.db_table exec_sql('ALTER SEQUENCE %s RESTART WITH %s MINVALUE -1;' % ( @@ -903,10 +898,14 @@ def migrar_model(model): def get_id_do_legado(old): return getattr(old, nome_pk) + + ultima_pk_legado = model_legado.objects.all().aggregate( + Max('pk'))['pk__max'] or 0 else: # a pk no legado tem mais de um campo old_records = iter_sql_records(tabela_legado) get_id_do_legado = None + ultima_pk_legado = model_legado.objects.count() ajuste_antes_salvar = AJUSTE_ANTES_SALVAR.get(model) ajuste_depois_salvar = AJUSTE_DEPOIS_SALVAR.get(model) @@ -949,10 +948,13 @@ def migrar_model(model): if ajuste_depois_salvar: ajuste_depois_salvar() - # se configuramos ids explicitamente devemos reiniciar a sequence + # reiniciamos a sequence logo após a última pk do legado + # + # É importante que seja do legado (e não da nova base), + # pois numa nova versão da migração podemos inserir registros + # não migrados antes sem conflito com pks criadas até lá if get_id_do_legado: - last_pk = get_last_pk(model) - reinicia_sequence(model, last_pk + 1) + reinicia_sequence(model, ultima_pk_legado + 1) # apaga registros migrados do legado if sql_delete_legado: From fdbe2956baace4842a5bd5b67b38f8fc7d2c9a30 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 24 Apr 2018 14:53:54 -0300 Subject: [PATCH 072/102] =?UTF-8?q?Interrompe=20migra=C3=A7=C3=A3o=20j?= =?UTF-8?q?=C3=A1=20realizada?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao.py | 8 ++++++-- sapl/legacy/migracao_dados.py | 4 +++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/sapl/legacy/migracao.py b/sapl/legacy/migracao.py index 4446324c4..4f0d27a08 100644 --- a/sapl/legacy/migracao.py +++ b/sapl/legacy/migracao.py @@ -1,6 +1,7 @@ import subprocess -from sapl.legacy.migracao_dados import REPO, gravar_marco, migrar_dados +from sapl.legacy.migracao_dados import (REPO, TAG_MARCO, gravar_marco, info, + migrar_dados) from sapl.legacy.migracao_documentos import migrar_documentos from sapl.legacy.migracao_usuarios import migrar_usuarios from sapl.legacy.scripts.exporta_zope.variaveis_comuns import TAG_ZOPE @@ -12,7 +13,10 @@ def adornar_msg(msg): def migrar(interativo=False): - assert TAG_ZOPE in {t.name for t in REPO.tags}, adornar_msg( + if TAG_MARCO in REPO.tags: + info('A migração já foi feita.') + return + assert TAG_ZOPE in REPO.tags, adornar_msg( 'Antes de migrar ' 'é necessário fazer a exportação de documentos do zope') migrar_dados(interativo=interativo) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 4b0ad3120..0efb9e52b 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -1294,6 +1294,8 @@ def time_constructor(loader, node): yaml.add_constructor(u'!time', time_constructor) +TAG_MARCO = 'marco' + def gravar_marco(): """Grava um dump de todos os dados como arquivos yaml no repo de marco @@ -1319,4 +1321,4 @@ def gravar_marco(): if 'master' not in REPO.heads or REPO.index.diff('HEAD'): # se de fato existe mudança REPO.index.commit('Grava marco') - REPO.git.execute('git tag -f marco'.split()) + REPO.git.execute('git tag -f'.split() + [TAG_MARCO]) From 8d7542b02b1b35199e9d569b38168908a4722e2d Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 24 Apr 2018 17:34:04 -0300 Subject: [PATCH 073/102] =?UTF-8?q?Reduz=20annex=20get=20apenas=20para=20o?= =?UTF-8?q?=20necess=C3=A1rio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit na migração de documentos --- sapl/legacy/migracao_documentos.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/sapl/legacy/migracao_documentos.py b/sapl/legacy/migracao_documentos.py index 57446f39c..bfd333d40 100644 --- a/sapl/legacy/migracao_documentos.py +++ b/sapl/legacy/migracao_documentos.py @@ -5,6 +5,7 @@ from os.path import join import yaml from django.db import transaction +from image_cropping.fields import ImageCropField from sapl.base.models import CasaLegislativa from sapl.legacy.migracao_dados import exec_legado @@ -44,11 +45,6 @@ def mover_documento(repo, origem, destino): for c in (origem, destino)] os.makedirs(os.path.dirname(destino), exist_ok=True) repo.git.mv(origem, destino) - # conserta link do git annex (antes do commit) - # em geral é o mais seguro a fazer, - # mas foi especificamente necessário pois o conteúdo das imagens - # é acessado antes do commit pelo cropping de imagem - repo.git.execute('git annex fix'.split() + [destino]) def migrar_logotipo(repo, casa, propriedades): @@ -69,6 +65,7 @@ def migrar_logotipo(repo, casa, propriedades): def migrar_propriedades_da_casa(repo): print('#### Migrando propriedades da casa ####') caminho = join(repo.working_dir, 'sapl_documentos/propriedades.yaml') + repo.git.execute('git annex get'.split() + [caminho]) with open(caminho, 'r') as arquivo: propriedades = yaml.safe_load(arquivo) casa = CasaLegislativa.objects.first() @@ -105,7 +102,8 @@ def migrar_docs_por_ids(repo, model): for campo, base_origem in DOCS[model]: print('#### Migrando {} de {} ####'.format(campo, model.__name__)) - dir_origem, nome_origem = os.path.split(join(repo.working_dir, base_origem)) + dir_origem, nome_origem = os.path.split( + join(repo.working_dir, base_origem)) nome_origem = nome_origem.format('(\d+)') pat = re.compile('^{}\.\w+$'.format(nome_origem)) if not os.path.isdir(dir_origem): @@ -119,6 +117,7 @@ def migrar_docs_por_ids(repo, model): for m in matches if m] objetos = {obj.id: obj for obj in model.objects.all()} upload_to = model._meta.get_field(campo).upload_to + tem_cropping = isinstance(model._meta.get_field(campo), ImageCropField) with transaction.atomic(): for id, origem in ids_origens: @@ -128,6 +127,10 @@ def migrar_docs_por_ids(repo, model): destino = upload_to(obj, os.path.basename(origem)) mover_documento(repo, origem, destino) setattr(obj, campo, destino) + if tem_cropping: + # conserta link do git annex (antes do commit) + # pois o conteúdo das imagens é acessado pelo cropping + repo.git.execute('git annex fix'.split() + [destino]) obj.save() else: msg = ' {} (pk={}) não encontrado para documento em [{}]' @@ -144,6 +147,10 @@ def migrar_documentos(repo): migrar_propriedades_da_casa(repo) + # garante que o conteúdo das fotos dos parlamentares esteja presente + # (necessário para o cropping de imagem) + repo.git.execute('git annex get sapl_documentos/parlamentar'.split()) + for model in [ Parlamentar, MateriaLegislativa, From 6973d45dabad0d86197fa0917221620e365de8b1 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 24 Apr 2018 18:45:53 -0300 Subject: [PATCH 074/102] =?UTF-8?q?Adiciona=20exporta=C3=A7=C3=A3o=20de=20?= =?UTF-8?q?DTML=20Method?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scripts/exporta_zope/exporta_zope.py | 28 ++++++++++++------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index b2a3cfe83..2c595844d 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -17,13 +17,13 @@ from functools import partial import git import magic import yaml -from unipath import Path - import ZODB.DB import ZODB.FileStorage -from variaveis_comuns import DIR_DADOS_MIGRACAO, TAG_ZOPE +from unipath import Path from ZODB.broken import Broken +from variaveis_comuns import DIR_DADOS_MIGRACAO, TAG_ZOPE + EXTENSOES = { 'application/msword': '.doc', 'application/pdf': '.pdf', @@ -94,10 +94,7 @@ def guess_extension(fullname, buffer): raise Exception(msg, e) -def dump_file(doc, path, salvar): - name = doc['__name__'] - fullname = os.path.join(path, name) - +def get_conteudo_file(doc): # A partir daqui usamos dict.pop('...') nos __Broken_state__ # para contornar um "vazamento" de memória que ocorre # ao percorrer a árvore de objetos @@ -119,13 +116,23 @@ def dump_file(doc, path, salvar): output.write(pdata.pop('data')) pdata = br(pdata.pop('next', None)) - conteudo = output.getvalue() + return output.getvalue() + + +def dump_file(doc, path, salvar, get_conteudo=get_conteudo_file): + name = doc['__name__'] + fullname = os.path.join(path, name) + conteudo = get_conteudo(doc) if conteudo: # pula arquivos vazios salvar(fullname, conteudo) return name +def get_conteudo_dtml_method(doc): + return doc['raw'] + + def enumerate_by_key_list(folder, key_list, type_key): for entry in folder.get(key_list, []): id, meta_type = entry['id'], entry[type_key] @@ -229,6 +236,8 @@ def dump_sde(strdoc, path, salvar, tipo): DUMP_FUNCTIONS = { 'File': dump_file, 'Image': dump_file, + 'DTML Method': partial(dump_file, + get_conteudo=get_conteudo_dtml_method), 'Folder': partial(dump_folder, enum=enumerate_folder), 'BTreeFolder2': partial(dump_folder, enum=enumerate_btree), 'SDE-Document': partial(dump_sde, tipo='sde.document'), @@ -311,8 +320,7 @@ def get_annex_hashes(repo): def ajusta_extensao(fullname, conteudo): base, extensao = os.path.splitext(fullname) - if extensao not in ['.xsl', '.xslt', '.yaml']: - # não trocamos as extensões XSL, XSLT e YAML + if extensao not in ['.xsl', '.xslt', '.yaml', '.css']: extensao = guess_extension(fullname, conteudo) return base + extensao From 1924ead0875a4e4320d431b55f5ff5fdcd29c7e8 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Wed, 25 Apr 2018 16:04:51 -0300 Subject: [PATCH 075/102] =?UTF-8?q?Interrompe=20exporta=C3=A7=C3=A3o=20de?= =?UTF-8?q?=20docs=20j=C3=A1=20realizada?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao.py | 2 +- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/sapl/legacy/migracao.py b/sapl/legacy/migracao.py index 4f0d27a08..92dd439e1 100644 --- a/sapl/legacy/migracao.py +++ b/sapl/legacy/migracao.py @@ -14,7 +14,7 @@ def adornar_msg(msg): def migrar(interativo=False): if TAG_MARCO in REPO.tags: - info('A migração já foi feita.') + info('A migração já está feita.') return assert TAG_ZOPE in REPO.tags, adornar_msg( 'Antes de migrar ' diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 2c595844d..c7442654e 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -354,6 +354,10 @@ def dump_sapl(sigla): destino = DIR_DADOS_MIGRACAO.child('repos', nome_banco_legado) destino.mkdir(parents=True) repo = git.Repo.init(destino) + if TAG_ZOPE in repo.tags: + info('A exportação de documentos já está feita.') + return + repo_execute(repo, 'git annex init') repo_execute(repo, 'git config annex.thin true') From a1a7d2865e94aca8d8d350b268a1291de1fc5617 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Wed, 25 Apr 2018 20:10:10 -0300 Subject: [PATCH 076/102] Corrige caminhos XSLT p docs exportados do zope Fix #1846 --- sapl/base/urls.py | 19 +++++++++++++++---- sapl/base/views.py | 25 +++++++++++++++++++------ sapl/legacy/migracao_documentos.py | 8 +++++--- sapl/urls.py | 5 +---- 4 files changed, 40 insertions(+), 17 deletions(-) diff --git a/sapl/base/urls.py b/sapl/base/urls.py index 0f720071f..849c18131 100644 --- a/sapl/base/urls.py +++ b/sapl/base/urls.py @@ -1,20 +1,22 @@ +import os + from django.conf.urls import include, url from django.contrib.auth import views from django.contrib.auth.decorators import permission_required from django.contrib.auth.views import (password_reset, password_reset_complete, password_reset_confirm, password_reset_done) -from django.views.generic.base import TemplateView +from django.views.generic.base import RedirectView, TemplateView from sapl.base.views import AutorCrud, ConfirmarEmailView, TipoAutorCrud -from sapl.settings import EMAIL_SEND_USER +from sapl.settings import EMAIL_SEND_USER, MEDIA_URL from .apps import AppConfig from .forms import LoginForm, NovaSenhaForm, RecuperarSenhaForm from .views import (AlterarSenha, AppConfigCrud, CasaLegislativaCrud, CreateUsuarioView, DeleteUsuarioView, EditUsuarioView, - HelpTopicView, ListarUsuarioView, RelatorioAtasView, - RelatorioDataFimPrazoTramitacaoView, + HelpTopicView, ListarUsuarioView, LogotipoView, + RelatorioAtasView, RelatorioDataFimPrazoTramitacaoView, RelatorioHistoricoTramitacaoView, RelatorioMateriasPorAnoAutorTipoView, RelatorioMateriasPorAutorView, @@ -120,4 +122,13 @@ urlpatterns = [ url(r'^sistema/search/', SaplSearchView(), name='haystack_search'), + # Folhas XSLT e extras referenciadas por documentos migrados do sapl 2.5 + url(r'^sapl/XSLT/HTML/(?P.*)$', RedirectView.as_view( + url=os.path.join(MEDIA_URL, 'sapl/public/XSLT/HTML/%(path)s'), + permanent=False)), + # url do logotipo usada em documentos migrados do sapl 2.5 + url(r'^sapl/sapl_documentos/props_sapl/logo_casa', + LogotipoView.as_view(), name='logotipo'), + + ] + recuperar_senha + alterar_senha + admin_user diff --git a/sapl/base/views.py b/sapl/base/views.py index 4dbf50ae2..f0e7d2e1b 100644 --- a/sapl/base/views.py +++ b/sapl/base/views.py @@ -1,5 +1,6 @@ -from django.conf import settings -from django.contrib.auth import get_user_model, update_session_auth_hash +import os + +from django.contrib.auth import get_user_model from django.contrib.auth.mixins import PermissionRequiredMixin from django.contrib.auth.models import Group from django.contrib.auth.tokens import default_token_generator @@ -12,14 +13,15 @@ from django.template import TemplateDoesNotExist from django.template.loader import get_template from django.utils.encoding import force_bytes from django.utils.http import urlsafe_base64_decode, urlsafe_base64_encode -from django.utils.translation import string_concat from django.utils.translation import ugettext_lazy as _ -from django.views.generic import (CreateView, DeleteView, DetailView, FormView, - ListView, UpdateView) -from django.views.generic.base import TemplateView +from django.utils.translation import string_concat +from django.views.generic import (CreateView, DeleteView, FormView, ListView, + UpdateView) +from django.views.generic.base import RedirectView, TemplateView from django_filters.views import FilterView from haystack.views import SearchView +from sapl import settings from sapl.base.forms import AutorForm, AutorFormForAdmin, TipoAutorForm from sapl.base.models import Autor, TipoAutor from sapl.crud.base import CrudAux, make_pagination @@ -759,3 +761,14 @@ class AlterarSenha(FormView): user.save() return super().form_valid(form) + + +STATIC_LOGO = os.path.join(settings.STATIC_URL, 'img/logo.png') + + +class LogotipoView(RedirectView): + + def get_redirect_url(self, *args, **kwargs): + casa = get_casalegislativa() + logo = casa and casa.logotipo and casa.logotipo.name + return os.path.join(settings.MEDIA_URL, logo) if logo else STATIC_LOGO diff --git a/sapl/legacy/migracao_documentos.py b/sapl/legacy/migracao_documentos.py index bfd333d40..2bb17c53e 100644 --- a/sapl/legacy/migracao_documentos.py +++ b/sapl/legacy/migracao_documentos.py @@ -138,12 +138,14 @@ def migrar_docs_por_ids(repo, model): def migrar_documentos(repo): - # aqui supomos que uma pasta chamada sapl_documentos está em + # aqui supomos que as pastas XSLT e sapl_documentos estão em # com o conteúdo exportado do zope - # Os arquivos da pasta serão (git) MOVIDOS para a nova estrutura! + # Os arquivos das pastas serão (git) MOVIDOS para a nova estrutura! # # Isto significa que para rodar novamente esta função é preciso - # restaurar a pasta sapl_documentos ao estado inicial + # restaurar o repo ao estado anterior + + mover_documento(repo, 'XSLT', 'sapl/public/XSLT') migrar_propriedades_da_casa(repo) diff --git a/sapl/urls.py b/sapl/urls.py index b6a5af3e4..07f382013 100644 --- a/sapl/urls.py +++ b/sapl/urls.py @@ -21,6 +21,7 @@ from django.views.generic.base import RedirectView, TemplateView from django.views.static import serve as view_static_server import sapl.api.urls +import sapl.audiencia.urls import sapl.base.urls import sapl.comissoes.urls import sapl.compilacao.urls @@ -33,7 +34,6 @@ import sapl.protocoloadm.urls import sapl.redireciona_urls.urls import sapl.relatorios.urls import sapl.sessao.urls -import sapl.audiencia.urls urlpatterns = [ url(r'^$', TemplateView.as_view(template_name='index.html'), @@ -62,9 +62,6 @@ urlpatterns = [ url(r'^favicon\.ico$', RedirectView.as_view( url='/static/img/favicon.ico', permanent=True)), - # Folhas XSLT e extras referenciadas por documentos migrados do sapl 2.5 - url(r'^XSLT/HTML/(?P.*)$', RedirectView.as_view( - url='/static/XSLT/HTML/%(path)s', permanent=False)), url(r'', include(sapl.redireciona_urls.urls)), ] From b5b528af2fcb673ea954d48ca2950c375c265770 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Wed, 25 Apr 2018 20:35:12 -0300 Subject: [PATCH 077/102] Corrige teste de prefixos de urls --- sapl/test_urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sapl/test_urls.py b/sapl/test_urls.py index 8423a98ed..115572a37 100644 --- a/sapl/test_urls.py +++ b/sapl/test_urls.py @@ -7,7 +7,6 @@ from django.contrib.contenttypes.models import ContentType from django.db import transaction from django.utils.translation import ugettext_lazy as _ from django.utils.translation import string_concat - from sapl.crud.base import PermissionRequiredForAppCrudMixin from sapl.rules.apps import AppConfig, update_groups from scripts.lista_urls import lista_urls @@ -164,7 +163,8 @@ apps_url_patterns_prefixs_and_users = { '/logout', '/ajuda', '/email', - '/recuperar-senha' + '/recuperar-senha', + '/sapl' ]}, 'comissoes': { 'users': {'operador_geral': ['/sistema', '/comissao'], From 499877255f82d14384c2565a28e74e7a1035e49e Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 26 Apr 2018 15:59:15 -0300 Subject: [PATCH 078/102] =?UTF-8?q?Retira=20c=C3=B3digo=20desnecess=C3=A1r?= =?UTF-8?q?io?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/static/XSLT/HTML/.objects | 11 -- sapl/static/XSLT/HTML/estilo.css | 95 ---------------- sapl/static/XSLT/HTML/indicacao.xsl | 51 --------- sapl/static/XSLT/HTML/mocao.xsl | 41 ------- sapl/static/XSLT/HTML/mocao2.xsl | 45 -------- sapl/static/XSLT/HTML/parecer.xsl | 41 ------- sapl/static/XSLT/HTML/pedido.xsl | 47 -------- sapl/static/XSLT/HTML/pedido2.xsl | 53 --------- sapl/static/XSLT/HTML/pl.xsl | 105 ------------------ sapl/static/XSLT/HTML/pl2.xsl | 100 ----------------- sapl/static/XSLT/HTML/requerimento.xsl | 52 --------- sapl/static/XSLT/HTML/requerimento2.xsl | 57 ---------- .../sessao/blocos_resumo/expedientes.html | 2 +- 13 files changed, 1 insertion(+), 699 deletions(-) delete mode 100644 sapl/static/XSLT/HTML/.objects delete mode 100644 sapl/static/XSLT/HTML/estilo.css delete mode 100644 sapl/static/XSLT/HTML/indicacao.xsl delete mode 100644 sapl/static/XSLT/HTML/mocao.xsl delete mode 100644 sapl/static/XSLT/HTML/mocao2.xsl delete mode 100644 sapl/static/XSLT/HTML/parecer.xsl delete mode 100644 sapl/static/XSLT/HTML/pedido.xsl delete mode 100644 sapl/static/XSLT/HTML/pedido2.xsl delete mode 100644 sapl/static/XSLT/HTML/pl.xsl delete mode 100644 sapl/static/XSLT/HTML/pl2.xsl delete mode 100644 sapl/static/XSLT/HTML/requerimento.xsl delete mode 100644 sapl/static/XSLT/HTML/requerimento2.xsl diff --git a/sapl/static/XSLT/HTML/.objects b/sapl/static/XSLT/HTML/.objects deleted file mode 100644 index ca1ce01f2..000000000 --- a/sapl/static/XSLT/HTML/.objects +++ /dev/null @@ -1,11 +0,0 @@ -estilo.css:DTML Method -indicacao.xsl:File -mocao.xsl:File -mocao2.xsl:File -parecer.xsl:File -pedido.xsl:File -pedido2.xsl:File -pl.xsl:File -pl2.xsl:File -requerimento.xsl:File -requerimento2.xsl:File diff --git a/sapl/static/XSLT/HTML/estilo.css b/sapl/static/XSLT/HTML/estilo.css deleted file mode 100644 index aac85f176..000000000 --- a/sapl/static/XSLT/HTML/estilo.css +++ /dev/null @@ -1,95 +0,0 @@ - body { - font-family: Times; - text-align: justify; - font-size: 12 pt; - margin: 5px 1cm 20px 2cm; -} - - p, - .p{ - font-family: Times; - text-align: justify; - font-size: 12pt; - text-indent: 1.5cm; - margin: 40px 0 20px 0; - } - - .pequeno { - font-family: Times; - text-align: left; - font-size: 13pt; - margin: 0px 0 0px 0; - } - - .cabecalho { - font-family: Times; - font-weight:bold; - text-align: left; - font-size: 15pt; - margin: 10px 0 0px 0; - } - - .texto { - font-family: Times; - text-align: justify; - font-size: 12pt; - margin: 0px 0px 0px 0px; - } - - .data { - text-align: right; - } - - .autor { - text-align: center; - } - - .center { - text-align: center; - } - - .semrecuo { - text-indent: 0; - } - - .ementa { - text-align: justify; - margin-left: 50%; - text-indent: 0; - } - - .titulos1 { - text-align: center; - margin: 10px 0 0px 0; - } - - .titulos2 { - text-align: center; - margin: 0px 0 0px 0; - } - - p.artigo { - text-align: justify; - text-indent: 1cm; - margin: 10px 0 0px 0; -} - - -#imagem { - float:left; - } - -#autores - { - -moz-column-count:3; /* Firefox */ - -webkit-column-count:3; /* Safari and Chrome */ - width:50px; - - } - -#col1 { width: 33%; float: left; center: 10px; } -#col2 { width: 33%; float: left; center: 10px; } -#col3 { width: 33%; float: left; center: 10px; } - - - diff --git a/sapl/static/XSLT/HTML/indicacao.xsl b/sapl/static/XSLT/HTML/indicacao.xsl deleted file mode 100644 index edb4575c7..000000000 --- a/sapl/static/XSLT/HTML/indicacao.xsl +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - - - - - - - - - - -
-
-

-


-

Câmara Municipal de Agudo

-

Estado do Rio Grande do Sul





-
-

- - - -

- - - -

- - - -

- - - -

- - - -

- - - diff --git a/sapl/static/XSLT/HTML/mocao.xsl b/sapl/static/XSLT/HTML/mocao.xsl deleted file mode 100644 index adb7ef3c9..000000000 --- a/sapl/static/XSLT/HTML/mocao.xsl +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/sapl/static/XSLT/HTML/mocao2.xsl b/sapl/static/XSLT/HTML/mocao2.xsl deleted file mode 100644 index 14f71c982..000000000 --- a/sapl/static/XSLT/HTML/mocao2.xsl +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - - - - -
-
- -


-

Câmara Municipal de Agudo

-

Estado do Rio Grande do Sul





-
- - - - - - -

- - - -

- - - -

- - - -

- - - -

- - - \ No newline at end of file diff --git a/sapl/static/XSLT/HTML/parecer.xsl b/sapl/static/XSLT/HTML/parecer.xsl deleted file mode 100644 index 14100320e..000000000 --- a/sapl/static/XSLT/HTML/parecer.xsl +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/sapl/static/XSLT/HTML/pedido.xsl b/sapl/static/XSLT/HTML/pedido.xsl deleted file mode 100644 index 9e5002d01..000000000 --- a/sapl/static/XSLT/HTML/pedido.xsl +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/sapl/static/XSLT/HTML/pedido2.xsl b/sapl/static/XSLT/HTML/pedido2.xsl deleted file mode 100644 index facc154c6..000000000 --- a/sapl/static/XSLT/HTML/pedido2.xsl +++ /dev/null @@ -1,53 +0,0 @@ - - - - - - - - - - -
-
- -


-

Câmara Municipal de Agudo

-

Estado do Rio Grande do Sul





-
- - - - - - -

- - - -

- - - -

- - - -

- - - -

- - - -

- - - -

- - - \ No newline at end of file diff --git a/sapl/static/XSLT/HTML/pl.xsl b/sapl/static/XSLT/HTML/pl.xsl deleted file mode 100644 index 0da6eddb4..000000000 --- a/sapl/static/XSLT/HTML/pl.xsl +++ /dev/null @@ -1,105 +0,0 @@ - - - - - - - - <xsl:value-of select="@id"/> - - - - - - - - - -
-

PROPOSIÇÃO

-
- - - -

- -

- - -

- -

- - -

- -

- - -

- -

- - -

- -

- - -

- -

- - -

- -

- - -

- -

- - -

- -

- - -

- -

- - -
-

JUSTIFICATIVA

-
-

- -

- - -
-

MENSAGEM

-
-

- -

- - \ No newline at end of file diff --git a/sapl/static/XSLT/HTML/pl2.xsl b/sapl/static/XSLT/HTML/pl2.xsl deleted file mode 100644 index f68506f95..000000000 --- a/sapl/static/XSLT/HTML/pl2.xsl +++ /dev/null @@ -1,100 +0,0 @@ - - - - - - - - - - -
-
- -


-

Câmara Municipal de Agudo

-

Estado do Rio Grande do Sul





-
- - - - - - - -

- - - -

- - - -

- - - -

-

- - - -

-

- - - -

-

- - - -

-

- - - -

-

- - - -

-

- - - -

- - - -

- - - -

- - - -

- - - -

- - - -

- - - -

- - - -

- - - \ No newline at end of file diff --git a/sapl/static/XSLT/HTML/requerimento.xsl b/sapl/static/XSLT/HTML/requerimento.xsl deleted file mode 100644 index 58f202b09..000000000 --- a/sapl/static/XSLT/HTML/requerimento.xsl +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - - - -
- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/sapl/static/XSLT/HTML/requerimento2.xsl b/sapl/static/XSLT/HTML/requerimento2.xsl deleted file mode 100644 index 3d63f6cf8..000000000 --- a/sapl/static/XSLT/HTML/requerimento2.xsl +++ /dev/null @@ -1,57 +0,0 @@ - - - - - - - - - - -
-
- -


-

Câmara Municipal de Agudo

-

Estado do Rio Grande do Sul





-
- - - - - - -

- - - -

- - - -

- - - -

- - - -

- - - -

- - - -

- - - -

- - - \ No newline at end of file diff --git a/sapl/templates/sessao/blocos_resumo/expedientes.html b/sapl/templates/sessao/blocos_resumo/expedientes.html index e233fcb58..5462fd3f3 100644 --- a/sapl/templates/sessao/blocos_resumo/expedientes.html +++ b/sapl/templates/sessao/blocos_resumo/expedientes.html @@ -6,7 +6,7 @@ {{e.tipo}}:

-
+

{{e.conteudo|safe}}

From 68dd711d7f318e27f01f8a42358d38966a1b0a62 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 26 Apr 2018 15:59:42 -0300 Subject: [PATCH 079/102] Remove style ao migrar ExpedienteSessao Fix #1844 --- sapl/legacy/migracao_dados.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 0efb9e52b..e40406447 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -15,6 +15,7 @@ import pyaml import pytz import reversion import yaml +from bs4 import BeautifulSoup from django.apps import apps from django.contrib.auth import get_user_model from django.contrib.auth.models import Group @@ -44,8 +45,8 @@ from sapl.parlamentares.models import (Legislatura, Mandato, Parlamentar, Partido, TipoAfastamento) from sapl.protocoloadm.models import (DocumentoAdministrativo, Protocolo, StatusTramitacaoAdministrativo) -from sapl.sessao.models import (ExpedienteMateria, OrdemDia, RegistroVotacao, - TipoResultadoVotacao) +from sapl.sessao.models import (ExpedienteMateria, ExpedienteSessao, OrdemDia, + RegistroVotacao, TipoResultadoVotacao) from sapl.utils import normalize from .scripts.normaliza_dump_mysql import normaliza_dump_mysql @@ -1245,6 +1246,21 @@ def adjust_tiporesultadovotacao(new, old): {'pk': new.pk, 'nome': new.nome}) +def remove_style(conteudo): + if 'style' not in conteudo: + return conteudo # atalho que acelera muito os casos sem style + + soup = BeautifulSoup(conteudo, 'html.parser') + for tag in soup.recursiveChildGenerator(): + if hasattr(tag, 'attrs'): + tag.attrs = {k: v for k, v in tag.attrs.items() if k != 'style'} + return str(soup) + + +def adjust_expediente_sessao(new, old): + new.conteudo = remove_style(new.conteudo) + + AJUSTE_ANTES_SALVAR = { Autor: adjust_autor, TipoAutor: adjust_tipo_autor, @@ -1266,6 +1282,7 @@ AJUSTE_ANTES_SALVAR = { StatusTramitacaoAdministrativo: adjust_statustramitacaoadm, Tramitacao: adjust_tramitacao, TipoResultadoVotacao: adjust_tiporesultadovotacao, + ExpedienteSessao: adjust_expediente_sessao, } AJUSTE_DEPOIS_SALVAR = { From 022c6f478d4ca5b87b979139fe35a46c36461c07 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 26 Apr 2018 17:35:45 -0300 Subject: [PATCH 080/102] Remover pacote de migracao antes de gerar --- sapl/legacy/migracao.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sapl/legacy/migracao.py b/sapl/legacy/migracao.py index 92dd439e1..66668a62f 100644 --- a/sapl/legacy/migracao.py +++ b/sapl/legacy/migracao.py @@ -31,6 +31,7 @@ def gerar_pacote(): # backup do banco print('Gerando backup do banco... ', end='', flush=True) arq_backup = DIR_REPO.child('{}.backup'.format(NOME_BANCO_LEGADO)) + arq_backup.remove() backup_cmd = ''' pg_dump --host localhost --port 5432 --username postgres --no-password --format custom --blobs --verbose --file {} {}'''.format( @@ -41,5 +42,6 @@ def gerar_pacote(): # tar de media/sapl print('Criando tar de media... ', end='', flush=True) arq_tar = DIR_REPO.child('{}.media.tar'.format(NOME_BANCO_LEGADO)) + arq_tar.remove() subprocess.check_output(['tar', 'cfh', arq_tar, '-C', DIR_REPO, 'sapl']) print('SUCESSO') From 4940f72194b6be3ae1ef149d4975b4f71d5ad4f1 Mon Sep 17 00:00:00 2001 From: Edward Date: Fri, 27 Apr 2018 13:08:03 -0300 Subject: [PATCH 081/102] Fixes #1915 (#1916) --- sapl/materia/forms.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sapl/materia/forms.py b/sapl/materia/forms.py index 3d455ea29..e083e7fec 100644 --- a/sapl/materia/forms.py +++ b/sapl/materia/forms.py @@ -582,8 +582,18 @@ class AnexadaForm(ModelForm): msg = _('A matéria a ser anexada não existe no cadastro' ' de matérias legislativas.') raise ValidationError(msg) - else: - cleaned_data['materia_anexada'] = materia_anexada + + materia_principal = self.instance.materia_principal + if materia_principal == materia_anexada: + raise ValidationError(_('Matéria não pode ser anexada a si mesma')) + + is_anexada = Anexada.objects.filter(materia_principal=materia_principal, + materia_anexada=materia_anexada + ).exists() + if is_anexada: + raise ValidationError(_('Materia já se encontra anexada')) + + cleaned_data['materia_anexada'] = materia_anexada return cleaned_data From e7e57c4e55be256075fd03e0667b73017502ae13 Mon Sep 17 00:00:00 2001 From: Edward Date: Fri, 27 Apr 2018 13:08:03 -0300 Subject: [PATCH 082/102] Fixes #1915 (#1916) --- sapl/materia/forms.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/sapl/materia/forms.py b/sapl/materia/forms.py index 3d455ea29..e083e7fec 100644 --- a/sapl/materia/forms.py +++ b/sapl/materia/forms.py @@ -582,8 +582,18 @@ class AnexadaForm(ModelForm): msg = _('A matéria a ser anexada não existe no cadastro' ' de matérias legislativas.') raise ValidationError(msg) - else: - cleaned_data['materia_anexada'] = materia_anexada + + materia_principal = self.instance.materia_principal + if materia_principal == materia_anexada: + raise ValidationError(_('Matéria não pode ser anexada a si mesma')) + + is_anexada = Anexada.objects.filter(materia_principal=materia_principal, + materia_anexada=materia_anexada + ).exists() + if is_anexada: + raise ValidationError(_('Materia já se encontra anexada')) + + cleaned_data['materia_anexada'] = materia_anexada return cleaned_data From c5e9d8ab5e304f02af2b913bed4c2bc143aaddab Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Fri, 27 Apr 2018 14:53:31 -0300 Subject: [PATCH 083/102] =?UTF-8?q?HOT-FIX:=20melhora=20o=20desempenho=20d?= =?UTF-8?q?e=20pesquisas=20na=20tela=20de=20relat=C3=B3rios?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/base/forms.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sapl/base/forms.py b/sapl/base/forms.py index 1a0637a22..513c3a7ca 100644 --- a/sapl/base/forms.py +++ b/sapl/base/forms.py @@ -525,6 +525,11 @@ class RelatorioAtasFilterSet(django_filters.FilterSet): model = SessaoPlenaria fields = ['data_inicio'] + @property + def qs(self): + parent = super(RelatorioAtasFilterSet, self).qs + return parent.distinct().prefetch_related('tipo').order_by('-ano', 'tipo', 'numero') + def __init__(self, *args, **kwargs): super(RelatorioAtasFilterSet, self).__init__( *args, **kwargs) @@ -588,7 +593,7 @@ class RelatorioHistoricoTramitacaoFilterSet(django_filters.FilterSet): @property def qs(self): parent = super(RelatorioHistoricoTramitacaoFilterSet, self).qs - return parent.distinct().order_by('-ano', 'tipo', 'numero') + return parent.distinct().prefetch_related('tipo').order_by('-ano', 'tipo', 'numero') class Meta: model = MateriaLegislativa @@ -628,7 +633,7 @@ class RelatorioDataFimPrazoTramitacaoFilterSet(django_filters.FilterSet): @property def qs(self): parent = super(RelatorioDataFimPrazoTramitacaoFilterSet, self).qs - return parent.distinct().order_by('-ano', 'tipo', 'numero') + return parent.distinct().prefetch_related('tipo').order_by('-ano', 'tipo', 'numero') class Meta: model = MateriaLegislativa From ff1c86b119acd64456d4a79f8f6aaaeca7b602fe Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Fri, 27 Apr 2018 14:53:31 -0300 Subject: [PATCH 084/102] =?UTF-8?q?HOT-FIX:=20melhora=20o=20desempenho=20d?= =?UTF-8?q?e=20pesquisas=20na=20tela=20de=20relat=C3=B3rios?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/base/forms.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sapl/base/forms.py b/sapl/base/forms.py index 1a0637a22..513c3a7ca 100644 --- a/sapl/base/forms.py +++ b/sapl/base/forms.py @@ -525,6 +525,11 @@ class RelatorioAtasFilterSet(django_filters.FilterSet): model = SessaoPlenaria fields = ['data_inicio'] + @property + def qs(self): + parent = super(RelatorioAtasFilterSet, self).qs + return parent.distinct().prefetch_related('tipo').order_by('-ano', 'tipo', 'numero') + def __init__(self, *args, **kwargs): super(RelatorioAtasFilterSet, self).__init__( *args, **kwargs) @@ -588,7 +593,7 @@ class RelatorioHistoricoTramitacaoFilterSet(django_filters.FilterSet): @property def qs(self): parent = super(RelatorioHistoricoTramitacaoFilterSet, self).qs - return parent.distinct().order_by('-ano', 'tipo', 'numero') + return parent.distinct().prefetch_related('tipo').order_by('-ano', 'tipo', 'numero') class Meta: model = MateriaLegislativa @@ -628,7 +633,7 @@ class RelatorioDataFimPrazoTramitacaoFilterSet(django_filters.FilterSet): @property def qs(self): parent = super(RelatorioDataFimPrazoTramitacaoFilterSet, self).qs - return parent.distinct().order_by('-ano', 'tipo', 'numero') + return parent.distinct().prefetch_related('tipo').order_by('-ano', 'tipo', 'numero') class Meta: model = MateriaLegislativa From 47d295c90c8dbe09edf3d026fe2e25f1a6c6e2cf Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 27 Apr 2018 15:23:23 -0300 Subject: [PATCH 085/102] =?UTF-8?q?Adiciona=20scraping=20de=20proposi?= =?UTF-8?q?=C3=A7=C3=B5es=20do=20SDE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/base/urls.py | 4 ++-- sapl/legacy/migracao.py | 40 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 2 deletions(-) diff --git a/sapl/base/urls.py b/sapl/base/urls.py index 849c18131..b52d46ad6 100644 --- a/sapl/base/urls.py +++ b/sapl/base/urls.py @@ -123,11 +123,11 @@ urlpatterns = [ url(r'^sistema/search/', SaplSearchView(), name='haystack_search'), # Folhas XSLT e extras referenciadas por documentos migrados do sapl 2.5 - url(r'^sapl/XSLT/HTML/(?P.*)$', RedirectView.as_view( + url(r'^(sapl/)?XSLT/HTML/(?P.*)$', RedirectView.as_view( url=os.path.join(MEDIA_URL, 'sapl/public/XSLT/HTML/%(path)s'), permanent=False)), # url do logotipo usada em documentos migrados do sapl 2.5 - url(r'^sapl/sapl_documentos/props_sapl/logo_casa', + url(r'^(sapl/)?sapl_documentos/props_sapl/logo_casa', LogotipoView.as_view(), name='logotipo'), diff --git a/sapl/legacy/migracao.py b/sapl/legacy/migracao.py index 66668a62f..4509d8ba6 100644 --- a/sapl/legacy/migracao.py +++ b/sapl/legacy/migracao.py @@ -1,4 +1,8 @@ import subprocess +from getpass import getpass + +import requests +from unipath import Path from sapl.legacy.migracao_dados import (REPO, TAG_MARCO, gravar_marco, info, migrar_dados) @@ -6,6 +10,7 @@ from sapl.legacy.migracao_documentos import migrar_documentos from sapl.legacy.migracao_usuarios import migrar_usuarios from sapl.legacy.scripts.exporta_zope.variaveis_comuns import TAG_ZOPE from sapl.legacy_migration_settings import DIR_REPO, NOME_BANCO_LEGADO +from sapl.materia.models import Proposicao def adornar_msg(msg): @@ -45,3 +50,38 @@ def gerar_pacote(): arq_tar.remove() subprocess.check_output(['tar', 'cfh', arq_tar, '-C', DIR_REPO, 'sapl']) print('SUCESSO') + + +PROPOSICAO_UPLOAD_TO = Proposicao._meta.get_field('texto_original').upload_to + + +def salva_conteudo_do_sde(proposicao, conteudo): + caminho_relativo = PROPOSICAO_UPLOAD_TO( + proposicao, 'proposicao_sde_{}.xml'.format(proposicao.pk)) + caminho_absoluto = Path(REPO.working_dir, caminho_relativo) + caminho_absoluto.parent.mkdir(parents=True) + with open(caminho_absoluto, 'wb') as arq: + arq.write(conteudo) + proposicao.texto_original = caminho_relativo + proposicao.save() + + +def scrap_sde(url, usuario, senha=None): + if not senha: + senha = getpass() + + # login + session = requests.session() + res = session.post('{}?retry=1'.format(url), + {'__ac_name': usuario, '__ac_password': senha}) + assert res.status_code == 200 + + url_proposicao = '{}/sapl_documentos/proposicao/{}/renderXML?xsl=__default__' # noqa + total = Proposicao.objects.count() + for num, proposicao in enumerate(Proposicao.objects.all()): + pk = proposicao.pk + res = session.get(url_proposicao.format(url, pk)) + print("pk: {} status: {} (progresso: {:.2%})".format( + pk, res.status_code, num / total)) + if res.status_code == 200: + salva_conteudo_do_sde(proposicao, res.content) From 0b4ead32702a033ca810c8d0c5bdf7df05671b09 Mon Sep 17 00:00:00 2001 From: Mariana Mendes Date: Wed, 2 May 2018 14:17:40 -0300 Subject: [PATCH 086/102] fix #1898 (#1917) * Fix #1898 * Fix #1898 * Add "remover_acentos" to the sapl utils * Remove the import unicodedata --- sapl/sessao/views.py | 6 +++--- sapl/utils.py | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/sapl/sessao/views.py b/sapl/sessao/views.py index d7844b3ce..5beec02ad 100644 --- a/sapl/sessao/views.py +++ b/sapl/sessao/views.py @@ -34,7 +34,7 @@ from sapl.parlamentares.models import (Filiacao, Legislatura, Mandato, Parlamentar, SessaoLegislativa) from sapl.sessao.apps import AppConfig from sapl.sessao.forms import ExpedienteMateriaForm, OrdemDiaForm -from sapl.utils import show_results_filter_set +from sapl.utils import show_results_filter_set, remover_acentos from .forms import (AdicionarVariasMateriasFilterSet, BancadaForm, BlocoForm, ExpedienteForm, ListMateriaForm, MesaForm, @@ -385,14 +385,14 @@ def customize_link_materia(context, pk, has_permission, is_expediente): context['rows'][i][3] = (resultado, None) return context - + def get_presencas_generic(model, sessao, legislatura): presencas = model.objects.filter( sessao_plenaria=sessao) presentes = [p.parlamentar for p in presencas] - presentes = sorted(presentes, key=lambda x: x.nome_parlamentar) + presentes = sorted(presentes, key=lambda x: remover_acentos(x.nome_parlamentar)) mandato = Mandato.objects.filter( legislatura=legislatura).order_by('parlamentar__nome_parlamentar') diff --git a/sapl/utils.py b/sapl/utils.py index 0fa20f706..8045f0e0b 100644 --- a/sapl/utils.py +++ b/sapl/utils.py @@ -2,6 +2,7 @@ import hashlib import logging import os import re +import unicodedata from functools import wraps from operator import itemgetter from unicodedata import normalize as unicodedata_normalize @@ -732,3 +733,7 @@ def RemoveTag(texto): i += 1 return textoSaida + +def remover_acentos(string): + return ''.join([c for c in unicodedata.normalize('NFD', string) + if unicodedata.category(c) != 'Mn']) \ No newline at end of file From cfb218f5dab5e62ae6c966de0614c2b4b8ad76be Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Wed, 2 May 2018 14:35:29 -0300 Subject: [PATCH 087/102] Release: 3.1.76 --- docker-compose.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 48e9a3b30..f721b66f0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ sapldb: ports: - "5432:5432" sapl: - image: interlegis/sapl:3.1.75 + image: interlegis/sapl:3.1.76 restart: always environment: ADMIN_PASSWORD: interlegis diff --git a/setup.py b/setup.py index edf2dc502..da929d87c 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ install_requires = [ ] setup( name='interlegis-sapl', - version='3.1.75', + version='3.1.76', packages=find_packages(), include_package_data=True, license='GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007', From dd6e7cc964fb2484cc855f8739d92e641df6f5bf Mon Sep 17 00:00:00 2001 From: Mariana Mendes Date: Wed, 2 May 2018 14:17:40 -0300 Subject: [PATCH 088/102] fix #1898 (#1917) * Fix #1898 * Fix #1898 * Add "remover_acentos" to the sapl utils * Remove the import unicodedata --- sapl/sessao/views.py | 6 +++--- sapl/utils.py | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/sapl/sessao/views.py b/sapl/sessao/views.py index d7844b3ce..5beec02ad 100644 --- a/sapl/sessao/views.py +++ b/sapl/sessao/views.py @@ -34,7 +34,7 @@ from sapl.parlamentares.models import (Filiacao, Legislatura, Mandato, Parlamentar, SessaoLegislativa) from sapl.sessao.apps import AppConfig from sapl.sessao.forms import ExpedienteMateriaForm, OrdemDiaForm -from sapl.utils import show_results_filter_set +from sapl.utils import show_results_filter_set, remover_acentos from .forms import (AdicionarVariasMateriasFilterSet, BancadaForm, BlocoForm, ExpedienteForm, ListMateriaForm, MesaForm, @@ -385,14 +385,14 @@ def customize_link_materia(context, pk, has_permission, is_expediente): context['rows'][i][3] = (resultado, None) return context - + def get_presencas_generic(model, sessao, legislatura): presencas = model.objects.filter( sessao_plenaria=sessao) presentes = [p.parlamentar for p in presencas] - presentes = sorted(presentes, key=lambda x: x.nome_parlamentar) + presentes = sorted(presentes, key=lambda x: remover_acentos(x.nome_parlamentar)) mandato = Mandato.objects.filter( legislatura=legislatura).order_by('parlamentar__nome_parlamentar') diff --git a/sapl/utils.py b/sapl/utils.py index 0fa20f706..8045f0e0b 100644 --- a/sapl/utils.py +++ b/sapl/utils.py @@ -2,6 +2,7 @@ import hashlib import logging import os import re +import unicodedata from functools import wraps from operator import itemgetter from unicodedata import normalize as unicodedata_normalize @@ -732,3 +733,7 @@ def RemoveTag(texto): i += 1 return textoSaida + +def remover_acentos(string): + return ''.join([c for c in unicodedata.normalize('NFD', string) + if unicodedata.category(c) != 'Mn']) \ No newline at end of file From 0c1ef242baee3b103c782c971a581703c8da8106 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Wed, 2 May 2018 15:34:37 -0300 Subject: [PATCH 089/102] =?UTF-8?q?Adiciona=20um=20script=20simples=20para?= =?UTF-8?q?=20gerar=20vers=C3=B5es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- release.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 release.sh diff --git a/release.sh b/release.sh new file mode 100755 index 000000000..78fee9187 --- /dev/null +++ b/release.sh @@ -0,0 +1,19 @@ +#/bin/bash + +VERSION=`git describe --tags --abbrev=0` +LAST_DIGIT=`echo $VERSION | cut -f 3 -d '.'` +MAIN_REV=`echo $VERSION | cut -f 1,2 -d '.'` +NEXT_NUMBER=$(($LAST_DIGIT + 1)) +NEXT_VERSION=$MAIN_REV'.'$NEXT_NUMBER + +sed -e s/$VERSION/$NEXT_VERSION/g docker-compose.yml > tmp1 +mv tmp1 docker-compose.yml + +sed -e s/$VERSION/$NEXT_VERSION/g setup.py > tmp2 +mv tmp2 setup.py + +git add docker-compose.yml setup.py +git commit -m "Release: $NEXT_VERSION" +git tag $NEXT_VERSION +git push origin $NEXT_VERSION +git push origin From 2520ed2f82f3e095fba7eaf3e8eeacd6404c50d6 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Wed, 2 May 2018 15:34:37 -0300 Subject: [PATCH 090/102] =?UTF-8?q?Adiciona=20um=20script=20simples=20para?= =?UTF-8?q?=20gerar=20vers=C3=B5es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- release.sh | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 release.sh diff --git a/release.sh b/release.sh new file mode 100755 index 000000000..78fee9187 --- /dev/null +++ b/release.sh @@ -0,0 +1,19 @@ +#/bin/bash + +VERSION=`git describe --tags --abbrev=0` +LAST_DIGIT=`echo $VERSION | cut -f 3 -d '.'` +MAIN_REV=`echo $VERSION | cut -f 1,2 -d '.'` +NEXT_NUMBER=$(($LAST_DIGIT + 1)) +NEXT_VERSION=$MAIN_REV'.'$NEXT_NUMBER + +sed -e s/$VERSION/$NEXT_VERSION/g docker-compose.yml > tmp1 +mv tmp1 docker-compose.yml + +sed -e s/$VERSION/$NEXT_VERSION/g setup.py > tmp2 +mv tmp2 setup.py + +git add docker-compose.yml setup.py +git commit -m "Release: $NEXT_VERSION" +git tag $NEXT_VERSION +git push origin $NEXT_VERSION +git push origin From fb38c73090f1a053755cba5dbc4f4f8c9d8d1440 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Thu, 3 May 2018 10:31:46 -0300 Subject: [PATCH 091/102] Fix #1921 --- sapl/parlamentares/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sapl/parlamentares/views.py b/sapl/parlamentares/views.py index b56215648..093bb83bd 100644 --- a/sapl/parlamentares/views.py +++ b/sapl/parlamentares/views.py @@ -192,7 +192,7 @@ class ColigacaoCrud(CrudAux): help_topic = 'coligacao' class ListView(CrudAux.ListView): - ordering = ('-numero_votos', 'nome') + ordering = ('legislatura', '-nome') def get_context_data(self, **kwargs): context = super(ColigacaoCrud.ListView, self).get_context_data( From eca5f280618159d049016be0db7ee62884643a81 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Thu, 3 May 2018 10:31:46 -0300 Subject: [PATCH 092/102] Fix #1921 --- sapl/parlamentares/views.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sapl/parlamentares/views.py b/sapl/parlamentares/views.py index b56215648..093bb83bd 100644 --- a/sapl/parlamentares/views.py +++ b/sapl/parlamentares/views.py @@ -192,7 +192,7 @@ class ColigacaoCrud(CrudAux): help_topic = 'coligacao' class ListView(CrudAux.ListView): - ordering = ('-numero_votos', 'nome') + ordering = ('legislatura', '-nome') def get_context_data(self, **kwargs): context = super(ColigacaoCrud.ListView, self).get_context_data( From 32f4c23853350b660f1fb266d52df34d26d916cd Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Thu, 3 May 2018 10:56:15 -0300 Subject: [PATCH 093/102] Fixes #1920 --- .../migrations/0014_auto_20180503_1055.py | 23 +++++++++++++++++++ sapl/comissoes/models.py | 2 ++ sapl/comissoes/views.py | 7 +++++- 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 sapl/comissoes/migrations/0014_auto_20180503_1055.py diff --git a/sapl/comissoes/migrations/0014_auto_20180503_1055.py b/sapl/comissoes/migrations/0014_auto_20180503_1055.py new file mode 100644 index 000000000..7a2524fed --- /dev/null +++ b/sapl/comissoes/migrations/0014_auto_20180503_1055.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.11 on 2018-05-03 13:55 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('comissoes', '0013_auto_20180312_1533'), + ] + + operations = [ + migrations.AlterModelOptions( + name='composicao', + options={'ordering': ['periodo'], 'verbose_name': 'Composição de Comissão', 'verbose_name_plural': 'Composições de Comissão'}, + ), + migrations.AlterModelOptions( + name='periodo', + options={'ordering': ['-data_inicio', '-data_fim'], 'verbose_name': 'Período de composição de Comissão', 'verbose_name_plural': 'Períodos de composição de Comissão'}, + ), + ] diff --git a/sapl/comissoes/models.py b/sapl/comissoes/models.py index f0b1b8d8b..0c3240fc6 100644 --- a/sapl/comissoes/models.py +++ b/sapl/comissoes/models.py @@ -105,6 +105,7 @@ class Periodo(models.Model): # PeriodoCompComissao class Meta: verbose_name = _('Período de composição de Comissão') verbose_name_plural = _('Períodos de composição de Comissão') + ordering = ['-data_inicio', '-data_fim'] def __str__(self): if self.data_inicio and self.data_fim: @@ -140,6 +141,7 @@ class Composicao(models.Model): # IGNORE class Meta: verbose_name = _('Composição de Comissão') verbose_name_plural = _('Composições de Comissão') + ordering = ['periodo'] def __str__(self): return '%s: %s' % (self.comissao.sigla, self.periodo) diff --git a/sapl/comissoes/views.py b/sapl/comissoes/views.py index 82c3a79c1..c5870389e 100644 --- a/sapl/comissoes/views.py +++ b/sapl/comissoes/views.py @@ -51,6 +51,9 @@ class PeriodoComposicaoCrud(CrudAux): class UpdateView(CrudAux.UpdateView): form_class = PeriodoForm + # class ListView(CrudAux.ListView): + + class ParticipacaoCrud(MasterDetailCrud): model = Participacao parent_field = 'composicao__comissao' @@ -112,7 +115,9 @@ class ComposicaoCrud(MasterDetailCrud): composicao_pk = self.take_composicao_pk() if composicao_pk == 0: - ultima_composicao = context['composicao_list'].last() + # Composicao eh ordenada por Periodo, que por sua vez esta em + # ordem descrescente de data de inicio (issue #1920) + ultima_composicao = context['composicao_list'].first() if ultima_composicao: context['composicao_pk'] = ultima_composicao.pk else: From c2aea79ff5a80ae1eb733c511344d503c1084c33 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Thu, 3 May 2018 10:56:15 -0300 Subject: [PATCH 094/102] Fixes #1920 --- .../migrations/0014_auto_20180503_1055.py | 23 +++++++++++++++++++ sapl/comissoes/models.py | 2 ++ sapl/comissoes/views.py | 7 +++++- 3 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 sapl/comissoes/migrations/0014_auto_20180503_1055.py diff --git a/sapl/comissoes/migrations/0014_auto_20180503_1055.py b/sapl/comissoes/migrations/0014_auto_20180503_1055.py new file mode 100644 index 000000000..7a2524fed --- /dev/null +++ b/sapl/comissoes/migrations/0014_auto_20180503_1055.py @@ -0,0 +1,23 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.11 on 2018-05-03 13:55 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('comissoes', '0013_auto_20180312_1533'), + ] + + operations = [ + migrations.AlterModelOptions( + name='composicao', + options={'ordering': ['periodo'], 'verbose_name': 'Composição de Comissão', 'verbose_name_plural': 'Composições de Comissão'}, + ), + migrations.AlterModelOptions( + name='periodo', + options={'ordering': ['-data_inicio', '-data_fim'], 'verbose_name': 'Período de composição de Comissão', 'verbose_name_plural': 'Períodos de composição de Comissão'}, + ), + ] diff --git a/sapl/comissoes/models.py b/sapl/comissoes/models.py index f0b1b8d8b..0c3240fc6 100644 --- a/sapl/comissoes/models.py +++ b/sapl/comissoes/models.py @@ -105,6 +105,7 @@ class Periodo(models.Model): # PeriodoCompComissao class Meta: verbose_name = _('Período de composição de Comissão') verbose_name_plural = _('Períodos de composição de Comissão') + ordering = ['-data_inicio', '-data_fim'] def __str__(self): if self.data_inicio and self.data_fim: @@ -140,6 +141,7 @@ class Composicao(models.Model): # IGNORE class Meta: verbose_name = _('Composição de Comissão') verbose_name_plural = _('Composições de Comissão') + ordering = ['periodo'] def __str__(self): return '%s: %s' % (self.comissao.sigla, self.periodo) diff --git a/sapl/comissoes/views.py b/sapl/comissoes/views.py index 82c3a79c1..c5870389e 100644 --- a/sapl/comissoes/views.py +++ b/sapl/comissoes/views.py @@ -51,6 +51,9 @@ class PeriodoComposicaoCrud(CrudAux): class UpdateView(CrudAux.UpdateView): form_class = PeriodoForm + # class ListView(CrudAux.ListView): + + class ParticipacaoCrud(MasterDetailCrud): model = Participacao parent_field = 'composicao__comissao' @@ -112,7 +115,9 @@ class ComposicaoCrud(MasterDetailCrud): composicao_pk = self.take_composicao_pk() if composicao_pk == 0: - ultima_composicao = context['composicao_list'].last() + # Composicao eh ordenada por Periodo, que por sua vez esta em + # ordem descrescente de data de inicio (issue #1920) + ultima_composicao = context['composicao_list'].first() if ultima_composicao: context['composicao_pk'] = ultima_composicao.pk else: From 2c034a84b393ff8a62504bf32fa4837e15c32d0c Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 3 May 2018 10:57:33 -0300 Subject: [PATCH 095/102] =?UTF-8?q?Imprime=20arquivos=20anteriores=20na=20?= =?UTF-8?q?exporta=C3=A7=C3=A3o=20de=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index c7442654e..e84a4d13b 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -333,7 +333,9 @@ def build_salvar(repo): def salvar(fullname, conteudo): sha = hashlib.sha256() sha.update(conteudo) - if sha.hexdigest() not in hashes: + if sha.hexdigest() in hashes: + print('- hash encontrado - {}'.format(fullname)) + else: fullname = ajusta_extensao(fullname, conteudo) if os.path.exists(fullname): # destrava arquivo pré-existente (o conteúdo mudou) @@ -355,7 +357,7 @@ def dump_sapl(sigla): destino.mkdir(parents=True) repo = git.Repo.init(destino) if TAG_ZOPE in repo.tags: - info('A exportação de documentos já está feita.') + print('A exportação de documentos já está feita -- abortando') return repo_execute(repo, 'git annex init') From 0690a77bab64901a3d95466c9c28ebee8c394af8 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 3 May 2018 11:51:32 -0300 Subject: [PATCH 096/102] =?UTF-8?q?Continua=20exporta=C3=A7=C3=A3o=20de=20?= =?UTF-8?q?docs=20mesmo=20c=20tipos=20n=C3=A3o=20reconhecidos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index e84a4d13b..a75200949 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -78,20 +78,21 @@ def br(obj): def guess_extension(fullname, buffer): mime = magic.from_buffer(buffer, mime=True) - try: - return EXTENSOES[mime] - except KeyError as e: + extensao = EXTENSOES.get(mime) + if extensao is not None: + return extensao + else: possibilidades = '\n'.join( [" '{}': '{}',".format(mime, ext) for ext in mimetypes.guess_all_extensions(mime)]) - msg = '''Extensão não conhecida para o arquivo: {} + print('''Extensão não conhecida para o arquivo: {} e mimetype: {} Algumas possibilidades são: {} Atualize o código do dicionário EXTENSOES! '''.format(fullname, mime, possibilidades) - print(msg) - raise Exception(msg, e) + ) + return '.DESCONHECIDO.{}'.format(mime.replace('/', '__')) def get_conteudo_file(doc): From 64716f175bd7427ec90581d6aa0408c0e28daf51 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Thu, 3 May 2018 13:52:11 -0300 Subject: [PATCH 097/102] =?UTF-8?q?HOT-FIX:=20desabilitando=20speedinfo=20?= =?UTF-8?q?na=20vers=C3=A3o=201.9=20do=20Django?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sapl/settings.py b/sapl/settings.py index 22377226b..ca24b9f02 100644 --- a/sapl/settings.py +++ b/sapl/settings.py @@ -127,7 +127,7 @@ MIDDLEWARE_CLASSES = ( 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.middleware.security.SecurityMiddleware', - 'speedinfo.middleware.ProfilerMiddleware', + # 'speedinfo.middleware.ProfilerMiddleware', # Bug na versão 1.9 ) CACHES = { From 0226923a668070c767288b7cd9377f0fd77444ed Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 3 May 2018 17:55:58 -0300 Subject: [PATCH 098/102] Reporta contagens diferentes na btree ao exportar docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ao invés de lancçar exceção --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index a75200949..63d0ed09e 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -155,7 +155,11 @@ def enumerate_btree(folder): obj, meta_type = br(obj), type(obj).__name__ yield id, obj, meta_type # verificação de consistência - assert contagem_esperada == contagem_real + if contagem_esperada != contagem_real: + print('ATENÇÃO: contagens diferentes na btree: ' + '{} esperada: {} real: {}'.format(folder, + contagem_esperada, + contagem_real)) nao_identificados = defaultdict(list) From 71665929c558820e7ee5509228c55f4acc46a597 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 3 May 2018 19:14:58 -0300 Subject: [PATCH 099/102] Grava marco com dump do postgres --- sapl/legacy/migracao.py | 14 +------------- sapl/legacy/migracao_dados.py | 12 ++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/sapl/legacy/migracao.py b/sapl/legacy/migracao.py index 4509d8ba6..059e49164 100644 --- a/sapl/legacy/migracao.py +++ b/sapl/legacy/migracao.py @@ -28,21 +28,9 @@ def migrar(interativo=False): migrar_usuarios(REPO.working_dir) migrar_documentos(REPO) gravar_marco() - gerar_pacote() -def gerar_pacote(): - - # backup do banco - print('Gerando backup do banco... ', end='', flush=True) - arq_backup = DIR_REPO.child('{}.backup'.format(NOME_BANCO_LEGADO)) - arq_backup.remove() - backup_cmd = ''' - pg_dump --host localhost --port 5432 --username postgres --no-password - --format custom --blobs --verbose --file {} {}'''.format( - arq_backup, NOME_BANCO_LEGADO) - subprocess.check_output(backup_cmd.split(), stderr=subprocess.DEVNULL) - print('SUCESSO') +def compactar_media(): # tar de media/sapl print('Criando tar de media... ', end='', flush=True) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index e40406447..0f1e4d6dc 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -1,6 +1,7 @@ import datetime import os import re +import subprocess import traceback from collections import OrderedDict, defaultdict, namedtuple from datetime import date @@ -1333,6 +1334,17 @@ def gravar_marco(): with open(nome_arq, 'w') as arq: pyaml.dump(data, arq) + # backup do banco + print('Gerando backup do banco... ', end='', flush=True) + arq_backup = DIR_REPO.child('{}.backup'.format(NOME_BANCO_LEGADO)) + arq_backup.remove() + backup_cmd = ''' + pg_dump --host localhost --port 5432 --username postgres --no-password + --format custom --blobs --verbose --file {} {}'''.format( + arq_backup, NOME_BANCO_LEGADO) + subprocess.check_output(backup_cmd.split(), stderr=subprocess.DEVNULL) + print('SUCESSO') + # salva mudanças REPO.git.add([dir_dados.name]) if 'master' not in REPO.heads or REPO.index.diff('HEAD'): From 3470756fb383aaf87cb5cb0b330cb6e3d9a59b53 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 4 May 2018 11:46:59 -0300 Subject: [PATCH 100/102] Corrige teste de prefixos de urls --- sapl/test_urls.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sapl/test_urls.py b/sapl/test_urls.py index 115572a37..0f1671c98 100644 --- a/sapl/test_urls.py +++ b/sapl/test_urls.py @@ -7,6 +7,7 @@ from django.contrib.contenttypes.models import ContentType from django.db import transaction from django.utils.translation import ugettext_lazy as _ from django.utils.translation import string_concat + from sapl.crud.base import PermissionRequiredForAppCrudMixin from sapl.rules.apps import AppConfig, update_groups from scripts.lista_urls import lista_urls @@ -164,7 +165,8 @@ apps_url_patterns_prefixs_and_users = { '/ajuda', '/email', '/recuperar-senha', - '/sapl' + '/sapl', + '/XSLT', ]}, 'comissoes': { 'users': {'operador_geral': ['/sistema', '/comissao'], From 40da78b14aee86f47af07edf4da949a9190da06e Mon Sep 17 00:00:00 2001 From: Victor Fabre Date: Mon, 7 May 2018 14:17:35 -0300 Subject: [PATCH 101/102] Fix #1925 (#1926) --- sapl/sessao/views.py | 5 +++++ sapl/templates/sessao/expediente.html | 1 + 2 files changed, 6 insertions(+) diff --git a/sapl/sessao/views.py b/sapl/sessao/views.py index 5beec02ad..746bbfec1 100644 --- a/sapl/sessao/views.py +++ b/sapl/sessao/views.py @@ -1448,6 +1448,11 @@ class ExpedienteView(FormMixin, DetailView): self.object = self.get_object() form = ExpedienteForm(request.POST) + if 'apagar-expediente' in request.POST: + ExpedienteSessao.objects.filter( + sessao_plenaria_id=self.object.id).delete() + return self.form_valid(form) + if form.is_valid(): list_tipo = request.POST.getlist('tipo') list_conteudo = request.POST.getlist('conteudo') diff --git a/sapl/templates/sessao/expediente.html b/sapl/templates/sessao/expediente.html index 940d73d97..106840b88 100644 --- a/sapl/templates/sessao/expediente.html +++ b/sapl/templates/sessao/expediente.html @@ -28,6 +28,7 @@
+ {% endif %} From 605165f8caff02112a140dec4ee650a515759a11 Mon Sep 17 00:00:00 2001 From: Victor Fabre Date: Mon, 7 May 2018 14:17:59 -0300 Subject: [PATCH 102/102] Fix 1918 (#1924) --- sapl/norma/views.py | 3 +-- sapl/templates/norma/normajuridica_detail.html | 2 +- sapl/templates/norma/subnav.yaml | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sapl/norma/views.py b/sapl/norma/views.py index d74bc8857..fd5396353 100644 --- a/sapl/norma/views.py +++ b/sapl/norma/views.py @@ -40,10 +40,9 @@ class NormaRelacionadaCrud(MasterDetailCrud): model = NormaRelacionada parent_field = 'norma_principal' help_topic = 'norma_juridica' - public = [RP_LIST, RP_DETAIL] class BaseMixin(MasterDetailCrud.BaseMixin): - list_field_names = ['norma_relacionada'] + list_field_names = ['norma_relacionada', 'tipo_vinculo'] class CreateView(MasterDetailCrud.CreateView): form_class = NormaRelacionadaForm diff --git a/sapl/templates/norma/normajuridica_detail.html b/sapl/templates/norma/normajuridica_detail.html index ed52564ed..6dcd3d592 100644 --- a/sapl/templates/norma/normajuridica_detail.html +++ b/sapl/templates/norma/normajuridica_detail.html @@ -35,7 +35,7 @@
-

Relacionamentos

+

Normas Relacionadas

{% if object.get_normas_relacionadas.0|length > 0 %} {% for p in object.get_normas_relacionadas.0 %} diff --git a/sapl/templates/norma/subnav.yaml b/sapl/templates/norma/subnav.yaml index 0599efdfb..d050a469f 100644 --- a/sapl/templates/norma/subnav.yaml +++ b/sapl/templates/norma/subnav.yaml @@ -2,8 +2,9 @@ - title: {% trans 'Início' %} url: normajuridica_detail -- title: {% trans 'Normas Relacionadas' %} +- title: {% trans 'Alterações em Outras Normas' %} url: normarelacionada_list + check_permission: norma.normarelacionada_list # Opção adicionada para chamar o TextoArticulado da norma. # para integração foram necessárias apenas criar a url norma_ta em urls.py