Merge pull request #989 from interlegis/908-restaurar-constraints

fix #908 restaurar constraints

sapl/base/migrations/0002_auto_20170331_1900.py

@@ -0,0 +1,51 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.9.12 on 2017-03-31 19:00
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('base', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Argumento',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('argumento', models.CharField(max_length=50, verbose_name='Argumento')),
+            ],
+            options={
+                'verbose_name': 'Argumento da constraint',
+                'verbose_name_plural': 'Argumentos da constraint',
+            },
+        ),
+        migrations.CreateModel(
+            name='Constraint',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('nome_tabela', models.CharField(max_length=50, verbose_name='Nome da tabela')),
+                ('nome_constraint', models.CharField(max_length=100, verbose_name='Nome da constraint')),
+                ('nome_model', models.CharField(max_length=50, verbose_name='Nome da model')),
+                ('tipo_constraint', models.CharField(max_length=50, verbose_name='Tipo da constraint')),
+            ],
+            options={
+                'verbose_name': 'Constraint removida',
+                'verbose_name_plural': 'Constraints removidas',
+            },
+        ),
+        migrations.AddField(
+            model_name='problemamigracao',
+            name='eh_importante',
+            field=models.BooleanField(default=False, verbose_name='É importante?'),
+        ),
+        migrations.AddField(
+            model_name='argumento',
+            name='constraint',
+            field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='base.Constraint'),
+        ),
+    ]

sapl/base/models.py

@@ -61,16 +61,45 @@ class ProblemaMigracao(models.Model):
     content_object = GenericForeignKey('content_type', 'object_id')
     nome_campo = models.CharField(max_length=100,
                                   blank=True,
-                                  verbose_name='Nome do(s) Campo(s)')
+                                  verbose_name=_('Nome do(s) Campo(s)'))
     problema = models.CharField(max_length=300, verbose_name=_('Problema'))
     descricao = models.CharField(max_length=300, verbose_name=_('Descrição'))
-    eh_stub = models.BooleanField(verbose_name='É stub?')
+    eh_stub = models.BooleanField(verbose_name=_('É stub?'))
+    eh_importante = models.BooleanField(
+        default=False, verbose_name=_('É importante?'))
 
     class Meta:
         verbose_name = _('Problema na Migração')
         verbose_name_plural = _('Problemas na Migração')
 
 
+@reversion.register()
+class Constraint(models.Model):
+    nome_tabela = models.CharField(
+        max_length=50, verbose_name=_('Nome da tabela'))
+    nome_constraint = models.CharField(
+        max_length=100, verbose_name=_('Nome da constraint'))
+    nome_model = models.CharField(
+        max_length=50, verbose_name=_('Nome da model'))
+    tipo_constraint = models.CharField(
+        max_length=50, verbose_name=_('Tipo da constraint'))
+
+    class Meta:
+        verbose_name = _('Constraint removida')
+        verbose_name_plural = _('Constraints removidas')
+
+
+@reversion.register()
+class Argumento(models.Model):
+    constraint = models.ForeignKey(Constraint)
+    argumento = models.CharField(
+        max_length=50, verbose_name=_('Argumento'))
+
+    class Meta:
+        verbose_name = _('Argumento da constraint')
+        verbose_name_plural = _('Argumentos da constraint')
+
+
 @reversion.register()
 class AppConfig(models.Model):
 

sapl/legacy/management/commands/recria_constraints.py

@@ -0,0 +1,12 @@
+from django.core.management.base import BaseCommand
+
+from sapl.legacy.migration import recria_constraints
+
+
+class Command(BaseCommand):
+
+    help = (u'Recria constraints do PostgreSQL excluidas durante '
+            'migração de dados')
+
+    def handle(self, *args, **options):
+        recria_constraints()

sapl/legacy/migration.py

@@ -13,13 +13,15 @@ from django.core.exceptions import ObjectDoesNotExist
 from django.db import OperationalError, ProgrammingError, connections, models
 from django.db.models import CharField, Max, ProtectedError, TextField
 from django.db.models.base import ModelBase
+from django.db.models.signals import post_delete, post_save
 from model_mommy import mommy
 from model_mommy.mommy import foreign_key_required, make
 
-from sapl.base.models import Autor, ProblemaMigracao
+from sapl.base.models import Argumento, Autor, Constraint, ProblemaMigracao
 from sapl.comissoes.models import Comissao, Composicao, Participacao
 from sapl.legacy.models import Protocolo as ProtocoloLegado
-from sapl.materia.models import (StatusTramitacao, TipoDocumento,
+from sapl.materia.models import (DocumentoAcessorio, MateriaLegislativa,
+                                 StatusTramitacao, TipoDocumento,
                                  TipoMateriaLegislativa, TipoProposicao,
                                  Tramitacao)
 from sapl.norma.models import (AssuntoNorma, NormaJuridica,
@@ -28,7 +30,7 @@ from sapl.parlamentares.models import Parlamentar
 from sapl.protocoloadm.models import Protocolo, StatusTramitacaoAdministrativo
 from sapl.sessao.models import ExpedienteMateria, OrdemDia
 from sapl.settings import PROJECT_DIR
-from sapl.utils import normalize
+from sapl.utils import delete_texto, normalize, save_texto
 
 # BASE ######################################################################
 #  apps to be migrated, in app dependency order (very important)
@@ -109,6 +111,10 @@ def warn(msg):
     print('CUIDADO! ' + msg)
 
 
+def erro(msg):
+    print('ERRO: ' + msg)
+
+
 def get_fk_related(field, value, label=None):
     if value is None and field.null is False:
         value = 0
@@ -197,6 +203,12 @@ def iter_sql_records(sql, db):
         record.__dict__.update(zip(fieldnames, row))
         yield record
 
+# Todos os models têm no máximo uma constraint unique together
+# Isso é necessário para que o método delete_constraints funcione corretamente
+assert all(len(model._meta.unique_together) <= 1
+           for app in appconfs
+           for model in app.models.values())
+
 
 def delete_constraints(model):
     # pega nome da unique constraint dado o nome da tabela
@@ -210,40 +222,66 @@ def delete_constraints(model):
     for r in result:
         if r[0].endswith('key'):
             words_list = r[0].split('_')
-            one_to_one_constraints.append([table, r[0], words_list, model])
+            constraint = Constraint.objects.create(
+                nome_tabela=table, nome_constraint=r[0],
+                nome_model=model.__name__, tipo_constraint='one_to_one')
+            for w in words_list:
+                Argumento.objects.create(constraint=constraint, argumento=w)
         else:
-            args = None
-            args_list = []
             if model._meta.unique_together:
-                args = model._meta.unique_together[0]
+                args_list = model._meta.unique_together[0]
-                args_list = list(args)
+                constraint = Constraint.objects.create(
-            unique_constraints.append([table, r[0], args_list, model])
+                    nome_tabela=table, nome_constraint=r[0],
+                    nome_model=model.__name__,
+                    tipo_constraint='unique_together')
+                for a in args_list:
+                    Argumento.objects.create(constraint=constraint,
+                                             argumento=a)
         warn('Excluindo unique constraint de nome %s' % r[0])
         exec_sql("ALTER TABLE %s DROP CONSTRAINT %s;" %
                  (table, r[0]))
 
 
-def recreate_constraints():
+def recria_constraints():
-    if one_to_one_constraints:
+    constraints = Constraint.objects.all()
-        for constraint in one_to_one_constraints:
+    for con in constraints:
-            table, name, args, model = constraint
+        if con.tipo_constraint == 'one_to_one':
+            nome_tabela = con.nome_tabela
+            nome_constraint = con.nome_constraint
+            args = [a.argumento for a in con.argumento_set.all()]
             args_string = ''
             args_string = "(" + "_".join(map(str, args[2:-1])) + ")"
-            exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" %
+            try:
-                     (table, name, args_string))
+                exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" %
-    if unique_constraints:
+                         (nome_tabela, nome_constraint, args_string))
-        for constraint in unique_constraints:
+            except ProgrammingError:
-            table, name, args, model = constraint
+                info('A constraint %s já foi recriada!' % nome_constraint)
+        if con.tipo_constraint == 'unique_together':
+            nome_tabela = con.nome_tabela
+            nome_constraint = con.nome_constraint
+            # Pegando explicitamente o primeiro valor do filter,
+            # pois pode ser que haja mais de uma ocorrência
+            model = ContentType.objects.filter(
+                model=con.nome_model.lower())[0].model_class()
+            args = [a.argumento for a in con.argumento_set.all()]
             for i in range(len(args)):
                 if isinstance(model._meta.get_field(args[i]),
                               models.ForeignKey):
                     args[i] = args[i] + '_id'
             args_string = ''
             args_string += "(" + ', '.join(map(str, args)) + ")"
-            exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" %
+            try:
-                     (table, name, args_string))
+                exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" %
-    one_to_one_constraints.clear()
+                         (nome_tabela, nome_constraint, args_string))
-    unique_constraints.clear()
+            except ProgrammingError:
+                info('A constraint %s já foi recriada!' % nome_constraint)
+            except Exception as err:
+                problema = re.findall('\(.*?\)', err.args[0])
+                erro('A constraint [%s] da tabela [%s] não pode ser recriada' %
+                     (nome_constraint, nome_tabela))
+                erro('Os dados %s = %s estão duplicados. '
+                     'Arrume antes de recriar as constraints!' %
+                     (problema[0], problema[1]))
 
 
 def obj_desnecessario(obj):
@@ -262,8 +300,8 @@ def get_last_value(model):
 
 def alter_sequence(model, id):
     sequence_name = '%s_id_seq' % model._meta.db_table
-    exec_sql('ALTER SEQUENCE %s RESTART WITH %s MINVALUE %s;' % (
+    exec_sql('ALTER SEQUENCE %s RESTART WITH %s MINVALUE -1;' % (
-        sequence_name, id, id))
+        sequence_name, id))
 
 
 def save_with_id(new, id):
@@ -278,8 +316,7 @@ def save_relation(obj, nome_campo='', problema='', descricao='',
                   eh_stub=False):
     link = ProblemaMigracao(
         content_object=obj, nome_campo=nome_campo, problema=problema,
-        descricao=descricao, eh_stub=eh_stub,
+        descricao=descricao, eh_stub=eh_stub,)
-    )
     link.save()
 
 
@@ -416,6 +453,8 @@ class DataMigrator:
         call([PROJECT_DIR.child('manage.py'), 'flush',
               '--database=default', '--no-input'], stdout=PIPE)
 
+        disconecta_sinais_indexacao()
+
         info('Começando migração: %s...' % obj)
         self._do_migrate(obj)
 
@@ -427,7 +466,7 @@ class DataMigrator:
         for obj in self.to_delete:
             msg = 'A entrada de PK %s da model %s não pode ser ' \
                 'excluida' % (obj.pk, obj._meta.model_name)
-            descricao = 'Um ou mais objetos protegidos '
+            descricao = 'Um ou mais objetos protegidos'
             warn(msg + ' => ' + descricao)
             save_relation(obj=obj, problema=msg,
                           descricao=descricao, eh_stub=False)
@@ -435,8 +474,8 @@ class DataMigrator:
         info('Deletando stubs desnecessários...')
         while self.delete_stubs():
             pass
-        info('Recriando unique constraints...')
+
-        # recreate_constraints()
+        conecta_sinais_indexacao()
 
     def _do_migrate(self, obj):
         if isinstance(obj, AppConfig):
@@ -731,3 +770,23 @@ def make_with_log(model, _quantity=None, make_m2m=False, **attrs):
     return stub
 
 make_with_log.required = foreign_key_required
+
+# DISCONNECT SIGNAL  ########################################################
+
+
+def disconecta_sinais_indexacao():
+    post_save.disconnect(save_texto, NormaJuridica)
+    post_save.disconnect(save_texto, DocumentoAcessorio)
+    post_save.disconnect(save_texto, MateriaLegislativa)
+    post_delete.disconnect(delete_texto, NormaJuridica)
+    post_delete.disconnect(delete_texto, DocumentoAcessorio)
+    post_delete.disconnect(delete_texto, MateriaLegislativa)
+
+
+def conecta_sinais_indexacao():
+    post_save.connect(save_texto, NormaJuridica)
+    post_save.connect(save_texto, DocumentoAcessorio)
+    post_save.connect(save_texto, MateriaLegislativa)
+    post_delete.connect(delete_texto, NormaJuridica)
+    post_delete.connect(delete_texto, DocumentoAcessorio)
+    post_delete.connect(delete_texto, MateriaLegislativa)

sapl/legacy/scripts/migra_um_db.sh

@@ -23,3 +23,9 @@ echo "--- MIGRACAO DE DADOS ---" | tee -a $LOG
 echo >> $LOG
 DATABASE_NAME=$1 ./manage.py migracao_25_31 -f --settings sapl.legacy_migration_settings |& tee -a $LOG
 echo >> $LOG
+
+
+echo "--- RECRIANDO CONSTRAINTS  ---" | tee -a $LOG
+echo >> $LOG
+DATABASE_NAME=$1 ./manage.py recria_constraints --settings sapl.legacy_migration_settings |& tee -a $LOG
+echo >> $LOG

sapl/materia/signals.py

@@ -1,29 +1,9 @@
-from subprocess import PIPE, call
-from threading import Thread
-
 from django.db.models.signals import post_delete, post_save
-
+from sapl.utils import save_texto, delete_texto
-from sapl.settings import PROJECT_DIR
 
 from .models import DocumentoAcessorio, MateriaLegislativa
 
 
-class UpdateIndexCommand(Thread):
-    def run(self):
-        call([PROJECT_DIR.child('manage.py'), 'update_index'],
-             stdout=PIPE)
-
-
-def save_texto(sender, instance, **kwargs):
-    update_index = UpdateIndexCommand()
-    update_index.start()
-
-
-def delete_texto(sender, instance, **kwargs):
-    update_index = UpdateIndexCommand()
-    update_index.start()
-
-
 post_save.connect(save_texto, sender=MateriaLegislativa)
 post_save.connect(save_texto, sender=DocumentoAcessorio)
 post_delete.connect(delete_texto, sender=MateriaLegislativa)

sapl/norma/signals.py

@@ -1,28 +1,8 @@
-from subprocess import PIPE, call
-from threading import Thread
-
 from django.db.models.signals import post_delete, post_save
-
+from sapl.utils import save_texto, delete_texto
-from sapl.settings import PROJECT_DIR
 
 from .models import NormaJuridica
 
 
-class UpdateIndexCommand(Thread):
-    def run(self):
-        call([PROJECT_DIR.child('manage.py'), 'update_index'],
-             stdout=PIPE)
-
-
-def save_texto(sender, instance, **kwargs):
-    update_index = UpdateIndexCommand()
-    update_index.start()
-
-
-def delete_texto(sender, instance, **kwargs):
-    update_index = UpdateIndexCommand()
-    update_index.start()
-
-
 post_save.connect(save_texto, sender=NormaJuridica)
 post_delete.connect(delete_texto, sender=NormaJuridica)

sapl/rules/map_rules.py

@@ -197,6 +197,8 @@ rules_group_geral = {
 
         (base.CasaLegislativa, __listdetailchange__),
         (base.ProblemaMigracao, []),
+        (base.Argumento, []),
+        (base.Constraint, []),
         (base.TipoAutor, __base__),
         (base.Autor, __base__),
 

sapl/rules/tests/test_rules.py

@@ -6,7 +6,8 @@ from django.contrib.contenttypes.models import ContentType
 from django.utils import six
 from django.utils.translation import ugettext_lazy as _
 
-from sapl.base.models import CasaLegislativa, ProblemaMigracao
+from sapl.base.models import (CasaLegislativa, ProblemaMigracao, Argumento,
+                              Constraint)
 from sapl.compilacao.models import (PerfilEstruturalTextoArticulado,
                                     TipoDispositivo,
                                     TipoDispositivoRelationship)
@@ -56,11 +57,15 @@ def test_models_in_rules_patterns(model_item):
 __fp__in__test_permission_of_models_in_rules_patterns = {
     map_rules.RP_ADD: [CasaLegislativa,
                        ProblemaMigracao,
+                       Argumento,
+                       Constraint,
                        TipoDispositivo,
                        TipoDispositivoRelationship,
                        PerfilEstruturalTextoArticulado],
 
     map_rules.RP_CHANGE: [ProblemaMigracao,
+                          Argumento,
+                          Constraint,
                           AcompanhamentoMateria,
                           TipoDispositivo,
                           TipoDispositivoRelationship,
@@ -68,17 +73,23 @@ __fp__in__test_permission_of_models_in_rules_patterns = {
 
     map_rules.RP_DELETE: [CasaLegislativa,
                           ProblemaMigracao,
+                          Argumento,
+                          Constraint,
                           TipoDispositivo,
                           TipoDispositivoRelationship,
                           PerfilEstruturalTextoArticulado],
 
     map_rules.RP_LIST: [ProblemaMigracao,
+                        Argumento,
+                        Constraint,
                         AcompanhamentoMateria,
                         TipoDispositivo,
                         TipoDispositivoRelationship,
                         PerfilEstruturalTextoArticulado],
 
     map_rules.RP_DETAIL: [ProblemaMigracao,
+                          Argumento,
+                          Constraint,
                           AcompanhamentoMateria,
                           TipoDispositivo,
                           TipoDispositivoRelationship,

sapl/utils.py

@@ -5,6 +5,8 @@ import re
 from datetime import date
 from functools import wraps
 from unicodedata import normalize as unicodedata_normalize
+from subprocess import PIPE, call
+from threading import Thread
 
 import django_filters
 import magic
@@ -22,7 +24,7 @@ from floppyforms import ClearableFileInput
 from reversion.admin import VersionAdmin
 
 from sapl.crispy_layout_mixin import SaplFormLayout, form_actions, to_row
-from sapl.settings import BASE_DIR
+from sapl.settings import BASE_DIR, PROJECT_DIR
 
 sapl_logger = logging.getLogger(BASE_DIR.name)
 
@@ -632,3 +634,19 @@ def texto_upload_path(instance, filename, subpath=''):
         }
 
     return path
+
+
+class UpdateIndexCommand(Thread):
+    def run(self):
+        call([PROJECT_DIR.child('manage.py'), 'update_index'],
+             stdout=PIPE)
+
+
+def save_texto(sender, instance, **kwargs):
+    update_index = UpdateIndexCommand()
+    update_index.start()
+
+
+def delete_texto(sender, instance, **kwargs):
+    update_index = UpdateIndexCommand()
+    update_index.start()