diff --git a/sapl/base/migrations/0002_auto_20170331_1900.py b/sapl/base/migrations/0002_auto_20170331_1900.py new file mode 100644 index 000000000..5ee726647 --- /dev/null +++ b/sapl/base/migrations/0002_auto_20170331_1900.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.12 on 2017-03-31 19:00 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='Argumento', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('argumento', models.CharField(max_length=50, verbose_name='Argumento')), + ], + options={ + 'verbose_name': 'Argumento da constraint', + 'verbose_name_plural': 'Argumentos da constraint', + }, + ), + migrations.CreateModel( + name='Constraint', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('nome_tabela', models.CharField(max_length=50, verbose_name='Nome da tabela')), + ('nome_constraint', models.CharField(max_length=100, verbose_name='Nome da constraint')), + ('nome_model', models.CharField(max_length=50, verbose_name='Nome da model')), + ('tipo_constraint', models.CharField(max_length=50, verbose_name='Tipo da constraint')), + ], + options={ + 'verbose_name': 'Constraint removida', + 'verbose_name_plural': 'Constraints removidas', + }, + ), + migrations.AddField( + model_name='problemamigracao', + name='eh_importante', + field=models.BooleanField(default=False, verbose_name='É importante?'), + ), + migrations.AddField( + model_name='argumento', + name='constraint', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='base.Constraint'), + ), + ] diff --git a/sapl/base/models.py b/sapl/base/models.py index 8825ea2a4..9dd0f5ec3 100644 --- a/sapl/base/models.py +++ b/sapl/base/models.py @@ -61,16 +61,45 @@ class ProblemaMigracao(models.Model): content_object = GenericForeignKey('content_type', 'object_id') nome_campo = models.CharField(max_length=100, blank=True, - verbose_name='Nome do(s) Campo(s)') + verbose_name=_('Nome do(s) Campo(s)')) problema = models.CharField(max_length=300, verbose_name=_('Problema')) descricao = models.CharField(max_length=300, verbose_name=_('Descrição')) - eh_stub = models.BooleanField(verbose_name='É stub?') + eh_stub = models.BooleanField(verbose_name=_('É stub?')) + eh_importante = models.BooleanField( + default=False, verbose_name=_('É importante?')) class Meta: verbose_name = _('Problema na Migração') verbose_name_plural = _('Problemas na Migração') +@reversion.register() +class Constraint(models.Model): + nome_tabela = models.CharField( + max_length=50, verbose_name=_('Nome da tabela')) + nome_constraint = models.CharField( + max_length=100, verbose_name=_('Nome da constraint')) + nome_model = models.CharField( + max_length=50, verbose_name=_('Nome da model')) + tipo_constraint = models.CharField( + max_length=50, verbose_name=_('Tipo da constraint')) + + class Meta: + verbose_name = _('Constraint removida') + verbose_name_plural = _('Constraints removidas') + + +@reversion.register() +class Argumento(models.Model): + constraint = models.ForeignKey(Constraint) + argumento = models.CharField( + max_length=50, verbose_name=_('Argumento')) + + class Meta: + verbose_name = _('Argumento da constraint') + verbose_name_plural = _('Argumentos da constraint') + + @reversion.register() class AppConfig(models.Model): diff --git a/sapl/legacy/management/commands/recria_constraints.py b/sapl/legacy/management/commands/recria_constraints.py new file mode 100644 index 000000000..9e999e5f6 --- /dev/null +++ b/sapl/legacy/management/commands/recria_constraints.py @@ -0,0 +1,12 @@ +from django.core.management.base import BaseCommand + +from sapl.legacy.migration import recria_constraints + + +class Command(BaseCommand): + + help = (u'Recria constraints do PostgreSQL excluidas durante ' + 'migração de dados') + + def handle(self, *args, **options): + recria_constraints() diff --git a/sapl/legacy/migration.py b/sapl/legacy/migration.py index ef87a3960..4a748d49f 100644 --- a/sapl/legacy/migration.py +++ b/sapl/legacy/migration.py @@ -13,13 +13,15 @@ from django.core.exceptions import ObjectDoesNotExist from django.db import OperationalError, ProgrammingError, connections, models from django.db.models import CharField, Max, ProtectedError, TextField from django.db.models.base import ModelBase +from django.db.models.signals import post_delete, post_save from model_mommy import mommy from model_mommy.mommy import foreign_key_required, make -from sapl.base.models import Autor, ProblemaMigracao +from sapl.base.models import Argumento, Autor, Constraint, ProblemaMigracao from sapl.comissoes.models import Comissao, Composicao, Participacao from sapl.legacy.models import Protocolo as ProtocoloLegado -from sapl.materia.models import (StatusTramitacao, TipoDocumento, +from sapl.materia.models import (DocumentoAcessorio, MateriaLegislativa, + StatusTramitacao, TipoDocumento, TipoMateriaLegislativa, TipoProposicao, Tramitacao) from sapl.norma.models import (AssuntoNorma, NormaJuridica, @@ -28,7 +30,7 @@ from sapl.parlamentares.models import Parlamentar from sapl.protocoloadm.models import Protocolo, StatusTramitacaoAdministrativo from sapl.sessao.models import ExpedienteMateria, OrdemDia from sapl.settings import PROJECT_DIR -from sapl.utils import normalize +from sapl.utils import delete_texto, normalize, save_texto # BASE ###################################################################### # apps to be migrated, in app dependency order (very important) @@ -109,6 +111,10 @@ def warn(msg): print('CUIDADO! ' + msg) +def erro(msg): + print('ERRO: ' + msg) + + def get_fk_related(field, value, label=None): if value is None and field.null is False: value = 0 @@ -197,6 +203,12 @@ def iter_sql_records(sql, db): record.__dict__.update(zip(fieldnames, row)) yield record +# Todos os models têm no máximo uma constraint unique together +# Isso é necessário para que o método delete_constraints funcione corretamente +assert all(len(model._meta.unique_together) <= 1 + for app in appconfs + for model in app.models.values()) + def delete_constraints(model): # pega nome da unique constraint dado o nome da tabela @@ -210,40 +222,66 @@ def delete_constraints(model): for r in result: if r[0].endswith('key'): words_list = r[0].split('_') - one_to_one_constraints.append([table, r[0], words_list, model]) + constraint = Constraint.objects.create( + nome_tabela=table, nome_constraint=r[0], + nome_model=model.__name__, tipo_constraint='one_to_one') + for w in words_list: + Argumento.objects.create(constraint=constraint, argumento=w) else: - args = None - args_list = [] if model._meta.unique_together: - args = model._meta.unique_together[0] - args_list = list(args) - unique_constraints.append([table, r[0], args_list, model]) + args_list = model._meta.unique_together[0] + constraint = Constraint.objects.create( + nome_tabela=table, nome_constraint=r[0], + nome_model=model.__name__, + tipo_constraint='unique_together') + for a in args_list: + Argumento.objects.create(constraint=constraint, + argumento=a) warn('Excluindo unique constraint de nome %s' % r[0]) exec_sql("ALTER TABLE %s DROP CONSTRAINT %s;" % (table, r[0])) -def recreate_constraints(): - if one_to_one_constraints: - for constraint in one_to_one_constraints: - table, name, args, model = constraint +def recria_constraints(): + constraints = Constraint.objects.all() + for con in constraints: + if con.tipo_constraint == 'one_to_one': + nome_tabela = con.nome_tabela + nome_constraint = con.nome_constraint + args = [a.argumento for a in con.argumento_set.all()] args_string = '' args_string = "(" + "_".join(map(str, args[2:-1])) + ")" - exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" % - (table, name, args_string)) - if unique_constraints: - for constraint in unique_constraints: - table, name, args, model = constraint + try: + exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" % + (nome_tabela, nome_constraint, args_string)) + except ProgrammingError: + info('A constraint %s já foi recriada!' % nome_constraint) + if con.tipo_constraint == 'unique_together': + nome_tabela = con.nome_tabela + nome_constraint = con.nome_constraint + # Pegando explicitamente o primeiro valor do filter, + # pois pode ser que haja mais de uma ocorrência + model = ContentType.objects.filter( + model=con.nome_model.lower())[0].model_class() + args = [a.argumento for a in con.argumento_set.all()] for i in range(len(args)): if isinstance(model._meta.get_field(args[i]), models.ForeignKey): args[i] = args[i] + '_id' args_string = '' args_string += "(" + ', '.join(map(str, args)) + ")" - exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" % - (table, name, args_string)) - one_to_one_constraints.clear() - unique_constraints.clear() + try: + exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" % + (nome_tabela, nome_constraint, args_string)) + except ProgrammingError: + info('A constraint %s já foi recriada!' % nome_constraint) + except Exception as err: + problema = re.findall('\(.*?\)', err.args[0]) + erro('A constraint [%s] da tabela [%s] não pode ser recriada' % + (nome_constraint, nome_tabela)) + erro('Os dados %s = %s estão duplicados. ' + 'Arrume antes de recriar as constraints!' % + (problema[0], problema[1])) def obj_desnecessario(obj): @@ -262,8 +300,8 @@ def get_last_value(model): def alter_sequence(model, id): sequence_name = '%s_id_seq' % model._meta.db_table - exec_sql('ALTER SEQUENCE %s RESTART WITH %s MINVALUE %s;' % ( - sequence_name, id, id)) + exec_sql('ALTER SEQUENCE %s RESTART WITH %s MINVALUE -1;' % ( + sequence_name, id)) def save_with_id(new, id): @@ -278,8 +316,7 @@ def save_relation(obj, nome_campo='', problema='', descricao='', eh_stub=False): link = ProblemaMigracao( content_object=obj, nome_campo=nome_campo, problema=problema, - descricao=descricao, eh_stub=eh_stub, - ) + descricao=descricao, eh_stub=eh_stub,) link.save() @@ -416,6 +453,8 @@ class DataMigrator: call([PROJECT_DIR.child('manage.py'), 'flush', '--database=default', '--no-input'], stdout=PIPE) + disconecta_sinais_indexacao() + info('Começando migração: %s...' % obj) self._do_migrate(obj) @@ -427,7 +466,7 @@ class DataMigrator: for obj in self.to_delete: msg = 'A entrada de PK %s da model %s não pode ser ' \ 'excluida' % (obj.pk, obj._meta.model_name) - descricao = 'Um ou mais objetos protegidos ' + descricao = 'Um ou mais objetos protegidos' warn(msg + ' => ' + descricao) save_relation(obj=obj, problema=msg, descricao=descricao, eh_stub=False) @@ -435,8 +474,8 @@ class DataMigrator: info('Deletando stubs desnecessários...') while self.delete_stubs(): pass - info('Recriando unique constraints...') - # recreate_constraints() + + conecta_sinais_indexacao() def _do_migrate(self, obj): if isinstance(obj, AppConfig): @@ -731,3 +770,23 @@ def make_with_log(model, _quantity=None, make_m2m=False, **attrs): return stub make_with_log.required = foreign_key_required + +# DISCONNECT SIGNAL ######################################################## + + +def disconecta_sinais_indexacao(): + post_save.disconnect(save_texto, NormaJuridica) + post_save.disconnect(save_texto, DocumentoAcessorio) + post_save.disconnect(save_texto, MateriaLegislativa) + post_delete.disconnect(delete_texto, NormaJuridica) + post_delete.disconnect(delete_texto, DocumentoAcessorio) + post_delete.disconnect(delete_texto, MateriaLegislativa) + + +def conecta_sinais_indexacao(): + post_save.connect(save_texto, NormaJuridica) + post_save.connect(save_texto, DocumentoAcessorio) + post_save.connect(save_texto, MateriaLegislativa) + post_delete.connect(delete_texto, NormaJuridica) + post_delete.connect(delete_texto, DocumentoAcessorio) + post_delete.connect(delete_texto, MateriaLegislativa) diff --git a/sapl/legacy/scripts/migra_um_db.sh b/sapl/legacy/scripts/migra_um_db.sh index 26240d8d4..f55dfb53a 100755 --- a/sapl/legacy/scripts/migra_um_db.sh +++ b/sapl/legacy/scripts/migra_um_db.sh @@ -23,3 +23,9 @@ echo "--- MIGRACAO DE DADOS ---" | tee -a $LOG echo >> $LOG DATABASE_NAME=$1 ./manage.py migracao_25_31 -f --settings sapl.legacy_migration_settings |& tee -a $LOG echo >> $LOG + + +echo "--- RECRIANDO CONSTRAINTS ---" | tee -a $LOG +echo >> $LOG +DATABASE_NAME=$1 ./manage.py recria_constraints --settings sapl.legacy_migration_settings |& tee -a $LOG +echo >> $LOG diff --git a/sapl/materia/signals.py b/sapl/materia/signals.py index d913e9cd8..9f08b104c 100644 --- a/sapl/materia/signals.py +++ b/sapl/materia/signals.py @@ -1,29 +1,9 @@ -from subprocess import PIPE, call -from threading import Thread - from django.db.models.signals import post_delete, post_save - -from sapl.settings import PROJECT_DIR +from sapl.utils import save_texto, delete_texto from .models import DocumentoAcessorio, MateriaLegislativa -class UpdateIndexCommand(Thread): - def run(self): - call([PROJECT_DIR.child('manage.py'), 'update_index'], - stdout=PIPE) - - -def save_texto(sender, instance, **kwargs): - update_index = UpdateIndexCommand() - update_index.start() - - -def delete_texto(sender, instance, **kwargs): - update_index = UpdateIndexCommand() - update_index.start() - - post_save.connect(save_texto, sender=MateriaLegislativa) post_save.connect(save_texto, sender=DocumentoAcessorio) post_delete.connect(delete_texto, sender=MateriaLegislativa) diff --git a/sapl/norma/signals.py b/sapl/norma/signals.py index 20d405e49..3089e563e 100644 --- a/sapl/norma/signals.py +++ b/sapl/norma/signals.py @@ -1,28 +1,8 @@ -from subprocess import PIPE, call -from threading import Thread - from django.db.models.signals import post_delete, post_save - -from sapl.settings import PROJECT_DIR +from sapl.utils import save_texto, delete_texto from .models import NormaJuridica -class UpdateIndexCommand(Thread): - def run(self): - call([PROJECT_DIR.child('manage.py'), 'update_index'], - stdout=PIPE) - - -def save_texto(sender, instance, **kwargs): - update_index = UpdateIndexCommand() - update_index.start() - - -def delete_texto(sender, instance, **kwargs): - update_index = UpdateIndexCommand() - update_index.start() - - post_save.connect(save_texto, sender=NormaJuridica) post_delete.connect(delete_texto, sender=NormaJuridica) diff --git a/sapl/rules/map_rules.py b/sapl/rules/map_rules.py index 906592fd0..05d869fd8 100644 --- a/sapl/rules/map_rules.py +++ b/sapl/rules/map_rules.py @@ -197,6 +197,8 @@ rules_group_geral = { (base.CasaLegislativa, __listdetailchange__), (base.ProblemaMigracao, []), + (base.Argumento, []), + (base.Constraint, []), (base.TipoAutor, __base__), (base.Autor, __base__), diff --git a/sapl/rules/tests/test_rules.py b/sapl/rules/tests/test_rules.py index 3662477b9..6fc7eb3fe 100644 --- a/sapl/rules/tests/test_rules.py +++ b/sapl/rules/tests/test_rules.py @@ -6,7 +6,8 @@ from django.contrib.contenttypes.models import ContentType from django.utils import six from django.utils.translation import ugettext_lazy as _ -from sapl.base.models import CasaLegislativa, ProblemaMigracao +from sapl.base.models import (CasaLegislativa, ProblemaMigracao, Argumento, + Constraint) from sapl.compilacao.models import (PerfilEstruturalTextoArticulado, TipoDispositivo, TipoDispositivoRelationship) @@ -56,11 +57,15 @@ def test_models_in_rules_patterns(model_item): __fp__in__test_permission_of_models_in_rules_patterns = { map_rules.RP_ADD: [CasaLegislativa, ProblemaMigracao, + Argumento, + Constraint, TipoDispositivo, TipoDispositivoRelationship, PerfilEstruturalTextoArticulado], map_rules.RP_CHANGE: [ProblemaMigracao, + Argumento, + Constraint, AcompanhamentoMateria, TipoDispositivo, TipoDispositivoRelationship, @@ -68,17 +73,23 @@ __fp__in__test_permission_of_models_in_rules_patterns = { map_rules.RP_DELETE: [CasaLegislativa, ProblemaMigracao, + Argumento, + Constraint, TipoDispositivo, TipoDispositivoRelationship, PerfilEstruturalTextoArticulado], map_rules.RP_LIST: [ProblemaMigracao, + Argumento, + Constraint, AcompanhamentoMateria, TipoDispositivo, TipoDispositivoRelationship, PerfilEstruturalTextoArticulado], map_rules.RP_DETAIL: [ProblemaMigracao, + Argumento, + Constraint, AcompanhamentoMateria, TipoDispositivo, TipoDispositivoRelationship, diff --git a/sapl/utils.py b/sapl/utils.py index 3a857f337..81ebea6cc 100644 --- a/sapl/utils.py +++ b/sapl/utils.py @@ -5,6 +5,8 @@ import re from datetime import date from functools import wraps from unicodedata import normalize as unicodedata_normalize +from subprocess import PIPE, call +from threading import Thread import django_filters import magic @@ -22,7 +24,7 @@ from floppyforms import ClearableFileInput from reversion.admin import VersionAdmin from sapl.crispy_layout_mixin import SaplFormLayout, form_actions, to_row -from sapl.settings import BASE_DIR +from sapl.settings import BASE_DIR, PROJECT_DIR sapl_logger = logging.getLogger(BASE_DIR.name) @@ -632,3 +634,19 @@ def texto_upload_path(instance, filename, subpath=''): } return path + + +class UpdateIndexCommand(Thread): + def run(self): + call([PROJECT_DIR.child('manage.py'), 'update_index'], + stdout=PIPE) + + +def save_texto(sender, instance, **kwargs): + update_index = UpdateIndexCommand() + update_index.start() + + +def delete_texto(sender, instance, **kwargs): + update_index = UpdateIndexCommand() + update_index.start()