Browse Source

Merge pull request #989 from interlegis/908-restaurar-constraints

fix #908 restaurar constraints
pull/1017/head
Luciano Henrique Nunes de Almeida 8 years ago
committed by GitHub
parent
commit
c19d4dd0d1
  1. 51
      sapl/base/migrations/0002_auto_20170331_1900.py
  2. 33
      sapl/base/models.py
  3. 12
      sapl/legacy/management/commands/recria_constraints.py
  4. 117
      sapl/legacy/migration.py
  5. 6
      sapl/legacy/scripts/migra_um_db.sh
  6. 22
      sapl/materia/signals.py
  7. 22
      sapl/norma/signals.py
  8. 2
      sapl/rules/map_rules.py
  9. 13
      sapl/rules/tests/test_rules.py
  10. 20
      sapl/utils.py

51
sapl/base/migrations/0002_auto_20170331_1900.py

@ -0,0 +1,51 @@
# -*- coding: utf-8 -*-
# Generated by Django 1.9.12 on 2017-03-31 19:00
from __future__ import unicode_literals
from django.db import migrations, models
import django.db.models.deletion
class Migration(migrations.Migration):
dependencies = [
('base', '0001_initial'),
]
operations = [
migrations.CreateModel(
name='Argumento',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('argumento', models.CharField(max_length=50, verbose_name='Argumento')),
],
options={
'verbose_name': 'Argumento da constraint',
'verbose_name_plural': 'Argumentos da constraint',
},
),
migrations.CreateModel(
name='Constraint',
fields=[
('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
('nome_tabela', models.CharField(max_length=50, verbose_name='Nome da tabela')),
('nome_constraint', models.CharField(max_length=100, verbose_name='Nome da constraint')),
('nome_model', models.CharField(max_length=50, verbose_name='Nome da model')),
('tipo_constraint', models.CharField(max_length=50, verbose_name='Tipo da constraint')),
],
options={
'verbose_name': 'Constraint removida',
'verbose_name_plural': 'Constraints removidas',
},
),
migrations.AddField(
model_name='problemamigracao',
name='eh_importante',
field=models.BooleanField(default=False, verbose_name='É importante?'),
),
migrations.AddField(
model_name='argumento',
name='constraint',
field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='base.Constraint'),
),
]

33
sapl/base/models.py

@ -61,16 +61,45 @@ class ProblemaMigracao(models.Model):
content_object = GenericForeignKey('content_type', 'object_id')
nome_campo = models.CharField(max_length=100,
blank=True,
verbose_name='Nome do(s) Campo(s)')
verbose_name=_('Nome do(s) Campo(s)'))
problema = models.CharField(max_length=300, verbose_name=_('Problema'))
descricao = models.CharField(max_length=300, verbose_name=_('Descrição'))
eh_stub = models.BooleanField(verbose_name='É stub?')
eh_stub = models.BooleanField(verbose_name=_('É stub?'))
eh_importante = models.BooleanField(
default=False, verbose_name=_('É importante?'))
class Meta:
verbose_name = _('Problema na Migração')
verbose_name_plural = _('Problemas na Migração')
@reversion.register()
class Constraint(models.Model):
nome_tabela = models.CharField(
max_length=50, verbose_name=_('Nome da tabela'))
nome_constraint = models.CharField(
max_length=100, verbose_name=_('Nome da constraint'))
nome_model = models.CharField(
max_length=50, verbose_name=_('Nome da model'))
tipo_constraint = models.CharField(
max_length=50, verbose_name=_('Tipo da constraint'))
class Meta:
verbose_name = _('Constraint removida')
verbose_name_plural = _('Constraints removidas')
@reversion.register()
class Argumento(models.Model):
constraint = models.ForeignKey(Constraint)
argumento = models.CharField(
max_length=50, verbose_name=_('Argumento'))
class Meta:
verbose_name = _('Argumento da constraint')
verbose_name_plural = _('Argumentos da constraint')
@reversion.register()
class AppConfig(models.Model):

12
sapl/legacy/management/commands/recria_constraints.py

@ -0,0 +1,12 @@
from django.core.management.base import BaseCommand
from sapl.legacy.migration import recria_constraints
class Command(BaseCommand):
help = (u'Recria constraints do PostgreSQL excluidas durante '
'migração de dados')
def handle(self, *args, **options):
recria_constraints()

117
sapl/legacy/migration.py

@ -13,13 +13,15 @@ from django.core.exceptions import ObjectDoesNotExist
from django.db import OperationalError, ProgrammingError, connections, models
from django.db.models import CharField, Max, ProtectedError, TextField
from django.db.models.base import ModelBase
from django.db.models.signals import post_delete, post_save
from model_mommy import mommy
from model_mommy.mommy import foreign_key_required, make
from sapl.base.models import Autor, ProblemaMigracao
from sapl.base.models import Argumento, Autor, Constraint, ProblemaMigracao
from sapl.comissoes.models import Comissao, Composicao, Participacao
from sapl.legacy.models import Protocolo as ProtocoloLegado
from sapl.materia.models import (StatusTramitacao, TipoDocumento,
from sapl.materia.models import (DocumentoAcessorio, MateriaLegislativa,
StatusTramitacao, TipoDocumento,
TipoMateriaLegislativa, TipoProposicao,
Tramitacao)
from sapl.norma.models import (AssuntoNorma, NormaJuridica,
@ -28,7 +30,7 @@ from sapl.parlamentares.models import Parlamentar
from sapl.protocoloadm.models import Protocolo, StatusTramitacaoAdministrativo
from sapl.sessao.models import ExpedienteMateria, OrdemDia
from sapl.settings import PROJECT_DIR
from sapl.utils import normalize
from sapl.utils import delete_texto, normalize, save_texto
# BASE ######################################################################
# apps to be migrated, in app dependency order (very important)
@ -109,6 +111,10 @@ def warn(msg):
print('CUIDADO! ' + msg)
def erro(msg):
print('ERRO: ' + msg)
def get_fk_related(field, value, label=None):
if value is None and field.null is False:
value = 0
@ -197,6 +203,12 @@ def iter_sql_records(sql, db):
record.__dict__.update(zip(fieldnames, row))
yield record
# Todos os models têm no máximo uma constraint unique together
# Isso é necessário para que o método delete_constraints funcione corretamente
assert all(len(model._meta.unique_together) <= 1
for app in appconfs
for model in app.models.values())
def delete_constraints(model):
# pega nome da unique constraint dado o nome da tabela
@ -210,40 +222,66 @@ def delete_constraints(model):
for r in result:
if r[0].endswith('key'):
words_list = r[0].split('_')
one_to_one_constraints.append([table, r[0], words_list, model])
constraint = Constraint.objects.create(
nome_tabela=table, nome_constraint=r[0],
nome_model=model.__name__, tipo_constraint='one_to_one')
for w in words_list:
Argumento.objects.create(constraint=constraint, argumento=w)
else:
args = None
args_list = []
if model._meta.unique_together:
args = model._meta.unique_together[0]
args_list = list(args)
unique_constraints.append([table, r[0], args_list, model])
args_list = model._meta.unique_together[0]
constraint = Constraint.objects.create(
nome_tabela=table, nome_constraint=r[0],
nome_model=model.__name__,
tipo_constraint='unique_together')
for a in args_list:
Argumento.objects.create(constraint=constraint,
argumento=a)
warn('Excluindo unique constraint de nome %s' % r[0])
exec_sql("ALTER TABLE %s DROP CONSTRAINT %s;" %
(table, r[0]))
def recreate_constraints():
if one_to_one_constraints:
for constraint in one_to_one_constraints:
table, name, args, model = constraint
def recria_constraints():
constraints = Constraint.objects.all()
for con in constraints:
if con.tipo_constraint == 'one_to_one':
nome_tabela = con.nome_tabela
nome_constraint = con.nome_constraint
args = [a.argumento for a in con.argumento_set.all()]
args_string = ''
args_string = "(" + "_".join(map(str, args[2:-1])) + ")"
exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" %
(table, name, args_string))
if unique_constraints:
for constraint in unique_constraints:
table, name, args, model = constraint
try:
exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" %
(nome_tabela, nome_constraint, args_string))
except ProgrammingError:
info('A constraint %s já foi recriada!' % nome_constraint)
if con.tipo_constraint == 'unique_together':
nome_tabela = con.nome_tabela
nome_constraint = con.nome_constraint
# Pegando explicitamente o primeiro valor do filter,
# pois pode ser que haja mais de uma ocorrência
model = ContentType.objects.filter(
model=con.nome_model.lower())[0].model_class()
args = [a.argumento for a in con.argumento_set.all()]
for i in range(len(args)):
if isinstance(model._meta.get_field(args[i]),
models.ForeignKey):
args[i] = args[i] + '_id'
args_string = ''
args_string += "(" + ', '.join(map(str, args)) + ")"
exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" %
(table, name, args_string))
one_to_one_constraints.clear()
unique_constraints.clear()
try:
exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" %
(nome_tabela, nome_constraint, args_string))
except ProgrammingError:
info('A constraint %s já foi recriada!' % nome_constraint)
except Exception as err:
problema = re.findall('\(.*?\)', err.args[0])
erro('A constraint [%s] da tabela [%s] não pode ser recriada' %
(nome_constraint, nome_tabela))
erro('Os dados %s = %s estão duplicados. '
'Arrume antes de recriar as constraints!' %
(problema[0], problema[1]))
def obj_desnecessario(obj):
@ -262,8 +300,8 @@ def get_last_value(model):
def alter_sequence(model, id):
sequence_name = '%s_id_seq' % model._meta.db_table
exec_sql('ALTER SEQUENCE %s RESTART WITH %s MINVALUE %s;' % (
sequence_name, id, id))
exec_sql('ALTER SEQUENCE %s RESTART WITH %s MINVALUE -1;' % (
sequence_name, id))
def save_with_id(new, id):
@ -278,8 +316,7 @@ def save_relation(obj, nome_campo='', problema='', descricao='',
eh_stub=False):
link = ProblemaMigracao(
content_object=obj, nome_campo=nome_campo, problema=problema,
descricao=descricao, eh_stub=eh_stub,
)
descricao=descricao, eh_stub=eh_stub,)
link.save()
@ -416,6 +453,8 @@ class DataMigrator:
call([PROJECT_DIR.child('manage.py'), 'flush',
'--database=default', '--no-input'], stdout=PIPE)
disconecta_sinais_indexacao()
info('Começando migração: %s...' % obj)
self._do_migrate(obj)
@ -427,7 +466,7 @@ class DataMigrator:
for obj in self.to_delete:
msg = 'A entrada de PK %s da model %s não pode ser ' \
'excluida' % (obj.pk, obj._meta.model_name)
descricao = 'Um ou mais objetos protegidos '
descricao = 'Um ou mais objetos protegidos'
warn(msg + ' => ' + descricao)
save_relation(obj=obj, problema=msg,
descricao=descricao, eh_stub=False)
@ -435,8 +474,8 @@ class DataMigrator:
info('Deletando stubs desnecessários...')
while self.delete_stubs():
pass
info('Recriando unique constraints...')
# recreate_constraints()
conecta_sinais_indexacao()
def _do_migrate(self, obj):
if isinstance(obj, AppConfig):
@ -731,3 +770,23 @@ def make_with_log(model, _quantity=None, make_m2m=False, **attrs):
return stub
make_with_log.required = foreign_key_required
# DISCONNECT SIGNAL ########################################################
def disconecta_sinais_indexacao():
post_save.disconnect(save_texto, NormaJuridica)
post_save.disconnect(save_texto, DocumentoAcessorio)
post_save.disconnect(save_texto, MateriaLegislativa)
post_delete.disconnect(delete_texto, NormaJuridica)
post_delete.disconnect(delete_texto, DocumentoAcessorio)
post_delete.disconnect(delete_texto, MateriaLegislativa)
def conecta_sinais_indexacao():
post_save.connect(save_texto, NormaJuridica)
post_save.connect(save_texto, DocumentoAcessorio)
post_save.connect(save_texto, MateriaLegislativa)
post_delete.connect(delete_texto, NormaJuridica)
post_delete.connect(delete_texto, DocumentoAcessorio)
post_delete.connect(delete_texto, MateriaLegislativa)

6
sapl/legacy/scripts/migra_um_db.sh

@ -23,3 +23,9 @@ echo "--- MIGRACAO DE DADOS ---" | tee -a $LOG
echo >> $LOG
DATABASE_NAME=$1 ./manage.py migracao_25_31 -f --settings sapl.legacy_migration_settings |& tee -a $LOG
echo >> $LOG
echo "--- RECRIANDO CONSTRAINTS ---" | tee -a $LOG
echo >> $LOG
DATABASE_NAME=$1 ./manage.py recria_constraints --settings sapl.legacy_migration_settings |& tee -a $LOG
echo >> $LOG

22
sapl/materia/signals.py

@ -1,29 +1,9 @@
from subprocess import PIPE, call
from threading import Thread
from django.db.models.signals import post_delete, post_save
from sapl.settings import PROJECT_DIR
from sapl.utils import save_texto, delete_texto
from .models import DocumentoAcessorio, MateriaLegislativa
class UpdateIndexCommand(Thread):
def run(self):
call([PROJECT_DIR.child('manage.py'), 'update_index'],
stdout=PIPE)
def save_texto(sender, instance, **kwargs):
update_index = UpdateIndexCommand()
update_index.start()
def delete_texto(sender, instance, **kwargs):
update_index = UpdateIndexCommand()
update_index.start()
post_save.connect(save_texto, sender=MateriaLegislativa)
post_save.connect(save_texto, sender=DocumentoAcessorio)
post_delete.connect(delete_texto, sender=MateriaLegislativa)

22
sapl/norma/signals.py

@ -1,28 +1,8 @@
from subprocess import PIPE, call
from threading import Thread
from django.db.models.signals import post_delete, post_save
from sapl.settings import PROJECT_DIR
from sapl.utils import save_texto, delete_texto
from .models import NormaJuridica
class UpdateIndexCommand(Thread):
def run(self):
call([PROJECT_DIR.child('manage.py'), 'update_index'],
stdout=PIPE)
def save_texto(sender, instance, **kwargs):
update_index = UpdateIndexCommand()
update_index.start()
def delete_texto(sender, instance, **kwargs):
update_index = UpdateIndexCommand()
update_index.start()
post_save.connect(save_texto, sender=NormaJuridica)
post_delete.connect(delete_texto, sender=NormaJuridica)

2
sapl/rules/map_rules.py

@ -197,6 +197,8 @@ rules_group_geral = {
(base.CasaLegislativa, __listdetailchange__),
(base.ProblemaMigracao, []),
(base.Argumento, []),
(base.Constraint, []),
(base.TipoAutor, __base__),
(base.Autor, __base__),

13
sapl/rules/tests/test_rules.py

@ -6,7 +6,8 @@ from django.contrib.contenttypes.models import ContentType
from django.utils import six
from django.utils.translation import ugettext_lazy as _
from sapl.base.models import CasaLegislativa, ProblemaMigracao
from sapl.base.models import (CasaLegislativa, ProblemaMigracao, Argumento,
Constraint)
from sapl.compilacao.models import (PerfilEstruturalTextoArticulado,
TipoDispositivo,
TipoDispositivoRelationship)
@ -56,11 +57,15 @@ def test_models_in_rules_patterns(model_item):
__fp__in__test_permission_of_models_in_rules_patterns = {
map_rules.RP_ADD: [CasaLegislativa,
ProblemaMigracao,
Argumento,
Constraint,
TipoDispositivo,
TipoDispositivoRelationship,
PerfilEstruturalTextoArticulado],
map_rules.RP_CHANGE: [ProblemaMigracao,
Argumento,
Constraint,
AcompanhamentoMateria,
TipoDispositivo,
TipoDispositivoRelationship,
@ -68,17 +73,23 @@ __fp__in__test_permission_of_models_in_rules_patterns = {
map_rules.RP_DELETE: [CasaLegislativa,
ProblemaMigracao,
Argumento,
Constraint,
TipoDispositivo,
TipoDispositivoRelationship,
PerfilEstruturalTextoArticulado],
map_rules.RP_LIST: [ProblemaMigracao,
Argumento,
Constraint,
AcompanhamentoMateria,
TipoDispositivo,
TipoDispositivoRelationship,
PerfilEstruturalTextoArticulado],
map_rules.RP_DETAIL: [ProblemaMigracao,
Argumento,
Constraint,
AcompanhamentoMateria,
TipoDispositivo,
TipoDispositivoRelationship,

20
sapl/utils.py

@ -5,6 +5,8 @@ import re
from datetime import date
from functools import wraps
from unicodedata import normalize as unicodedata_normalize
from subprocess import PIPE, call
from threading import Thread
import django_filters
import magic
@ -22,7 +24,7 @@ from floppyforms import ClearableFileInput
from reversion.admin import VersionAdmin
from sapl.crispy_layout_mixin import SaplFormLayout, form_actions, to_row
from sapl.settings import BASE_DIR
from sapl.settings import BASE_DIR, PROJECT_DIR
sapl_logger = logging.getLogger(BASE_DIR.name)
@ -632,3 +634,19 @@ def texto_upload_path(instance, filename, subpath=''):
}
return path
class UpdateIndexCommand(Thread):
def run(self):
call([PROJECT_DIR.child('manage.py'), 'update_index'],
stdout=PIPE)
def save_texto(sender, instance, **kwargs):
update_index = UpdateIndexCommand()
update_index.start()
def delete_texto(sender, instance, **kwargs):
update_index = UpdateIndexCommand()
update_index.start()

Loading…
Cancel
Save