From 183c7334959eb1d4bd104d1687158505898519ee Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Tue, 19 Apr 2016 14:52:08 -0300 Subject: [PATCH] =?UTF-8?q?Arruma=20exclus=C3=A3o=20de=20stubs=20desnecess?= =?UTF-8?q?=C3=A1rios?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Luciano Almeida --- base/admin.py | 16 ++++ base/migrations/0011_problemamigracao.py | 32 +++++++ base/models.py | 19 +++- legacy/migration.py | 96 ++++++++++++++++--- materia/migrations/0028_auto_20160419_1056.py | 26 +++++ materia/models.py | 4 +- sapl/legacy_migration_settings.py | 4 + 7 files changed, 181 insertions(+), 16 deletions(-) create mode 100644 base/migrations/0011_problemamigracao.py create mode 100644 materia/migrations/0028_auto_20160419_1056.py diff --git a/base/admin.py b/base/admin.py index 5c02ebd40..c97c44834 100644 --- a/base/admin.py +++ b/base/admin.py @@ -1,3 +1,19 @@ from sapl.utils import register_all_models_in_admin +from django.contrib import admin +from base.models import ProblemaMigracao register_all_models_in_admin(__name__) + +admin.site.unregister(ProblemaMigracao) + + +@admin.register(ProblemaMigracao) +class ProblemaMigracaoAdmin(admin.ModelAdmin): + list_display = ["content_type", "object_id", "problema", + "descricao", "get_url"] + + def get_url(self, obj): + return "%s" % (obj.endereco, obj.endereco) + + get_url.short_description = "Endereco" + get_url.allow_tags = True diff --git a/base/migrations/0011_problemamigracao.py b/base/migrations/0011_problemamigracao.py new file mode 100644 index 000000000..aa8eae5ce --- /dev/null +++ b/base/migrations/0011_problemamigracao.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9 on 2016-04-19 16:02 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('base', '0010_auto_20160309_1323'), + ] + + operations = [ + migrations.CreateModel( + name='ProblemaMigracao', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('object_id', models.PositiveIntegerField(verbose_name='ID do Objeto')), + ('problema', models.CharField(max_length=300, null=True, verbose_name='Problema')), + ('descricao', models.CharField(max_length=300, null=True, verbose_name='Descrição')), + ('endereco', models.URLField(null=True, verbose_name='Endereço')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', verbose_name='Tipo de Content')), + ], + options={ + 'verbose_name_plural': 'Problemas na Migração', + 'verbose_name': 'Problema na Migração', + }, + ), + ] diff --git a/base/models.py b/base/models.py index ad7108bac..f9a63251e 100644 --- a/base/models.py +++ b/base/models.py @@ -1,6 +1,7 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ - +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType from sapl.utils import UF @@ -45,3 +46,19 @@ class CasaLegislativa(models.Model): class Meta: verbose_name = _('Casa Legislativa') verbose_name_plural = _('Casas Legislativas') + + +class ProblemaMigracao(models.Model): + content_type = models.ForeignKey(ContentType, + verbose_name=_('Tipo de Content')) + object_id = models.PositiveIntegerField(verbose_name=_('ID do Objeto')) + content_object = GenericForeignKey('content_type', 'object_id') + problema = models.CharField(max_length=300, null=True, + verbose_name=_('Problema')) + descricao = models.CharField(max_length=300, null=True, + verbose_name=_('Descrição')) + endereco = models.URLField(null=True, verbose_name=_('Endereço')) + + class Meta: + verbose_name = _('Problema na Migração') + verbose_name_plural = _('Problemas na Migração') diff --git a/legacy/migration.py b/legacy/migration.py index f1291ca16..f4a01b93c 100644 --- a/legacy/migration.py +++ b/legacy/migration.py @@ -5,12 +5,17 @@ import yaml from django.apps import apps from django.apps.config import AppConfig from django.core.exceptions import ObjectDoesNotExist +from django.core.urlresolvers import reverse from django.db import connections, models -from django.db.models import CharField, TextField +from django.db.models import CharField, ForeignKey, TextField from django.db.models.base import ModelBase + from model_mommy import mommy +from model_mommy.mommy import make, foreign_key_required +from base.models import ProblemaMigracao from comissoes.models import Composicao, Participacao +from materia.models import TipoMateriaLegislativa from parlamentares.models import Parlamentar from sessao.models import SessaoPlenaria @@ -26,7 +31,6 @@ appconfs = [apps.get_app_config(n) for n in [ 'lexml', 'protocoloadm', ]] -stubs_list = [] unique_constraints = [] name_sets = [set(m.__name__ for m in ac.get_models()) for ac in appconfs] @@ -81,9 +85,9 @@ def get_renames(): return field_renames, model_renames - # MIGRATION ################################################################# + def info(msg): print('INFO: ' + msg) @@ -115,14 +119,17 @@ def get_fk_related(field, value, label=None): not f.choices and not f.blank} value = mommy.make(field.related_model, **fields_dict) - warn(msg + ' => STUB criada para campos não nuláveis!') + descricao = 'stub criado para campos não nuláveis!' + save_relation(value, msg, descricao) + warn(msg + ' => ' + descricao) else: value = None warn(msg + ' => usando None para valores iguais a zero!') else: value = make_stub(field.related_model, value) - stubs_list.append((value.id, field)) - warn(msg + ' => STUB criada!') + descricao = 'stub criado para entrada orfã!' + warn(msg + ' => ' + descricao) + save_relation(value, msg, descricao) else: assert value return value @@ -132,6 +139,11 @@ def get_field(model, fieldname): return model._meta.get_field(fieldname) +def get_url(object): + info = (object._meta.app_label, object._meta.model_name) + return reverse('admin:%s_%s_change' % info, args=(object.pk,)) + + def exec_sql(sql, db='default'): cursor = connections[db].cursor() cursor.execute(sql) @@ -178,6 +190,23 @@ def recreate_constraints(): args_string += "(" + ', '.join(map(str, args)) + ")" exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" % (table, name, args_string)) + unique_constraints.clear() + + +def stub_desnecessario(obj): + lista = [ + f for f in obj._meta.get_fields() + if (f.one_to_many or f.one_to_one) and f.auto_created + ] + desnecessario = not any( + rr.related_model.objects.filter(**{rr.field.name: obj}).exists() + for rr in lista) + if isinstance(obj, TipoMateriaLegislativa): + desnecessario = not any( + rr.related_model.objects.filter( + **{rr.field.name+'_origem_externa': obj}).exists() + for rr in lista) + return desnecessario def save_with_id(new, id): @@ -193,9 +222,16 @@ def save_with_id(new, id): assert new.id == id, 'New id is different from provided!' +def save_relation(obj, problema='', descricao=''): + link = ProblemaMigracao(content_object=obj, problema=problema, + descricao=descricao, endereco=get_url(obj)) + link.save() + + def make_stub(model, id): new = mommy.prepare(model) save_with_id(new, id) + return new @@ -249,6 +285,7 @@ class DataMigrator: # warning: model/app migration order is of utmost importance self.to_delete = [] + ProblemaMigracao.objects.all().delete() info('Começando migração: %s...' % obj) self._do_migrate(obj) # exclude logically deleted in legacy base @@ -256,7 +293,8 @@ class DataMigrator: for obj in self.to_delete: obj.delete() info('Deletando stubs desnecessários...') - self.delete_stubs() + while self.delete_stubs(): + pass info('Recriando unique constraints...') recreate_constraints() @@ -313,12 +351,24 @@ class DataMigrator: self.to_delete.append(new) def delete_stubs(self): - for line in stubs_list: - stub, field = line - # Filter all objects in model and delete from related model - # if quantity is equal to zero - if field.model.objects.filter(**{field.name: stub}).exists(): - field.related_model.objects.get(**{'id': stub}).delete() + excluidos = 0 + for obj in ProblemaMigracao.objects.all().reverse(): + if obj.content_object: + original = obj.content_type.get_all_objects_for_this_type( + id=obj.object_id) + if stub_desnecessario(original[0]): + # Se qtd_exclusoes for maior que 1, está deletando mais + # objetos do que deveria.. + qtd_exclusoes, *_ = original.delete() + assert qtd_exclusoes == 1 + qtd_exclusoes, *_ = obj.delete() + assert qtd_exclusoes == 1 + excluidos = excluidos + 1 + else: + resultado, *_ = obj.delete() + assert resultado == 1 + excluidos = excluidos + 1 + return excluidos def migrate(obj=appconfs): @@ -378,3 +428,23 @@ def check_app_no_ind_excluido(app): for model in app.models.values(): assert not any(get_ind_excluido(obj) for obj in model.objects.all()) print('OK!') + +# MOMMY MAKE WITH LOG ###################################################### + + +def make_with_log(model, _quantity=None, make_m2m=False, **attrs): + all_fields = model._meta.get_fields() + fields_dict = {} + fields_dict = {f.name: '????????????'[:f.max_length] + for f in all_fields + if isinstance(f, (CharField, TextField)) and + not f.choices and not f.blank} + stub = make(model, _quantity, make_m2m, **fields_dict) + problema = 'Um stub foi necessário durante a criação de um outro stub' + descricao = 'Essa entrada é necessária para um dos stubs criados' + ' anteriormente' + warn(problema) + save_relation(obj=stub, problema=problema, descricao=descricao) + return stub + +make_with_log.required = foreign_key_required diff --git a/materia/migrations/0028_auto_20160419_1056.py b/materia/migrations/0028_auto_20160419_1056.py new file mode 100644 index 000000000..bd4eb7f22 --- /dev/null +++ b/materia/migrations/0028_auto_20160419_1056.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9 on 2016-04-19 13:56 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('materia', '0027_auto_20160404_1409'), + ] + + operations = [ + migrations.AlterField( + model_name='tramitacao', + name='unidade_tramitacao_destino', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tramitacoes_destino', to='materia.UnidadeTramitacao', verbose_name='Unidade Destino'), + ), + migrations.AlterField( + model_name='tramitacao', + name='unidade_tramitacao_local', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tramitacoes_origem', to='materia.UnidadeTramitacao', verbose_name='Unidade Local'), + ), + ] diff --git a/materia/models.py b/materia/models.py index d80d751c4..c241f7632 100644 --- a/materia/models.py +++ b/materia/models.py @@ -532,13 +532,13 @@ class Tramitacao(models.Model): data_tramitacao = models.DateField(verbose_name=_('Data Tramitação')) unidade_tramitacao_local = models.ForeignKey( UnidadeTramitacao, - related_name='+', + related_name='tramitacoes_origem', verbose_name=_('Unidade Local')) data_encaminhamento = models.DateField( blank=True, null=True, verbose_name=_('Data Encaminhamento')) unidade_tramitacao_destino = models.ForeignKey( UnidadeTramitacao, - related_name='+', + related_name='tramitacoes_destino', verbose_name=_('Unidade Destino')) urgente = models.BooleanField(verbose_name=_('Urgente ?')) turno = models.CharField( diff --git a/sapl/legacy_migration_settings.py b/sapl/legacy_migration_settings.py index 433cc2329..4c0425e57 100644 --- a/sapl/legacy_migration_settings.py +++ b/sapl/legacy_migration_settings.py @@ -16,3 +16,7 @@ DATABASES['legacy'] = { } DATABASE_ROUTERS = ['legacy.router.LegacyRouter', ] + +MOMMY_CUSTOM_FIELDS_GEN = { + 'django.db.models.ForeignKey': 'legacy.migration.make_with_log' +}