diff --git a/base/admin.py b/base/admin.py index 5c02ebd40..99f81f8af 100644 --- a/base/admin.py +++ b/base/admin.py @@ -1,3 +1,26 @@ +from django.contrib import admin +from django.core.urlresolvers import reverse + +from base.models import ProblemaMigracao from sapl.utils import register_all_models_in_admin register_all_models_in_admin(__name__) + +admin.site.unregister(ProblemaMigracao) + + +@admin.register(ProblemaMigracao) +class ProblemaMigracaoAdmin(admin.ModelAdmin): + list_display = ["content_type", "object_id", "problema", + "descricao", "get_url"] + + def get_url(self, obj): + + info = (obj.content_object._meta.app_label, + obj.content_object._meta.model_name) + endereco = reverse('admin:%s_%s_change' % info, + args=(obj.content_object.pk,)) + return "%s" % (endereco, endereco) + + get_url.short_description = "Endereço" + get_url.allow_tags = True diff --git a/base/migrations/0011_problemamigracao.py b/base/migrations/0011_problemamigracao.py new file mode 100644 index 000000000..aa8eae5ce --- /dev/null +++ b/base/migrations/0011_problemamigracao.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9 on 2016-04-19 16:02 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('base', '0010_auto_20160309_1323'), + ] + + operations = [ + migrations.CreateModel( + name='ProblemaMigracao', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('object_id', models.PositiveIntegerField(verbose_name='ID do Objeto')), + ('problema', models.CharField(max_length=300, null=True, verbose_name='Problema')), + ('descricao', models.CharField(max_length=300, null=True, verbose_name='Descrição')), + ('endereco', models.URLField(null=True, verbose_name='Endereço')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', verbose_name='Tipo de Content')), + ], + options={ + 'verbose_name_plural': 'Problemas na Migração', + 'verbose_name': 'Problema na Migração', + }, + ), + ] diff --git a/base/migrations/0012_problemamigracao_eh_stub.py b/base/migrations/0012_problemamigracao_eh_stub.py new file mode 100644 index 000000000..b145c3383 --- /dev/null +++ b/base/migrations/0012_problemamigracao_eh_stub.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-04-26 17:41 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0011_problemamigracao'), + ] + + operations = [ + migrations.AddField( + model_name='problemamigracao', + name='eh_stub', + field=models.BooleanField(default=False, verbose_name='É stub?'), + preserve_default=False, + ), + ] diff --git a/base/migrations/0013_remove_problemamigracao_endereco.py b/base/migrations/0013_remove_problemamigracao_endereco.py new file mode 100644 index 000000000..67526693d --- /dev/null +++ b/base/migrations/0013_remove_problemamigracao_endereco.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-05-02 17:36 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0012_problemamigracao_eh_stub'), + ] + + operations = [ + migrations.RemoveField( + model_name='problemamigracao', + name='endereco', + ), + ] diff --git a/base/migrations/0014_auto_20160502_1635.py b/base/migrations/0014_auto_20160502_1635.py new file mode 100644 index 000000000..6fbae0bb9 --- /dev/null +++ b/base/migrations/0014_auto_20160502_1635.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-05-02 19:35 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0013_remove_problemamigracao_endereco'), + ] + + operations = [ + migrations.AlterField( + model_name='problemamigracao', + name='descricao', + field=models.CharField(default='', max_length=300, verbose_name='Descrição'), + preserve_default=False, + ), + migrations.AlterField( + model_name='problemamigracao', + name='problema', + field=models.CharField(default='', max_length=300, verbose_name='Problema'), + preserve_default=False, + ), + ] diff --git a/base/models.py b/base/models.py index ad7108bac..f1fb3811f 100644 --- a/base/models.py +++ b/base/models.py @@ -1,3 +1,5 @@ +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType from django.db import models from django.utils.translation import ugettext_lazy as _ @@ -45,3 +47,17 @@ class CasaLegislativa(models.Model): class Meta: verbose_name = _('Casa Legislativa') verbose_name_plural = _('Casas Legislativas') + + +class ProblemaMigracao(models.Model): + content_type = models.ForeignKey(ContentType, + verbose_name=_('Tipo de Content')) + object_id = models.PositiveIntegerField(verbose_name=_('ID do Objeto')) + content_object = GenericForeignKey('content_type', 'object_id') + problema = models.CharField(max_length=300, verbose_name=_('Problema')) + descricao = models.CharField(max_length=300, verbose_name=_('Descrição')) + eh_stub = models.BooleanField(verbose_name='É stub?') + + class Meta: + verbose_name = _('Problema na Migração') + verbose_name_plural = _('Problemas na Migração') diff --git a/legacy/migration.py b/legacy/migration.py index f1291ca16..ddc8e0bfd 100644 --- a/legacy/migration.py +++ b/legacy/migration.py @@ -9,7 +9,9 @@ from django.db import connections, models from django.db.models import CharField, TextField from django.db.models.base import ModelBase from model_mommy import mommy +from model_mommy.mommy import foreign_key_required, make +from base.models import ProblemaMigracao from comissoes.models import Composicao, Participacao from parlamentares.models import Parlamentar from sessao.models import SessaoPlenaria @@ -26,7 +28,6 @@ appconfs = [apps.get_app_config(n) for n in [ 'lexml', 'protocoloadm', ]] -stubs_list = [] unique_constraints = [] name_sets = [set(m.__name__ for m in ac.get_models()) for ac in appconfs] @@ -81,9 +82,9 @@ def get_renames(): return field_renames, model_renames - # MIGRATION ################################################################# + def info(msg): print('INFO: ' + msg) @@ -93,7 +94,6 @@ def warn(msg): def get_fk_related(field, value, label=None): - fields_dict = {} if value is None and field.null is False: value = 0 if value is not None: @@ -105,24 +105,20 @@ def get_fk_related(field, value, label=None): field.name, value, field.model.__name__, label or '---') if value == 0: - # se FK == 0, criamos um stub e colocamos o valor '????????' - # para qualquer CharField ou TextField que possa haver if not field.null: - all_fields = field.related_model._meta.get_fields() - fields_dict = {f.name: '????????????'[:f.max_length] - for f in all_fields - if isinstance(f, (CharField, TextField)) and - not f.choices and not f.blank} + fields_dict = get_fields_dict(field.related_model) value = mommy.make(field.related_model, **fields_dict) - warn(msg + ' => STUB criada para campos não nuláveis!') + descricao = 'stub criado para campos não nuláveis!' + save_relation(value, msg, descricao, eh_stub=True) + warn(msg + ' => ' + descricao) else: value = None - warn(msg + ' => usando None para valores iguais a zero!') else: value = make_stub(field.related_model, value) - stubs_list.append((value.id, field)) - warn(msg + ' => STUB criada!') + descricao = 'stub criado para entrada orfã!' + warn(msg + ' => ' + descricao) + save_relation(value, msg, descricao, eh_stub=True) else: assert value return value @@ -173,11 +169,23 @@ def recreate_constraints(): for i in range(len(args)): if isinstance(model._meta.get_field(args[i]), models.ForeignKey): - args[i] = args[i]+'_id' + args[i] = args[i] + '_id' args_string = '' args_string += "(" + ', '.join(map(str, args)) + ")" exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" % (table, name, args_string)) + unique_constraints.clear() + + +def stub_desnecessario(obj): + lista_fields = [ + f for f in obj._meta.get_fields() + if (f.one_to_many or f.one_to_one) and f.auto_created + ] + desnecessario = not any( + rr.related_model.objects.filter(**{rr.field.name: obj}).exists() + for rr in lista_fields) + return desnecessario def save_with_id(new, id): @@ -193,15 +201,34 @@ def save_with_id(new, id): assert new.id == id, 'New id is different from provided!' +def save_relation(obj, problema='', descricao='', eh_stub=False): + link = ProblemaMigracao(content_object=obj, problema=problema, + descricao=descricao, eh_stub=eh_stub) + link.save() + + def make_stub(model, id): - new = mommy.prepare(model) + fields_dict = get_fields_dict(model) + new = mommy.prepare(model, **fields_dict) save_with_id(new, id) + return new +def get_fields_dict(model): + all_fields = model._meta.get_fields() + fields_dict = {} + fields_dict = {f.name: '????????????'[:f.max_length] + for f in all_fields + if isinstance(f, (CharField, TextField)) and + not f.choices and not f.blank} + return fields_dict + + class DataMigrator: def __init__(self): self.field_renames, self.model_renames = get_renames() + self.data_mudada = {} def populate_renamed_fields(self, new, old): renames = self.field_renames[type(new)] @@ -209,7 +236,7 @@ class DataMigrator: for field in new._meta.fields: old_field_name = renames.get(field.name) field_type = field.get_internal_type() - msg = ("Campo %s (%s) da model %s " % + msg = ("O valor do campo %s (%s) da model %s era inválido" % (field.name, field_type, field.model.__name__)) if old_field_name: old_value = getattr(old, old_field_name) @@ -230,18 +257,24 @@ class DataMigrator: combined_names = "(" + ")|(".join(names) + ")" matches = re.search('(ano_\w+)', combined_names) if not matches: + descricao = 'A data 0001-01-01 foi colocada no lugar' warn(msg + - '=> colocando valor 0000-01-01 para DateField') + ' => ' + descricao) value = '0001-01-01' + self.data_mudada['obj'] = new + self.data_mudada['descricao'] = descricao + self.data_mudada['problema'] = msg else: value = '%d-01-01' % getattr(old, matches.group(0)) + descricao = ('A data %s para foi colocada no lugar' + % value) + self.data_mudada['obj'] = new + self.data_mudada['descricao'] = descricao + self.data_mudada['problema'] = msg warn(msg + - "=> colocando %s para DateField não nulável" % - (value)) + '=> ' + descricao) if field_type == 'CharField' or field_type == 'TextField': if value is None: - warn(msg + "=> colocando string vazia para valor %s" % - (value)) value = '' setattr(new, field.name, value) @@ -249,6 +282,7 @@ class DataMigrator: # warning: model/app migration order is of utmost importance self.to_delete = [] + ProblemaMigracao.objects.all().delete() info('Começando migração: %s...' % obj) self._do_migrate(obj) # exclude logically deleted in legacy base @@ -256,7 +290,8 @@ class DataMigrator: for obj in self.to_delete: obj.delete() info('Deletando stubs desnecessários...') - self.delete_stubs() + while self.delete_stubs(): + pass info('Recriando unique constraints...') recreate_constraints() @@ -309,16 +344,29 @@ class DataMigrator: if adjust: adjust(new, old) save(new, old) + if self.data_mudada: + save_relation(**self.data_mudada) + self.data_mudada.clear() if getattr(old, 'ind_excluido', False): self.to_delete.append(new) def delete_stubs(self): - for line in stubs_list: - stub, field = line - # Filter all objects in model and delete from related model - # if quantity is equal to zero - if field.model.objects.filter(**{field.name: stub}).exists(): - field.related_model.objects.get(**{'id': stub}).delete() + excluidos = 0 + for obj in ProblemaMigracao.objects.all(): + if obj.content_object and obj.eh_stub: + original = obj.content_type.get_all_objects_for_this_type( + id=obj.object_id) + if stub_desnecessario(original[0]): + qtd_exclusoes, *_ = original.delete() + assert qtd_exclusoes == 1 + qtd_exclusoes, *_ = obj.delete() + assert qtd_exclusoes == 1 + excluidos = excluidos + 1 + elif not obj.content_object and not obj.eh_stub: + qtd_exclusoes, *_ = obj.delete() + assert qtd_exclusoes == 1 + excluidos = excluidos + 1 + return excluidos def migrate(obj=appconfs): @@ -346,13 +394,14 @@ def adjust_participacao(new_participacao, old): def adjust_parlamentar(new_parlamentar, old): - value = new_parlamentar.unidade_deliberativa - # Field is defined as not null in legacy db, - # but data includes null values - # => transform None to False - if value is None: - warn('nulo convertido para falso') - new_parlamentar.unidade_deliberativa = False + if old.ind_unid_deliberativa: + value = new_parlamentar.unidade_deliberativa + # Field is defined as not null in legacy db, + # but data includes null values + # => transform None to False + if value is None: + warn('nulo convertido para falso') + new_parlamentar.unidade_deliberativa = False def adjust_sessaoplenaria(new, old): @@ -378,3 +427,18 @@ def check_app_no_ind_excluido(app): for model in app.models.values(): assert not any(get_ind_excluido(obj) for obj in model.objects.all()) print('OK!') + +# MOMMY MAKE WITH LOG ###################################################### + + +def make_with_log(model, _quantity=None, make_m2m=False, **attrs): + fields_dict = get_fields_dict(model) + stub = make(model, _quantity, make_m2m, **fields_dict) + problema = 'Um stub foi necessário durante a criação de um outro stub' + descricao = 'Essa entrada é necessária para um dos stubs criados' + ' anteriormente' + warn(problema) + save_relation(stub, problema, descricao, eh_stub=True) + return stub + +make_with_log.required = foreign_key_required diff --git a/materia/migrations/0028_auto_20160419_1056.py b/materia/migrations/0028_auto_20160419_1056.py new file mode 100644 index 000000000..bd4eb7f22 --- /dev/null +++ b/materia/migrations/0028_auto_20160419_1056.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9 on 2016-04-19 13:56 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('materia', '0027_auto_20160404_1409'), + ] + + operations = [ + migrations.AlterField( + model_name='tramitacao', + name='unidade_tramitacao_destino', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tramitacoes_destino', to='materia.UnidadeTramitacao', verbose_name='Unidade Destino'), + ), + migrations.AlterField( + model_name='tramitacao', + name='unidade_tramitacao_local', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tramitacoes_origem', to='materia.UnidadeTramitacao', verbose_name='Unidade Local'), + ), + ] diff --git a/materia/migrations/0029_merge.py b/materia/migrations/0029_merge.py new file mode 100644 index 000000000..3a37b2421 --- /dev/null +++ b/materia/migrations/0029_merge.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-04-19 18:02 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('materia', '0028_auto_20160419_1056'), + ('materia', '0028_auto_20160419_1000'), + ] + + operations = [ + ] diff --git a/materia/migrations/0030_auto_20160502_1630.py b/materia/migrations/0030_auto_20160502_1630.py new file mode 100644 index 000000000..6f8fd60be --- /dev/null +++ b/materia/migrations/0030_auto_20160502_1630.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-05-02 19:30 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('materia', '0029_merge'), + ] + + operations = [ + migrations.AlterField( + model_name='materialegislativa', + name='tipo_origem_externa', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='tipo_origem_externa_set', to='materia.TipoMateriaLegislativa', verbose_name='Tipo'), + ), + ] diff --git a/materia/models.py b/materia/models.py index 15e3e2590..0eb2037c8 100644 --- a/materia/models.py +++ b/materia/models.py @@ -80,7 +80,7 @@ class MateriaLegislativa(models.Model): TipoMateriaLegislativa, blank=True, null=True, - related_name='+', + related_name='tipo_origem_externa_set', verbose_name=_('Tipo')) numero_origem_externa = models.CharField( max_length=5, blank=True, verbose_name=_('Número')) @@ -538,13 +538,13 @@ class Tramitacao(models.Model): data_tramitacao = models.DateField(verbose_name=_('Data Tramitação')) unidade_tramitacao_local = models.ForeignKey( UnidadeTramitacao, - related_name='+', + related_name='tramitacoes_origem', verbose_name=_('Unidade Local')) data_encaminhamento = models.DateField( blank=True, null=True, verbose_name=_('Data Encaminhamento')) unidade_tramitacao_destino = models.ForeignKey( UnidadeTramitacao, - related_name='+', + related_name='tramitacoes_destino', verbose_name=_('Unidade Destino')) urgente = models.BooleanField(verbose_name=_('Urgente ?')) turno = models.CharField( diff --git a/sapl/legacy_migration_settings.py b/sapl/legacy_migration_settings.py index 433cc2329..4c0425e57 100644 --- a/sapl/legacy_migration_settings.py +++ b/sapl/legacy_migration_settings.py @@ -16,3 +16,7 @@ DATABASES['legacy'] = { } DATABASE_ROUTERS = ['legacy.router.LegacyRouter', ] + +MOMMY_CUSTOM_FIELDS_GEN = { + 'django.db.models.ForeignKey': 'legacy.migration.make_with_log' +}