From 4ffbb9404f3033c3bb8a81bac7911ae22a79398b Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Mon, 28 Mar 2016 16:51:23 -0300 Subject: [PATCH 01/16] =?UTF-8?q?Ajusta=20migra=C3=A7=C3=A3o=20para=20nova?= =?UTF-8?q?s=20mudan=C3=A7as=20das=20models?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Luciano Almeida --- legacy/migration.py | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/legacy/migration.py b/legacy/migration.py index d98b17b19..34c6d2a34 100644 --- a/legacy/migration.py +++ b/legacy/migration.py @@ -93,7 +93,6 @@ def warn(msg): def get_fk_related(field, value, label=None): fields_dict = {} - if value is None and field.null is False: value = 0 if value is not None: @@ -169,7 +168,6 @@ def make_stub(model, id): class DataMigrator: - def __init__(self): self.field_renames, self.model_renames = get_renames() @@ -179,6 +177,8 @@ class DataMigrator: for field in new._meta.fields: old_field_name = renames.get(field.name) field_type = field.get_internal_type() + msg = ("Field %s (%s) from model %s " % + (field.name, field_type, field.model.__name__)) if old_field_name: old_value = getattr(old, old_field_name) if isinstance(field, models.ForeignKey): @@ -191,13 +191,23 @@ class DataMigrator: value = get_fk_related(field, old_value, label) else: value = getattr(old, old_field_name) + if (field_type == 'DateField' and + field.null is False and value is None): + names = [old_fields.name for old_fields + in old._meta.get_fields()] + combined_names = "(" + ")|(".join(names) + ")" + matches = re.search('(ano_\w+)', combined_names) + if not matches: + warn(msg + '=> setting 0000-01-01 value to DateField') + value = '0001-01-01' + else: + value = '%d-01-01' % getattr(old, matches.group(0)) + warn(msg + "=> settig %s for not null DateField" % + (value)) if field_type == 'CharField' or field_type == 'TextField': if value is None: - warn( - "Field %s (%s) from model %s" - " => settig empty string '' for %s value" % - (field.name, field_type, field.model.__name__, - value)) + warn(msg + "=> settig empty string '' for %s value" % + (value)) value = '' setattr(new, field.name, value) From 7c94e0591d3eab56d3937031f63f4864391bcc4e Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Tue, 5 Apr 2016 10:30:39 -0300 Subject: [PATCH 02/16] Arruma conflito com master e traduz textos de info Signed-off-by: Luciano Almeida --- legacy/migration.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/legacy/migration.py b/legacy/migration.py index 34c6d2a34..685d9dcba 100644 --- a/legacy/migration.py +++ b/legacy/migration.py @@ -104,7 +104,7 @@ def get_fk_related(field, value, label=None): field.name, value, field.model.__name__, label or '---') if value == 0: - # se FK == 0, criamos um stub e colocamos o valor '???????? + # se FK == 0, criamos um stub e colocamos o valor '????????' # para qualquer CharField ou TextField que possa haver if not field.null: all_fields = field.related_model._meta.get_fields() @@ -177,7 +177,7 @@ class DataMigrator: for field in new._meta.fields: old_field_name = renames.get(field.name) field_type = field.get_internal_type() - msg = ("Field %s (%s) from model %s " % + msg = ("Campo %s (%s) da model %s " % (field.name, field_type, field.model.__name__)) if old_field_name: old_value = getattr(old, old_field_name) @@ -187,7 +187,7 @@ class DataMigrator: old_type._meta.pk.name != 'id': label = old.pk else: - label = '-- WITHOUT PK --' + label = '-- SEM PK --' value = get_fk_related(field, old_value, label) else: value = getattr(old, old_field_name) @@ -198,15 +198,17 @@ class DataMigrator: combined_names = "(" + ")|(".join(names) + ")" matches = re.search('(ano_\w+)', combined_names) if not matches: - warn(msg + '=> setting 0000-01-01 value to DateField') + warn(msg + + '=> colocando valor 0000-01-01 para DateField') value = '0001-01-01' else: value = '%d-01-01' % getattr(old, matches.group(0)) - warn(msg + "=> settig %s for not null DateField" % + warn(msg + + "=> colocando %s para DateField não nulável" % (value)) if field_type == 'CharField' or field_type == 'TextField': if value is None: - warn(msg + "=> settig empty string '' for %s value" % + warn(msg + "=> colocando string vazia para valor %s" % (value)) value = '' setattr(new, field.name, value) @@ -215,13 +217,13 @@ class DataMigrator: # warning: model/app migration order is of utmost importance self.to_delete = [] - info('Starting %s migration...' % obj) + info('Começando migração: %s...' % obj) self._do_migrate(obj) # exclude logically deleted in legacy base - info('Deleting models with ind_excluido...') + info('Deletando models com ind_excluido...') for obj in self.to_delete: obj.delete() - info('Deleting unnecessary stubs...') + info('Deletando stubs desnecessários...') self.delete_stubs() def _do_migrate(self, obj): @@ -239,7 +241,7 @@ class DataMigrator: 'Parameter must be a Model, AppConfig or a sequence of them') def migrate_model(self, model): - print('Migrating %s...' % model.__name__) + print('Migrando %s...' % model.__name__) legacy_model_name = self.model_renames.get(model, model.__name__) legacy_model = legacy_app.get_model(legacy_model_name) @@ -314,7 +316,7 @@ def adjust_parlamentar(new_parlamentar, old): # but data includes null values # => transform None to False if value is None: - warn('null converted to False') + warn('nulo convertido para falso') new_parlamentar.unidade_deliberativa = False From 017a3e80f331d5b69898436139a315ee6b8fb0bd Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Thu, 31 Mar 2016 11:40:00 -0300 Subject: [PATCH 03/16] Concerta problema de duplicidade Signed-off-by: Luciano Almeida --- legacy/migration.py | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/legacy/migration.py b/legacy/migration.py index 685d9dcba..0bd1f008a 100644 --- a/legacy/migration.py +++ b/legacy/migration.py @@ -28,6 +28,8 @@ appconfs = [apps.get_app_config(n) for n in [ stubs_list = [] +unique_constraints = [] + name_sets = [set(m.__name__ for m in ac.get_models()) for ac in appconfs] # apps do not overlap @@ -148,6 +150,39 @@ def iter_sql_records(sql, db): yield record +def delete_constraints(model): + global unique_constraints + # pega nome da unique constraint dado o nome da tabela + table = model._meta.db_table + cursor = exec_sql("SELECT conname FROM pg_constraint WHERE conrelid = " + "(SELECT oid FROM pg_class WHERE relname LIKE " + "'%s') and contype = 'u';" % (table)) + result = cursor.fetchone() + # if theres a result then delete + if result: + args = model._meta.unique_together[0] + args_list = list(args) + + unique_constraints.append([table, result[0], args_list, model]) + exec_sql("ALTER TABLE %s DROP CONSTRAINT %s;" % + (table, result[0])) + + +def recreate_constraints(): + global unique_constraints + if unique_constraints: + for constraint in unique_constraints: + table, name, args, model = constraint + for i in range(len(args)): + if isinstance(model._meta.get_field(args[i]), + models.ForeignKey): + args[i] = args[i]+'_id' + args_string = '' + args_string += "(" + ', '.join(map(str, args)) + ")" + exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" % + (table, name, args_string)) + + def save_with_id(new, id): sequence_name = '%s_id_seq' % type(new)._meta.db_table cursor = exec_sql('SELECT last_value from %s;' % sequence_name) @@ -225,6 +260,8 @@ class DataMigrator: obj.delete() info('Deletando stubs desnecessários...') self.delete_stubs() + info('Recreating unique constraints...') + recreate_constraints() def _do_migrate(self, obj): if isinstance(obj, AppConfig): @@ -250,6 +287,7 @@ class DataMigrator: # Clear all model entries # They may have been created in a previous migration attempt model.objects.all().delete() + delete_constraints(model) # setup migration strategy for tables with or without a pk if legacy_pk_name == 'id': From 658270bef026513765a7e2e50f9f6e1d4f50c645 Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Tue, 5 Apr 2016 13:59:10 -0300 Subject: [PATCH 04/16] Remove global e corrige alguns textos Signed-off-by: Luciano Almeida --- legacy/migration.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/legacy/migration.py b/legacy/migration.py index 0bd1f008a..f1291ca16 100644 --- a/legacy/migration.py +++ b/legacy/migration.py @@ -27,7 +27,6 @@ appconfs = [apps.get_app_config(n) for n in [ 'protocoloadm', ]] stubs_list = [] - unique_constraints = [] name_sets = [set(m.__name__ for m in ac.get_models()) for ac in appconfs] @@ -151,25 +150,23 @@ def iter_sql_records(sql, db): def delete_constraints(model): - global unique_constraints # pega nome da unique constraint dado o nome da tabela table = model._meta.db_table cursor = exec_sql("SELECT conname FROM pg_constraint WHERE conrelid = " "(SELECT oid FROM pg_class WHERE relname LIKE " "'%s') and contype = 'u';" % (table)) result = cursor.fetchone() - # if theres a result then delete + # se existir um resultado, unique constraint será deletado if result: + warn('Excluindo unique constraint de nome %s' % result) args = model._meta.unique_together[0] args_list = list(args) - unique_constraints.append([table, result[0], args_list, model]) exec_sql("ALTER TABLE %s DROP CONSTRAINT %s;" % (table, result[0])) def recreate_constraints(): - global unique_constraints if unique_constraints: for constraint in unique_constraints: table, name, args, model = constraint @@ -260,7 +257,7 @@ class DataMigrator: obj.delete() info('Deletando stubs desnecessários...') self.delete_stubs() - info('Recreating unique constraints...') + info('Recriando unique constraints...') recreate_constraints() def _do_migrate(self, obj): From 2f33438f3c520cca45c62f861842f620d170ce01 Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Tue, 19 Apr 2016 14:52:08 -0300 Subject: [PATCH 05/16] =?UTF-8?q?Arruma=20exclus=C3=A3o=20de=20stubs=20des?= =?UTF-8?q?necess=C3=A1rios?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Luciano Almeida --- base/admin.py | 16 ++++ base/migrations/0011_problemamigracao.py | 32 +++++++ base/models.py | 19 +++- legacy/migration.py | 96 ++++++++++++++++--- materia/migrations/0028_auto_20160419_1056.py | 26 +++++ materia/models.py | 4 +- sapl/legacy_migration_settings.py | 4 + 7 files changed, 181 insertions(+), 16 deletions(-) create mode 100644 base/migrations/0011_problemamigracao.py create mode 100644 materia/migrations/0028_auto_20160419_1056.py diff --git a/base/admin.py b/base/admin.py index 5c02ebd40..c97c44834 100644 --- a/base/admin.py +++ b/base/admin.py @@ -1,3 +1,19 @@ from sapl.utils import register_all_models_in_admin +from django.contrib import admin +from base.models import ProblemaMigracao register_all_models_in_admin(__name__) + +admin.site.unregister(ProblemaMigracao) + + +@admin.register(ProblemaMigracao) +class ProblemaMigracaoAdmin(admin.ModelAdmin): + list_display = ["content_type", "object_id", "problema", + "descricao", "get_url"] + + def get_url(self, obj): + return "%s" % (obj.endereco, obj.endereco) + + get_url.short_description = "Endereco" + get_url.allow_tags = True diff --git a/base/migrations/0011_problemamigracao.py b/base/migrations/0011_problemamigracao.py new file mode 100644 index 000000000..aa8eae5ce --- /dev/null +++ b/base/migrations/0011_problemamigracao.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9 on 2016-04-19 16:02 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('base', '0010_auto_20160309_1323'), + ] + + operations = [ + migrations.CreateModel( + name='ProblemaMigracao', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('object_id', models.PositiveIntegerField(verbose_name='ID do Objeto')), + ('problema', models.CharField(max_length=300, null=True, verbose_name='Problema')), + ('descricao', models.CharField(max_length=300, null=True, verbose_name='Descrição')), + ('endereco', models.URLField(null=True, verbose_name='Endereço')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', verbose_name='Tipo de Content')), + ], + options={ + 'verbose_name_plural': 'Problemas na Migração', + 'verbose_name': 'Problema na Migração', + }, + ), + ] diff --git a/base/models.py b/base/models.py index ad7108bac..f9a63251e 100644 --- a/base/models.py +++ b/base/models.py @@ -1,6 +1,7 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ - +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType from sapl.utils import UF @@ -45,3 +46,19 @@ class CasaLegislativa(models.Model): class Meta: verbose_name = _('Casa Legislativa') verbose_name_plural = _('Casas Legislativas') + + +class ProblemaMigracao(models.Model): + content_type = models.ForeignKey(ContentType, + verbose_name=_('Tipo de Content')) + object_id = models.PositiveIntegerField(verbose_name=_('ID do Objeto')) + content_object = GenericForeignKey('content_type', 'object_id') + problema = models.CharField(max_length=300, null=True, + verbose_name=_('Problema')) + descricao = models.CharField(max_length=300, null=True, + verbose_name=_('Descrição')) + endereco = models.URLField(null=True, verbose_name=_('Endereço')) + + class Meta: + verbose_name = _('Problema na Migração') + verbose_name_plural = _('Problemas na Migração') diff --git a/legacy/migration.py b/legacy/migration.py index f1291ca16..f4a01b93c 100644 --- a/legacy/migration.py +++ b/legacy/migration.py @@ -5,12 +5,17 @@ import yaml from django.apps import apps from django.apps.config import AppConfig from django.core.exceptions import ObjectDoesNotExist +from django.core.urlresolvers import reverse from django.db import connections, models -from django.db.models import CharField, TextField +from django.db.models import CharField, ForeignKey, TextField from django.db.models.base import ModelBase + from model_mommy import mommy +from model_mommy.mommy import make, foreign_key_required +from base.models import ProblemaMigracao from comissoes.models import Composicao, Participacao +from materia.models import TipoMateriaLegislativa from parlamentares.models import Parlamentar from sessao.models import SessaoPlenaria @@ -26,7 +31,6 @@ appconfs = [apps.get_app_config(n) for n in [ 'lexml', 'protocoloadm', ]] -stubs_list = [] unique_constraints = [] name_sets = [set(m.__name__ for m in ac.get_models()) for ac in appconfs] @@ -81,9 +85,9 @@ def get_renames(): return field_renames, model_renames - # MIGRATION ################################################################# + def info(msg): print('INFO: ' + msg) @@ -115,14 +119,17 @@ def get_fk_related(field, value, label=None): not f.choices and not f.blank} value = mommy.make(field.related_model, **fields_dict) - warn(msg + ' => STUB criada para campos não nuláveis!') + descricao = 'stub criado para campos não nuláveis!' + save_relation(value, msg, descricao) + warn(msg + ' => ' + descricao) else: value = None warn(msg + ' => usando None para valores iguais a zero!') else: value = make_stub(field.related_model, value) - stubs_list.append((value.id, field)) - warn(msg + ' => STUB criada!') + descricao = 'stub criado para entrada orfã!' + warn(msg + ' => ' + descricao) + save_relation(value, msg, descricao) else: assert value return value @@ -132,6 +139,11 @@ def get_field(model, fieldname): return model._meta.get_field(fieldname) +def get_url(object): + info = (object._meta.app_label, object._meta.model_name) + return reverse('admin:%s_%s_change' % info, args=(object.pk,)) + + def exec_sql(sql, db='default'): cursor = connections[db].cursor() cursor.execute(sql) @@ -178,6 +190,23 @@ def recreate_constraints(): args_string += "(" + ', '.join(map(str, args)) + ")" exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" % (table, name, args_string)) + unique_constraints.clear() + + +def stub_desnecessario(obj): + lista = [ + f for f in obj._meta.get_fields() + if (f.one_to_many or f.one_to_one) and f.auto_created + ] + desnecessario = not any( + rr.related_model.objects.filter(**{rr.field.name: obj}).exists() + for rr in lista) + if isinstance(obj, TipoMateriaLegislativa): + desnecessario = not any( + rr.related_model.objects.filter( + **{rr.field.name+'_origem_externa': obj}).exists() + for rr in lista) + return desnecessario def save_with_id(new, id): @@ -193,9 +222,16 @@ def save_with_id(new, id): assert new.id == id, 'New id is different from provided!' +def save_relation(obj, problema='', descricao=''): + link = ProblemaMigracao(content_object=obj, problema=problema, + descricao=descricao, endereco=get_url(obj)) + link.save() + + def make_stub(model, id): new = mommy.prepare(model) save_with_id(new, id) + return new @@ -249,6 +285,7 @@ class DataMigrator: # warning: model/app migration order is of utmost importance self.to_delete = [] + ProblemaMigracao.objects.all().delete() info('Começando migração: %s...' % obj) self._do_migrate(obj) # exclude logically deleted in legacy base @@ -256,7 +293,8 @@ class DataMigrator: for obj in self.to_delete: obj.delete() info('Deletando stubs desnecessários...') - self.delete_stubs() + while self.delete_stubs(): + pass info('Recriando unique constraints...') recreate_constraints() @@ -313,12 +351,24 @@ class DataMigrator: self.to_delete.append(new) def delete_stubs(self): - for line in stubs_list: - stub, field = line - # Filter all objects in model and delete from related model - # if quantity is equal to zero - if field.model.objects.filter(**{field.name: stub}).exists(): - field.related_model.objects.get(**{'id': stub}).delete() + excluidos = 0 + for obj in ProblemaMigracao.objects.all().reverse(): + if obj.content_object: + original = obj.content_type.get_all_objects_for_this_type( + id=obj.object_id) + if stub_desnecessario(original[0]): + # Se qtd_exclusoes for maior que 1, está deletando mais + # objetos do que deveria.. + qtd_exclusoes, *_ = original.delete() + assert qtd_exclusoes == 1 + qtd_exclusoes, *_ = obj.delete() + assert qtd_exclusoes == 1 + excluidos = excluidos + 1 + else: + resultado, *_ = obj.delete() + assert resultado == 1 + excluidos = excluidos + 1 + return excluidos def migrate(obj=appconfs): @@ -378,3 +428,23 @@ def check_app_no_ind_excluido(app): for model in app.models.values(): assert not any(get_ind_excluido(obj) for obj in model.objects.all()) print('OK!') + +# MOMMY MAKE WITH LOG ###################################################### + + +def make_with_log(model, _quantity=None, make_m2m=False, **attrs): + all_fields = model._meta.get_fields() + fields_dict = {} + fields_dict = {f.name: '????????????'[:f.max_length] + for f in all_fields + if isinstance(f, (CharField, TextField)) and + not f.choices and not f.blank} + stub = make(model, _quantity, make_m2m, **fields_dict) + problema = 'Um stub foi necessário durante a criação de um outro stub' + descricao = 'Essa entrada é necessária para um dos stubs criados' + ' anteriormente' + warn(problema) + save_relation(obj=stub, problema=problema, descricao=descricao) + return stub + +make_with_log.required = foreign_key_required diff --git a/materia/migrations/0028_auto_20160419_1056.py b/materia/migrations/0028_auto_20160419_1056.py new file mode 100644 index 000000000..bd4eb7f22 --- /dev/null +++ b/materia/migrations/0028_auto_20160419_1056.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9 on 2016-04-19 13:56 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('materia', '0027_auto_20160404_1409'), + ] + + operations = [ + migrations.AlterField( + model_name='tramitacao', + name='unidade_tramitacao_destino', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tramitacoes_destino', to='materia.UnidadeTramitacao', verbose_name='Unidade Destino'), + ), + migrations.AlterField( + model_name='tramitacao', + name='unidade_tramitacao_local', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tramitacoes_origem', to='materia.UnidadeTramitacao', verbose_name='Unidade Local'), + ), + ] diff --git a/materia/models.py b/materia/models.py index d80d751c4..c241f7632 100644 --- a/materia/models.py +++ b/materia/models.py @@ -532,13 +532,13 @@ class Tramitacao(models.Model): data_tramitacao = models.DateField(verbose_name=_('Data Tramitação')) unidade_tramitacao_local = models.ForeignKey( UnidadeTramitacao, - related_name='+', + related_name='tramitacoes_origem', verbose_name=_('Unidade Local')) data_encaminhamento = models.DateField( blank=True, null=True, verbose_name=_('Data Encaminhamento')) unidade_tramitacao_destino = models.ForeignKey( UnidadeTramitacao, - related_name='+', + related_name='tramitacoes_destino', verbose_name=_('Unidade Destino')) urgente = models.BooleanField(verbose_name=_('Urgente ?')) turno = models.CharField( diff --git a/sapl/legacy_migration_settings.py b/sapl/legacy_migration_settings.py index 433cc2329..4c0425e57 100644 --- a/sapl/legacy_migration_settings.py +++ b/sapl/legacy_migration_settings.py @@ -16,3 +16,7 @@ DATABASES['legacy'] = { } DATABASE_ROUTERS = ['legacy.router.LegacyRouter', ] + +MOMMY_CUSTOM_FIELDS_GEN = { + 'django.db.models.ForeignKey': 'legacy.migration.make_with_log' +} From f18f4955917cf113132dbed91c63fc50f516dba0 Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Tue, 19 Apr 2016 15:14:34 -0300 Subject: [PATCH 06/16] Arruma problema de merge no makemigrations Signed-off-by: Luciano Almeida --- materia/migrations/0029_merge.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 materia/migrations/0029_merge.py diff --git a/materia/migrations/0029_merge.py b/materia/migrations/0029_merge.py new file mode 100644 index 000000000..3a37b2421 --- /dev/null +++ b/materia/migrations/0029_merge.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-04-19 18:02 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('materia', '0028_auto_20160419_1056'), + ('materia', '0028_auto_20160419_1000'), + ] + + operations = [ + ] From 3c47701d8895ff6d38127ac49449bac6ba06c7ff Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Tue, 19 Apr 2016 16:04:45 -0300 Subject: [PATCH 07/16] =?UTF-8?q?Cria=20fun=C3=A7=C3=A3o=20para=20pegar=20?= =?UTF-8?q?fields=5Fdict?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Luciano Almeida --- legacy/migration.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/legacy/migration.py b/legacy/migration.py index f4a01b93c..40c73b3f8 100644 --- a/legacy/migration.py +++ b/legacy/migration.py @@ -97,7 +97,7 @@ def warn(msg): def get_fk_related(field, value, label=None): - fields_dict = {} + # fields_dict = {} if value is None and field.null is False: value = 0 if value is not None: @@ -112,11 +112,12 @@ def get_fk_related(field, value, label=None): # se FK == 0, criamos um stub e colocamos o valor '????????' # para qualquer CharField ou TextField que possa haver if not field.null: - all_fields = field.related_model._meta.get_fields() - fields_dict = {f.name: '????????????'[:f.max_length] - for f in all_fields - if isinstance(f, (CharField, TextField)) and - not f.choices and not f.blank} + # all_fields = field.related_model._meta.get_fields() + # fields_dict = {f.name: '????????????'[:f.max_length] + # for f in all_fields + # if isinstance(f, (CharField, TextField)) and + # not f.choices and not f.blank} + fields_dict = get_fields_dict(field.related_model) value = mommy.make(field.related_model, **fields_dict) descricao = 'stub criado para campos não nuláveis!' @@ -229,12 +230,23 @@ def save_relation(obj, problema='', descricao=''): def make_stub(model, id): - new = mommy.prepare(model) + fields_dict = get_fields_dict(model) + new = mommy.prepare(model, **fields_dict) save_with_id(new, id) return new +def get_fields_dict(model): + all_fields = model._meta.get_fields() + fields_dict = {} + fields_dict = {f.name: '????????????'[:f.max_length] + for f in all_fields + if isinstance(f, (CharField, TextField)) and + not f.choices and not f.blank} + return fields_dict + + class DataMigrator: def __init__(self): self.field_renames, self.model_renames = get_renames() @@ -352,7 +364,7 @@ class DataMigrator: def delete_stubs(self): excluidos = 0 - for obj in ProblemaMigracao.objects.all().reverse(): + for obj in ProblemaMigracao.objects.all(): if obj.content_object: original = obj.content_type.get_all_objects_for_this_type( id=obj.object_id) @@ -433,12 +445,9 @@ def check_app_no_ind_excluido(app): def make_with_log(model, _quantity=None, make_m2m=False, **attrs): + import ipdb; ipdb.set_trace() all_fields = model._meta.get_fields() - fields_dict = {} - fields_dict = {f.name: '????????????'[:f.max_length] - for f in all_fields - if isinstance(f, (CharField, TextField)) and - not f.choices and not f.blank} + fields_dict = get_fields_dict(model) stub = make(model, _quantity, make_m2m, **fields_dict) problema = 'Um stub foi necessário durante a criação de um outro stub' descricao = 'Essa entrada é necessária para um dos stubs criados' From ef49081329e2a95267c80bebd7dce8d3ce50229e Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 19 Apr 2016 17:50:45 -0300 Subject: [PATCH 08/16] =?UTF-8?q?Retira=20c=C3=B3digo=20desnecess=C3=A1rio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- legacy/migration.py | 33 ++++++++++----------------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/legacy/migration.py b/legacy/migration.py index 40c73b3f8..1b3d9e062 100644 --- a/legacy/migration.py +++ b/legacy/migration.py @@ -7,11 +7,10 @@ from django.apps.config import AppConfig from django.core.exceptions import ObjectDoesNotExist from django.core.urlresolvers import reverse from django.db import connections, models -from django.db.models import CharField, ForeignKey, TextField +from django.db.models import CharField, TextField from django.db.models.base import ModelBase - from model_mommy import mommy -from model_mommy.mommy import make, foreign_key_required +from model_mommy.mommy import foreign_key_required, make from base.models import ProblemaMigracao from comissoes.models import Composicao, Participacao @@ -97,7 +96,6 @@ def warn(msg): def get_fk_related(field, value, label=None): - # fields_dict = {} if value is None and field.null is False: value = 0 if value is not None: @@ -109,14 +107,7 @@ def get_fk_related(field, value, label=None): field.name, value, field.model.__name__, label or '---') if value == 0: - # se FK == 0, criamos um stub e colocamos o valor '????????' - # para qualquer CharField ou TextField que possa haver if not field.null: - # all_fields = field.related_model._meta.get_fields() - # fields_dict = {f.name: '????????????'[:f.max_length] - # for f in all_fields - # if isinstance(f, (CharField, TextField)) and - # not f.choices and not f.blank} fields_dict = get_fields_dict(field.related_model) value = mommy.make(field.related_model, **fields_dict) @@ -186,7 +177,7 @@ def recreate_constraints(): for i in range(len(args)): if isinstance(model._meta.get_field(args[i]), models.ForeignKey): - args[i] = args[i]+'_id' + args[i] = args[i] + '_id' args_string = '' args_string += "(" + ', '.join(map(str, args)) + ")" exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" % @@ -200,12 +191,12 @@ def stub_desnecessario(obj): if (f.one_to_many or f.one_to_one) and f.auto_created ] desnecessario = not any( - rr.related_model.objects.filter(**{rr.field.name: obj}).exists() - for rr in lista) + rr.related_model.objects.filter(**{rr.field.name: obj}).exists() + for rr in lista) if isinstance(obj, TipoMateriaLegislativa): desnecessario = not any( rr.related_model.objects.filter( - **{rr.field.name+'_origem_externa': obj}).exists() + **{rr.field.name + '_origem_externa': obj}).exists() for rr in lista) return desnecessario @@ -248,6 +239,7 @@ def get_fields_dict(model): class DataMigrator: + def __init__(self): self.field_renames, self.model_renames = get_renames() @@ -286,11 +278,8 @@ class DataMigrator: warn(msg + "=> colocando %s para DateField não nulável" % (value)) - if field_type == 'CharField' or field_type == 'TextField': - if value is None: - warn(msg + "=> colocando string vazia para valor %s" % - (value)) - value = '' + if field_type in ['CharField', 'TextField'] and value is None: + value = '' setattr(new, field.name, value) def migrate(self, obj=appconfs): @@ -367,7 +356,7 @@ class DataMigrator: for obj in ProblemaMigracao.objects.all(): if obj.content_object: original = obj.content_type.get_all_objects_for_this_type( - id=obj.object_id) + id=obj.object_id) if stub_desnecessario(original[0]): # Se qtd_exclusoes for maior que 1, está deletando mais # objetos do que deveria.. @@ -445,8 +434,6 @@ def check_app_no_ind_excluido(app): def make_with_log(model, _quantity=None, make_m2m=False, **attrs): - import ipdb; ipdb.set_trace() - all_fields = model._meta.get_fields() fields_dict = get_fields_dict(model) stub = make(model, _quantity, make_m2m, **fields_dict) problema = 'Um stub foi necessário durante a criação de um outro stub' From 183c7334959eb1d4bd104d1687158505898519ee Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Tue, 19 Apr 2016 14:52:08 -0300 Subject: [PATCH 09/16] =?UTF-8?q?Arruma=20exclus=C3=A3o=20de=20stubs=20des?= =?UTF-8?q?necess=C3=A1rios?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Luciano Almeida --- base/admin.py | 16 ++++ base/migrations/0011_problemamigracao.py | 32 +++++++ base/models.py | 19 +++- legacy/migration.py | 96 ++++++++++++++++--- materia/migrations/0028_auto_20160419_1056.py | 26 +++++ materia/models.py | 4 +- sapl/legacy_migration_settings.py | 4 + 7 files changed, 181 insertions(+), 16 deletions(-) create mode 100644 base/migrations/0011_problemamigracao.py create mode 100644 materia/migrations/0028_auto_20160419_1056.py diff --git a/base/admin.py b/base/admin.py index 5c02ebd40..c97c44834 100644 --- a/base/admin.py +++ b/base/admin.py @@ -1,3 +1,19 @@ from sapl.utils import register_all_models_in_admin +from django.contrib import admin +from base.models import ProblemaMigracao register_all_models_in_admin(__name__) + +admin.site.unregister(ProblemaMigracao) + + +@admin.register(ProblemaMigracao) +class ProblemaMigracaoAdmin(admin.ModelAdmin): + list_display = ["content_type", "object_id", "problema", + "descricao", "get_url"] + + def get_url(self, obj): + return "%s" % (obj.endereco, obj.endereco) + + get_url.short_description = "Endereco" + get_url.allow_tags = True diff --git a/base/migrations/0011_problemamigracao.py b/base/migrations/0011_problemamigracao.py new file mode 100644 index 000000000..aa8eae5ce --- /dev/null +++ b/base/migrations/0011_problemamigracao.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9 on 2016-04-19 16:02 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('contenttypes', '0002_remove_content_type_name'), + ('base', '0010_auto_20160309_1323'), + ] + + operations = [ + migrations.CreateModel( + name='ProblemaMigracao', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('object_id', models.PositiveIntegerField(verbose_name='ID do Objeto')), + ('problema', models.CharField(max_length=300, null=True, verbose_name='Problema')), + ('descricao', models.CharField(max_length=300, null=True, verbose_name='Descrição')), + ('endereco', models.URLField(null=True, verbose_name='Endereço')), + ('content_type', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='contenttypes.ContentType', verbose_name='Tipo de Content')), + ], + options={ + 'verbose_name_plural': 'Problemas na Migração', + 'verbose_name': 'Problema na Migração', + }, + ), + ] diff --git a/base/models.py b/base/models.py index ad7108bac..f9a63251e 100644 --- a/base/models.py +++ b/base/models.py @@ -1,6 +1,7 @@ from django.db import models from django.utils.translation import ugettext_lazy as _ - +from django.contrib.contenttypes.fields import GenericForeignKey +from django.contrib.contenttypes.models import ContentType from sapl.utils import UF @@ -45,3 +46,19 @@ class CasaLegislativa(models.Model): class Meta: verbose_name = _('Casa Legislativa') verbose_name_plural = _('Casas Legislativas') + + +class ProblemaMigracao(models.Model): + content_type = models.ForeignKey(ContentType, + verbose_name=_('Tipo de Content')) + object_id = models.PositiveIntegerField(verbose_name=_('ID do Objeto')) + content_object = GenericForeignKey('content_type', 'object_id') + problema = models.CharField(max_length=300, null=True, + verbose_name=_('Problema')) + descricao = models.CharField(max_length=300, null=True, + verbose_name=_('Descrição')) + endereco = models.URLField(null=True, verbose_name=_('Endereço')) + + class Meta: + verbose_name = _('Problema na Migração') + verbose_name_plural = _('Problemas na Migração') diff --git a/legacy/migration.py b/legacy/migration.py index f1291ca16..f4a01b93c 100644 --- a/legacy/migration.py +++ b/legacy/migration.py @@ -5,12 +5,17 @@ import yaml from django.apps import apps from django.apps.config import AppConfig from django.core.exceptions import ObjectDoesNotExist +from django.core.urlresolvers import reverse from django.db import connections, models -from django.db.models import CharField, TextField +from django.db.models import CharField, ForeignKey, TextField from django.db.models.base import ModelBase + from model_mommy import mommy +from model_mommy.mommy import make, foreign_key_required +from base.models import ProblemaMigracao from comissoes.models import Composicao, Participacao +from materia.models import TipoMateriaLegislativa from parlamentares.models import Parlamentar from sessao.models import SessaoPlenaria @@ -26,7 +31,6 @@ appconfs = [apps.get_app_config(n) for n in [ 'lexml', 'protocoloadm', ]] -stubs_list = [] unique_constraints = [] name_sets = [set(m.__name__ for m in ac.get_models()) for ac in appconfs] @@ -81,9 +85,9 @@ def get_renames(): return field_renames, model_renames - # MIGRATION ################################################################# + def info(msg): print('INFO: ' + msg) @@ -115,14 +119,17 @@ def get_fk_related(field, value, label=None): not f.choices and not f.blank} value = mommy.make(field.related_model, **fields_dict) - warn(msg + ' => STUB criada para campos não nuláveis!') + descricao = 'stub criado para campos não nuláveis!' + save_relation(value, msg, descricao) + warn(msg + ' => ' + descricao) else: value = None warn(msg + ' => usando None para valores iguais a zero!') else: value = make_stub(field.related_model, value) - stubs_list.append((value.id, field)) - warn(msg + ' => STUB criada!') + descricao = 'stub criado para entrada orfã!' + warn(msg + ' => ' + descricao) + save_relation(value, msg, descricao) else: assert value return value @@ -132,6 +139,11 @@ def get_field(model, fieldname): return model._meta.get_field(fieldname) +def get_url(object): + info = (object._meta.app_label, object._meta.model_name) + return reverse('admin:%s_%s_change' % info, args=(object.pk,)) + + def exec_sql(sql, db='default'): cursor = connections[db].cursor() cursor.execute(sql) @@ -178,6 +190,23 @@ def recreate_constraints(): args_string += "(" + ', '.join(map(str, args)) + ")" exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" % (table, name, args_string)) + unique_constraints.clear() + + +def stub_desnecessario(obj): + lista = [ + f for f in obj._meta.get_fields() + if (f.one_to_many or f.one_to_one) and f.auto_created + ] + desnecessario = not any( + rr.related_model.objects.filter(**{rr.field.name: obj}).exists() + for rr in lista) + if isinstance(obj, TipoMateriaLegislativa): + desnecessario = not any( + rr.related_model.objects.filter( + **{rr.field.name+'_origem_externa': obj}).exists() + for rr in lista) + return desnecessario def save_with_id(new, id): @@ -193,9 +222,16 @@ def save_with_id(new, id): assert new.id == id, 'New id is different from provided!' +def save_relation(obj, problema='', descricao=''): + link = ProblemaMigracao(content_object=obj, problema=problema, + descricao=descricao, endereco=get_url(obj)) + link.save() + + def make_stub(model, id): new = mommy.prepare(model) save_with_id(new, id) + return new @@ -249,6 +285,7 @@ class DataMigrator: # warning: model/app migration order is of utmost importance self.to_delete = [] + ProblemaMigracao.objects.all().delete() info('Começando migração: %s...' % obj) self._do_migrate(obj) # exclude logically deleted in legacy base @@ -256,7 +293,8 @@ class DataMigrator: for obj in self.to_delete: obj.delete() info('Deletando stubs desnecessários...') - self.delete_stubs() + while self.delete_stubs(): + pass info('Recriando unique constraints...') recreate_constraints() @@ -313,12 +351,24 @@ class DataMigrator: self.to_delete.append(new) def delete_stubs(self): - for line in stubs_list: - stub, field = line - # Filter all objects in model and delete from related model - # if quantity is equal to zero - if field.model.objects.filter(**{field.name: stub}).exists(): - field.related_model.objects.get(**{'id': stub}).delete() + excluidos = 0 + for obj in ProblemaMigracao.objects.all().reverse(): + if obj.content_object: + original = obj.content_type.get_all_objects_for_this_type( + id=obj.object_id) + if stub_desnecessario(original[0]): + # Se qtd_exclusoes for maior que 1, está deletando mais + # objetos do que deveria.. + qtd_exclusoes, *_ = original.delete() + assert qtd_exclusoes == 1 + qtd_exclusoes, *_ = obj.delete() + assert qtd_exclusoes == 1 + excluidos = excluidos + 1 + else: + resultado, *_ = obj.delete() + assert resultado == 1 + excluidos = excluidos + 1 + return excluidos def migrate(obj=appconfs): @@ -378,3 +428,23 @@ def check_app_no_ind_excluido(app): for model in app.models.values(): assert not any(get_ind_excluido(obj) for obj in model.objects.all()) print('OK!') + +# MOMMY MAKE WITH LOG ###################################################### + + +def make_with_log(model, _quantity=None, make_m2m=False, **attrs): + all_fields = model._meta.get_fields() + fields_dict = {} + fields_dict = {f.name: '????????????'[:f.max_length] + for f in all_fields + if isinstance(f, (CharField, TextField)) and + not f.choices and not f.blank} + stub = make(model, _quantity, make_m2m, **fields_dict) + problema = 'Um stub foi necessário durante a criação de um outro stub' + descricao = 'Essa entrada é necessária para um dos stubs criados' + ' anteriormente' + warn(problema) + save_relation(obj=stub, problema=problema, descricao=descricao) + return stub + +make_with_log.required = foreign_key_required diff --git a/materia/migrations/0028_auto_20160419_1056.py b/materia/migrations/0028_auto_20160419_1056.py new file mode 100644 index 000000000..bd4eb7f22 --- /dev/null +++ b/materia/migrations/0028_auto_20160419_1056.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9 on 2016-04-19 13:56 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('materia', '0027_auto_20160404_1409'), + ] + + operations = [ + migrations.AlterField( + model_name='tramitacao', + name='unidade_tramitacao_destino', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tramitacoes_destino', to='materia.UnidadeTramitacao', verbose_name='Unidade Destino'), + ), + migrations.AlterField( + model_name='tramitacao', + name='unidade_tramitacao_local', + field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='tramitacoes_origem', to='materia.UnidadeTramitacao', verbose_name='Unidade Local'), + ), + ] diff --git a/materia/models.py b/materia/models.py index d80d751c4..c241f7632 100644 --- a/materia/models.py +++ b/materia/models.py @@ -532,13 +532,13 @@ class Tramitacao(models.Model): data_tramitacao = models.DateField(verbose_name=_('Data Tramitação')) unidade_tramitacao_local = models.ForeignKey( UnidadeTramitacao, - related_name='+', + related_name='tramitacoes_origem', verbose_name=_('Unidade Local')) data_encaminhamento = models.DateField( blank=True, null=True, verbose_name=_('Data Encaminhamento')) unidade_tramitacao_destino = models.ForeignKey( UnidadeTramitacao, - related_name='+', + related_name='tramitacoes_destino', verbose_name=_('Unidade Destino')) urgente = models.BooleanField(verbose_name=_('Urgente ?')) turno = models.CharField( diff --git a/sapl/legacy_migration_settings.py b/sapl/legacy_migration_settings.py index 433cc2329..4c0425e57 100644 --- a/sapl/legacy_migration_settings.py +++ b/sapl/legacy_migration_settings.py @@ -16,3 +16,7 @@ DATABASES['legacy'] = { } DATABASE_ROUTERS = ['legacy.router.LegacyRouter', ] + +MOMMY_CUSTOM_FIELDS_GEN = { + 'django.db.models.ForeignKey': 'legacy.migration.make_with_log' +} From d314b36a7728ecb2cbcea20e75f6a211dc7f480c Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Tue, 19 Apr 2016 15:14:34 -0300 Subject: [PATCH 10/16] Arruma problema de merge no makemigrations Signed-off-by: Luciano Almeida --- materia/migrations/0029_merge.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 materia/migrations/0029_merge.py diff --git a/materia/migrations/0029_merge.py b/materia/migrations/0029_merge.py new file mode 100644 index 000000000..3a37b2421 --- /dev/null +++ b/materia/migrations/0029_merge.py @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-04-19 18:02 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('materia', '0028_auto_20160419_1056'), + ('materia', '0028_auto_20160419_1000'), + ] + + operations = [ + ] From 2d90203db5d75016514e3bb9c416e14d5bb3140d Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Tue, 19 Apr 2016 16:04:45 -0300 Subject: [PATCH 11/16] =?UTF-8?q?Cria=20fun=C3=A7=C3=A3o=20para=20pegar=20?= =?UTF-8?q?fields=5Fdict?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Luciano Almeida --- legacy/migration.py | 35 ++++++++++++++++++++++------------- 1 file changed, 22 insertions(+), 13 deletions(-) diff --git a/legacy/migration.py b/legacy/migration.py index f4a01b93c..40c73b3f8 100644 --- a/legacy/migration.py +++ b/legacy/migration.py @@ -97,7 +97,7 @@ def warn(msg): def get_fk_related(field, value, label=None): - fields_dict = {} + # fields_dict = {} if value is None and field.null is False: value = 0 if value is not None: @@ -112,11 +112,12 @@ def get_fk_related(field, value, label=None): # se FK == 0, criamos um stub e colocamos o valor '????????' # para qualquer CharField ou TextField que possa haver if not field.null: - all_fields = field.related_model._meta.get_fields() - fields_dict = {f.name: '????????????'[:f.max_length] - for f in all_fields - if isinstance(f, (CharField, TextField)) and - not f.choices and not f.blank} + # all_fields = field.related_model._meta.get_fields() + # fields_dict = {f.name: '????????????'[:f.max_length] + # for f in all_fields + # if isinstance(f, (CharField, TextField)) and + # not f.choices and not f.blank} + fields_dict = get_fields_dict(field.related_model) value = mommy.make(field.related_model, **fields_dict) descricao = 'stub criado para campos não nuláveis!' @@ -229,12 +230,23 @@ def save_relation(obj, problema='', descricao=''): def make_stub(model, id): - new = mommy.prepare(model) + fields_dict = get_fields_dict(model) + new = mommy.prepare(model, **fields_dict) save_with_id(new, id) return new +def get_fields_dict(model): + all_fields = model._meta.get_fields() + fields_dict = {} + fields_dict = {f.name: '????????????'[:f.max_length] + for f in all_fields + if isinstance(f, (CharField, TextField)) and + not f.choices and not f.blank} + return fields_dict + + class DataMigrator: def __init__(self): self.field_renames, self.model_renames = get_renames() @@ -352,7 +364,7 @@ class DataMigrator: def delete_stubs(self): excluidos = 0 - for obj in ProblemaMigracao.objects.all().reverse(): + for obj in ProblemaMigracao.objects.all(): if obj.content_object: original = obj.content_type.get_all_objects_for_this_type( id=obj.object_id) @@ -433,12 +445,9 @@ def check_app_no_ind_excluido(app): def make_with_log(model, _quantity=None, make_m2m=False, **attrs): + import ipdb; ipdb.set_trace() all_fields = model._meta.get_fields() - fields_dict = {} - fields_dict = {f.name: '????????????'[:f.max_length] - for f in all_fields - if isinstance(f, (CharField, TextField)) and - not f.choices and not f.blank} + fields_dict = get_fields_dict(model) stub = make(model, _quantity, make_m2m, **fields_dict) problema = 'Um stub foi necessário durante a criação de um outro stub' descricao = 'Essa entrada é necessária para um dos stubs criados' From 4140b68a7a6d3e0d9e9b66bfb14e8312a496bf1e Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Thu, 28 Apr 2016 09:59:45 -0300 Subject: [PATCH 12/16] =?UTF-8?q?Adiciona=20datas=20invalidas=20=C3=A0=20m?= =?UTF-8?q?odel=20ProblemaMigracao?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Luciano Almeida --- .../0012_problemamigracao_eh_stub.py | 21 +++++ base/models.py | 1 + legacy/migration.py | 78 ++++++++++--------- ...remove_parlamentar_unidade_deliberativa.py | 19 +++++ 4 files changed, 81 insertions(+), 38 deletions(-) create mode 100644 base/migrations/0012_problemamigracao_eh_stub.py create mode 100644 parlamentares/migrations/0017_remove_parlamentar_unidade_deliberativa.py diff --git a/base/migrations/0012_problemamigracao_eh_stub.py b/base/migrations/0012_problemamigracao_eh_stub.py new file mode 100644 index 000000000..b145c3383 --- /dev/null +++ b/base/migrations/0012_problemamigracao_eh_stub.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-04-26 17:41 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0011_problemamigracao'), + ] + + operations = [ + migrations.AddField( + model_name='problemamigracao', + name='eh_stub', + field=models.BooleanField(default=False, verbose_name='É stub?'), + preserve_default=False, + ), + ] diff --git a/base/models.py b/base/models.py index f9a63251e..9b4ddc80f 100644 --- a/base/models.py +++ b/base/models.py @@ -58,6 +58,7 @@ class ProblemaMigracao(models.Model): descricao = models.CharField(max_length=300, null=True, verbose_name=_('Descrição')) endereco = models.URLField(null=True, verbose_name=_('Endereço')) + eh_stub = models.BooleanField(verbose_name='É stub?') class Meta: verbose_name = _('Problema na Migração') diff --git a/legacy/migration.py b/legacy/migration.py index 40c73b3f8..4ad6c2367 100644 --- a/legacy/migration.py +++ b/legacy/migration.py @@ -97,7 +97,6 @@ def warn(msg): def get_fk_related(field, value, label=None): - # fields_dict = {} if value is None and field.null is False: value = 0 if value is not None: @@ -112,25 +111,19 @@ def get_fk_related(field, value, label=None): # se FK == 0, criamos um stub e colocamos o valor '????????' # para qualquer CharField ou TextField que possa haver if not field.null: - # all_fields = field.related_model._meta.get_fields() - # fields_dict = {f.name: '????????????'[:f.max_length] - # for f in all_fields - # if isinstance(f, (CharField, TextField)) and - # not f.choices and not f.blank} fields_dict = get_fields_dict(field.related_model) value = mommy.make(field.related_model, **fields_dict) descricao = 'stub criado para campos não nuláveis!' - save_relation(value, msg, descricao) + save_relation(value, msg, descricao, eh_stub=True) warn(msg + ' => ' + descricao) else: value = None - warn(msg + ' => usando None para valores iguais a zero!') else: value = make_stub(field.related_model, value) descricao = 'stub criado para entrada orfã!' warn(msg + ' => ' + descricao) - save_relation(value, msg, descricao) + save_relation(value, msg, descricao, eh_stub=True) else: assert value return value @@ -186,7 +179,7 @@ def recreate_constraints(): for i in range(len(args)): if isinstance(model._meta.get_field(args[i]), models.ForeignKey): - args[i] = args[i]+'_id' + args[i] = args[i] + '_id' args_string = '' args_string += "(" + ', '.join(map(str, args)) + ")" exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" % @@ -200,13 +193,13 @@ def stub_desnecessario(obj): if (f.one_to_many or f.one_to_one) and f.auto_created ] desnecessario = not any( - rr.related_model.objects.filter(**{rr.field.name: obj}).exists() - for rr in lista) + rr.related_model.objects.filter(**{rr.field.name: obj}).exists() + for rr in lista if rr) if isinstance(obj, TipoMateriaLegislativa): desnecessario = not any( rr.related_model.objects.filter( - **{rr.field.name+'_origem_externa': obj}).exists() - for rr in lista) + **{rr.field.name + '_origem_externa': obj}).exists() + for rr in lista if rr) return desnecessario @@ -223,9 +216,10 @@ def save_with_id(new, id): assert new.id == id, 'New id is different from provided!' -def save_relation(obj, problema='', descricao=''): +def save_relation(obj, problema='', descricao='', eh_stub=False): link = ProblemaMigracao(content_object=obj, problema=problema, - descricao=descricao, endereco=get_url(obj)) + descricao=descricao, endereco=get_url(obj), + eh_stub=eh_stub) link.save() @@ -250,6 +244,7 @@ def get_fields_dict(model): class DataMigrator: def __init__(self): self.field_renames, self.model_renames = get_renames() + self.data_mudada = {} def populate_renamed_fields(self, new, old): renames = self.field_renames[type(new)] @@ -257,7 +252,7 @@ class DataMigrator: for field in new._meta.fields: old_field_name = renames.get(field.name) field_type = field.get_internal_type() - msg = ("Campo %s (%s) da model %s " % + msg = ("O valor do campo %s (%s) da model %s era inválido" % (field.name, field_type, field.model.__name__)) if old_field_name: old_value = getattr(old, old_field_name) @@ -278,18 +273,24 @@ class DataMigrator: combined_names = "(" + ")|(".join(names) + ")" matches = re.search('(ano_\w+)', combined_names) if not matches: + descricao = 'A data 0001-01-01 foi colocada no lugar' warn(msg + - '=> colocando valor 0000-01-01 para DateField') + '=> ' + descricao) value = '0001-01-01' + self.data_mudada['obj'] = new + self.data_mudada['descricao'] = descricao + self.data_mudada['problema'] = msg else: value = '%d-01-01' % getattr(old, matches.group(0)) + descricao = ('A data %s para foi colocada no lugar' + % value) + self.data_mudada['obj'] = new + self.data_mudada['descricao'] = descricao + self.data_mudada['problema'] = msg warn(msg + - "=> colocando %s para DateField não nulável" % - (value)) + '=> ' + descricao) if field_type == 'CharField' or field_type == 'TextField': if value is None: - warn(msg + "=> colocando string vazia para valor %s" % - (value)) value = '' setattr(new, field.name, value) @@ -359,26 +360,27 @@ class DataMigrator: if adjust: adjust(new, old) save(new, old) + if self.data_mudada: + save_relation(**self.data_mudada) + self.data_mudada.clear() if getattr(old, 'ind_excluido', False): self.to_delete.append(new) def delete_stubs(self): excluidos = 0 for obj in ProblemaMigracao.objects.all(): - if obj.content_object: + if obj.content_object and obj.eh_stub: original = obj.content_type.get_all_objects_for_this_type( - id=obj.object_id) + id=obj.object_id) if stub_desnecessario(original[0]): - # Se qtd_exclusoes for maior que 1, está deletando mais - # objetos do que deveria.. qtd_exclusoes, *_ = original.delete() assert qtd_exclusoes == 1 qtd_exclusoes, *_ = obj.delete() assert qtd_exclusoes == 1 excluidos = excluidos + 1 - else: - resultado, *_ = obj.delete() - assert resultado == 1 + elif not obj.content_object and not obj.eh_stub: + qtd_exclusoes, *_ = obj.delete() + assert qtd_exclusoes == 1 excluidos = excluidos + 1 return excluidos @@ -408,13 +410,14 @@ def adjust_participacao(new_participacao, old): def adjust_parlamentar(new_parlamentar, old): - value = new_parlamentar.unidade_deliberativa - # Field is defined as not null in legacy db, - # but data includes null values - # => transform None to False - if value is None: - warn('nulo convertido para falso') - new_parlamentar.unidade_deliberativa = False + if old.ind_unid_deliberativa: + value = new_parlamentar.unidade_deliberativa + # Field is defined as not null in legacy db, + # but data includes null values + # => transform None to False + if value is None: + warn('nulo convertido para falso') + new_parlamentar.unidade_deliberativa = False def adjust_sessaoplenaria(new, old): @@ -445,7 +448,6 @@ def check_app_no_ind_excluido(app): def make_with_log(model, _quantity=None, make_m2m=False, **attrs): - import ipdb; ipdb.set_trace() all_fields = model._meta.get_fields() fields_dict = get_fields_dict(model) stub = make(model, _quantity, make_m2m, **fields_dict) @@ -453,7 +455,7 @@ def make_with_log(model, _quantity=None, make_m2m=False, **attrs): descricao = 'Essa entrada é necessária para um dos stubs criados' ' anteriormente' warn(problema) - save_relation(obj=stub, problema=problema, descricao=descricao) + save_relation(stub, problema, descricao, eh_stub=True) return stub make_with_log.required = foreign_key_required diff --git a/parlamentares/migrations/0017_remove_parlamentar_unidade_deliberativa.py b/parlamentares/migrations/0017_remove_parlamentar_unidade_deliberativa.py new file mode 100644 index 000000000..111ad79ee --- /dev/null +++ b/parlamentares/migrations/0017_remove_parlamentar_unidade_deliberativa.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-04-25 16:29 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('parlamentares', '0016_auto_20160404_1409'), + ] + + operations = [ + migrations.RemoveField( + model_name='parlamentar', + name='unidade_deliberativa', + ), + ] From 8edb5ed0d75d4292cfb69a0b195bf486d6bfe154 Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Thu, 28 Apr 2016 10:31:20 -0300 Subject: [PATCH 13/16] Arruma imports Signed-off-by: Luciano Almeida --- base/admin.py | 3 ++- base/models.py | 5 +++-- legacy/migration.py | 6 ++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/base/admin.py b/base/admin.py index c97c44834..e9caf1ca6 100644 --- a/base/admin.py +++ b/base/admin.py @@ -1,6 +1,7 @@ -from sapl.utils import register_all_models_in_admin from django.contrib import admin + from base.models import ProblemaMigracao +from sapl.utils import register_all_models_in_admin register_all_models_in_admin(__name__) diff --git a/base/models.py b/base/models.py index 9b4ddc80f..cff2a32f2 100644 --- a/base/models.py +++ b/base/models.py @@ -1,7 +1,8 @@ -from django.db import models -from django.utils.translation import ugettext_lazy as _ from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType +from django.db import models +from django.utils.translation import ugettext_lazy as _ + from sapl.utils import UF diff --git a/legacy/migration.py b/legacy/migration.py index 4ad6c2367..c4f4103d6 100644 --- a/legacy/migration.py +++ b/legacy/migration.py @@ -7,11 +7,10 @@ from django.apps.config import AppConfig from django.core.exceptions import ObjectDoesNotExist from django.core.urlresolvers import reverse from django.db import connections, models -from django.db.models import CharField, ForeignKey, TextField +from django.db.models import CharField, TextField from django.db.models.base import ModelBase - from model_mommy import mommy -from model_mommy.mommy import make, foreign_key_required +from model_mommy.mommy import foreign_key_required, make from base.models import ProblemaMigracao from comissoes.models import Composicao, Participacao @@ -448,7 +447,6 @@ def check_app_no_ind_excluido(app): def make_with_log(model, _quantity=None, make_m2m=False, **attrs): - all_fields = model._meta.get_fields() fields_dict = get_fields_dict(model) stub = make(model, _quantity, make_m2m, **fields_dict) problema = 'Um stub foi necessário durante a criação de um outro stub' From 9af25222de77c19a1e307a23dad51f6e194e0bcf Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Mon, 2 May 2016 15:16:15 -0300 Subject: [PATCH 14/16] =?UTF-8?q?Ajusta=20sugest=C3=B5es=20de=20corre?= =?UTF-8?q?=C3=A7=C3=A3o=20do=20PR?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Luciano Almeida --- base/admin.py | 9 +++++---- .../0013_remove_problemamigracao_endereco.py | 19 +++++++++++++++++++ base/models.py | 3 --- legacy/migration.py | 19 ++++--------------- materia/models.py | 2 +- 5 files changed, 29 insertions(+), 23 deletions(-) create mode 100644 base/migrations/0013_remove_problemamigracao_endereco.py diff --git a/base/admin.py b/base/admin.py index 0d8e82faf..37e400946 100644 --- a/base/admin.py +++ b/base/admin.py @@ -2,8 +2,7 @@ from django.contrib import admin from base.models import ProblemaMigracao from sapl.utils import register_all_models_in_admin -from django.contrib import admin -from base.models import ProblemaMigracao +from django.core.urlresolvers import reverse register_all_models_in_admin(__name__) @@ -16,7 +15,9 @@ class ProblemaMigracaoAdmin(admin.ModelAdmin): "descricao", "get_url"] def get_url(self, obj): - return "%s" % (obj.endereco, obj.endereco) + info = (obj._meta.app_label, obj._meta.model_name) + endereco = reverse('admin:%s_%s_change' % info, args=(obj.pk,)) + return "%s" % (endereco, endereco) - get_url.short_description = "Endereco" + get_url.short_description = "Endereço" get_url.allow_tags = True diff --git a/base/migrations/0013_remove_problemamigracao_endereco.py b/base/migrations/0013_remove_problemamigracao_endereco.py new file mode 100644 index 000000000..67526693d --- /dev/null +++ b/base/migrations/0013_remove_problemamigracao_endereco.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-05-02 17:36 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0012_problemamigracao_eh_stub'), + ] + + operations = [ + migrations.RemoveField( + model_name='problemamigracao', + name='endereco', + ), + ] diff --git a/base/models.py b/base/models.py index 8af741c0d..9a94c9eb8 100644 --- a/base/models.py +++ b/base/models.py @@ -2,8 +2,6 @@ from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.db import models from django.utils.translation import ugettext_lazy as _ -from django.contrib.contenttypes.fields import GenericForeignKey -from django.contrib.contenttypes.models import ContentType from sapl.utils import UF @@ -59,7 +57,6 @@ class ProblemaMigracao(models.Model): verbose_name=_('Problema')) descricao = models.CharField(max_length=300, null=True, verbose_name=_('Descrição')) - endereco = models.URLField(null=True, verbose_name=_('Endereço')) eh_stub = models.BooleanField(verbose_name='É stub?') class Meta: diff --git a/legacy/migration.py b/legacy/migration.py index f1210d363..d69da781c 100644 --- a/legacy/migration.py +++ b/legacy/migration.py @@ -130,11 +130,6 @@ def get_field(model, fieldname): return model._meta.get_field(fieldname) -def get_url(object): - info = (object._meta.app_label, object._meta.model_name) - return reverse('admin:%s_%s_change' % info, args=(object.pk,)) - - def exec_sql(sql, db='default'): cursor = connections[db].cursor() cursor.execute(sql) @@ -185,18 +180,13 @@ def recreate_constraints(): def stub_desnecessario(obj): - lista = [ + lista_fields = [ f for f in obj._meta.get_fields() if (f.one_to_many or f.one_to_one) and f.auto_created ] desnecessario = not any( rr.related_model.objects.filter(**{rr.field.name: obj}).exists() - for rr in lista if rr) - if isinstance(obj, TipoMateriaLegislativa): - desnecessario = not any( - rr.related_model.objects.filter( - **{rr.field.name + '_origem_externa': obj}).exists() - for rr in lista if rr) + for rr in lista_fields) return desnecessario @@ -215,8 +205,7 @@ def save_with_id(new, id): def save_relation(obj, problema='', descricao='', eh_stub=False): link = ProblemaMigracao(content_object=obj, problema=problema, - descricao=descricao, endereco=get_url(obj), - eh_stub=eh_stub) + descricao=descricao, eh_stub=eh_stub) link.save() @@ -272,7 +261,7 @@ class DataMigrator: if not matches: descricao = 'A data 0001-01-01 foi colocada no lugar' warn(msg + - '=> ' + descricao) + ' => ' + descricao) value = '0001-01-01' self.data_mudada['obj'] = new self.data_mudada['descricao'] = descricao diff --git a/materia/models.py b/materia/models.py index c241f7632..08d5165f6 100644 --- a/materia/models.py +++ b/materia/models.py @@ -80,7 +80,7 @@ class MateriaLegislativa(models.Model): TipoMateriaLegislativa, blank=True, null=True, - related_name='+', + related_name='tipo_origem_externa_set', verbose_name=_('Tipo')) numero_origem_externa = models.CharField( max_length=5, blank=True, verbose_name=_('Número')) From 80ea177f2316502cd53908818685a4c173e829cf Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Mon, 2 May 2016 15:57:26 -0300 Subject: [PATCH 15/16] Exclui arquivo de migrate pra resolver conflito Signed-off-by: Luciano Almeida --- ...remove_parlamentar_unidade_deliberativa.py | 19 ------------------- 1 file changed, 19 deletions(-) delete mode 100644 parlamentares/migrations/0017_remove_parlamentar_unidade_deliberativa.py diff --git a/parlamentares/migrations/0017_remove_parlamentar_unidade_deliberativa.py b/parlamentares/migrations/0017_remove_parlamentar_unidade_deliberativa.py deleted file mode 100644 index 111ad79ee..000000000 --- a/parlamentares/migrations/0017_remove_parlamentar_unidade_deliberativa.py +++ /dev/null @@ -1,19 +0,0 @@ -# -*- coding: utf-8 -*- -# Generated by Django 1.9.5 on 2016-04-25 16:29 -from __future__ import unicode_literals - -from django.db import migrations - - -class Migration(migrations.Migration): - - dependencies = [ - ('parlamentares', '0016_auto_20160404_1409'), - ] - - operations = [ - migrations.RemoveField( - model_name='parlamentar', - name='unidade_deliberativa', - ), - ] From 7c97c32469ba8477108f7614cea0d60fdee45f5e Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Mon, 2 May 2016 16:40:43 -0300 Subject: [PATCH 16/16] Concerta alguns problemas de model e admin Signed-off-by: Luciano Almeida --- base/admin.py | 9 ++++--- base/migrations/0014_auto_20160502_1635.py | 27 +++++++++++++++++++ base/models.py | 7 +++-- legacy/migration.py | 2 -- materia/migrations/0030_auto_20160502_1630.py | 21 +++++++++++++++ 5 files changed, 57 insertions(+), 9 deletions(-) create mode 100644 base/migrations/0014_auto_20160502_1635.py create mode 100644 materia/migrations/0030_auto_20160502_1630.py diff --git a/base/admin.py b/base/admin.py index 37e400946..99f81f8af 100644 --- a/base/admin.py +++ b/base/admin.py @@ -1,8 +1,8 @@ from django.contrib import admin +from django.core.urlresolvers import reverse from base.models import ProblemaMigracao from sapl.utils import register_all_models_in_admin -from django.core.urlresolvers import reverse register_all_models_in_admin(__name__) @@ -15,8 +15,11 @@ class ProblemaMigracaoAdmin(admin.ModelAdmin): "descricao", "get_url"] def get_url(self, obj): - info = (obj._meta.app_label, obj._meta.model_name) - endereco = reverse('admin:%s_%s_change' % info, args=(obj.pk,)) + + info = (obj.content_object._meta.app_label, + obj.content_object._meta.model_name) + endereco = reverse('admin:%s_%s_change' % info, + args=(obj.content_object.pk,)) return "%s" % (endereco, endereco) get_url.short_description = "Endereço" diff --git a/base/migrations/0014_auto_20160502_1635.py b/base/migrations/0014_auto_20160502_1635.py new file mode 100644 index 000000000..6fbae0bb9 --- /dev/null +++ b/base/migrations/0014_auto_20160502_1635.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-05-02 19:35 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0013_remove_problemamigracao_endereco'), + ] + + operations = [ + migrations.AlterField( + model_name='problemamigracao', + name='descricao', + field=models.CharField(default='', max_length=300, verbose_name='Descrição'), + preserve_default=False, + ), + migrations.AlterField( + model_name='problemamigracao', + name='problema', + field=models.CharField(default='', max_length=300, verbose_name='Problema'), + preserve_default=False, + ), + ] diff --git a/base/models.py b/base/models.py index 9a94c9eb8..f1fb3811f 100644 --- a/base/models.py +++ b/base/models.py @@ -2,6 +2,7 @@ from django.contrib.contenttypes.fields import GenericForeignKey from django.contrib.contenttypes.models import ContentType from django.db import models from django.utils.translation import ugettext_lazy as _ + from sapl.utils import UF @@ -53,10 +54,8 @@ class ProblemaMigracao(models.Model): verbose_name=_('Tipo de Content')) object_id = models.PositiveIntegerField(verbose_name=_('ID do Objeto')) content_object = GenericForeignKey('content_type', 'object_id') - problema = models.CharField(max_length=300, null=True, - verbose_name=_('Problema')) - descricao = models.CharField(max_length=300, null=True, - verbose_name=_('Descrição')) + problema = models.CharField(max_length=300, verbose_name=_('Problema')) + descricao = models.CharField(max_length=300, verbose_name=_('Descrição')) eh_stub = models.BooleanField(verbose_name='É stub?') class Meta: diff --git a/legacy/migration.py b/legacy/migration.py index d69da781c..ddc8e0bfd 100644 --- a/legacy/migration.py +++ b/legacy/migration.py @@ -5,7 +5,6 @@ import yaml from django.apps import apps from django.apps.config import AppConfig from django.core.exceptions import ObjectDoesNotExist -from django.core.urlresolvers import reverse from django.db import connections, models from django.db.models import CharField, TextField from django.db.models.base import ModelBase @@ -14,7 +13,6 @@ from model_mommy.mommy import foreign_key_required, make from base.models import ProblemaMigracao from comissoes.models import Composicao, Participacao -from materia.models import TipoMateriaLegislativa from parlamentares.models import Parlamentar from sessao.models import SessaoPlenaria diff --git a/materia/migrations/0030_auto_20160502_1630.py b/materia/migrations/0030_auto_20160502_1630.py new file mode 100644 index 000000000..6f8fd60be --- /dev/null +++ b/materia/migrations/0030_auto_20160502_1630.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-05-02 19:30 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('materia', '0029_merge'), + ] + + operations = [ + migrations.AlterField( + model_name='materialegislativa', + name='tipo_origem_externa', + field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='tipo_origem_externa_set', to='materia.TipoMateriaLegislativa', verbose_name='Tipo'), + ), + ]