From 4140b68a7a6d3e0d9e9b66bfb14e8312a496bf1e Mon Sep 17 00:00:00 2001 From: Luciano Almeida Date: Thu, 28 Apr 2016 09:59:45 -0300 Subject: [PATCH] =?UTF-8?q?Adiciona=20datas=20invalidas=20=C3=A0=20model?= =?UTF-8?q?=20ProblemaMigracao?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Luciano Almeida --- .../0012_problemamigracao_eh_stub.py | 21 +++++ base/models.py | 1 + legacy/migration.py | 78 ++++++++++--------- ...remove_parlamentar_unidade_deliberativa.py | 19 +++++ 4 files changed, 81 insertions(+), 38 deletions(-) create mode 100644 base/migrations/0012_problemamigracao_eh_stub.py create mode 100644 parlamentares/migrations/0017_remove_parlamentar_unidade_deliberativa.py diff --git a/base/migrations/0012_problemamigracao_eh_stub.py b/base/migrations/0012_problemamigracao_eh_stub.py new file mode 100644 index 000000000..b145c3383 --- /dev/null +++ b/base/migrations/0012_problemamigracao_eh_stub.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-04-26 17:41 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('base', '0011_problemamigracao'), + ] + + operations = [ + migrations.AddField( + model_name='problemamigracao', + name='eh_stub', + field=models.BooleanField(default=False, verbose_name='É stub?'), + preserve_default=False, + ), + ] diff --git a/base/models.py b/base/models.py index f9a63251e..9b4ddc80f 100644 --- a/base/models.py +++ b/base/models.py @@ -58,6 +58,7 @@ class ProblemaMigracao(models.Model): descricao = models.CharField(max_length=300, null=True, verbose_name=_('Descrição')) endereco = models.URLField(null=True, verbose_name=_('Endereço')) + eh_stub = models.BooleanField(verbose_name='É stub?') class Meta: verbose_name = _('Problema na Migração') diff --git a/legacy/migration.py b/legacy/migration.py index 40c73b3f8..4ad6c2367 100644 --- a/legacy/migration.py +++ b/legacy/migration.py @@ -97,7 +97,6 @@ def warn(msg): def get_fk_related(field, value, label=None): - # fields_dict = {} if value is None and field.null is False: value = 0 if value is not None: @@ -112,25 +111,19 @@ def get_fk_related(field, value, label=None): # se FK == 0, criamos um stub e colocamos o valor '????????' # para qualquer CharField ou TextField que possa haver if not field.null: - # all_fields = field.related_model._meta.get_fields() - # fields_dict = {f.name: '????????????'[:f.max_length] - # for f in all_fields - # if isinstance(f, (CharField, TextField)) and - # not f.choices and not f.blank} fields_dict = get_fields_dict(field.related_model) value = mommy.make(field.related_model, **fields_dict) descricao = 'stub criado para campos não nuláveis!' - save_relation(value, msg, descricao) + save_relation(value, msg, descricao, eh_stub=True) warn(msg + ' => ' + descricao) else: value = None - warn(msg + ' => usando None para valores iguais a zero!') else: value = make_stub(field.related_model, value) descricao = 'stub criado para entrada orfã!' warn(msg + ' => ' + descricao) - save_relation(value, msg, descricao) + save_relation(value, msg, descricao, eh_stub=True) else: assert value return value @@ -186,7 +179,7 @@ def recreate_constraints(): for i in range(len(args)): if isinstance(model._meta.get_field(args[i]), models.ForeignKey): - args[i] = args[i]+'_id' + args[i] = args[i] + '_id' args_string = '' args_string += "(" + ', '.join(map(str, args)) + ")" exec_sql("ALTER TABLE %s ADD CONSTRAINT %s UNIQUE %s;" % @@ -200,13 +193,13 @@ def stub_desnecessario(obj): if (f.one_to_many or f.one_to_one) and f.auto_created ] desnecessario = not any( - rr.related_model.objects.filter(**{rr.field.name: obj}).exists() - for rr in lista) + rr.related_model.objects.filter(**{rr.field.name: obj}).exists() + for rr in lista if rr) if isinstance(obj, TipoMateriaLegislativa): desnecessario = not any( rr.related_model.objects.filter( - **{rr.field.name+'_origem_externa': obj}).exists() - for rr in lista) + **{rr.field.name + '_origem_externa': obj}).exists() + for rr in lista if rr) return desnecessario @@ -223,9 +216,10 @@ def save_with_id(new, id): assert new.id == id, 'New id is different from provided!' -def save_relation(obj, problema='', descricao=''): +def save_relation(obj, problema='', descricao='', eh_stub=False): link = ProblemaMigracao(content_object=obj, problema=problema, - descricao=descricao, endereco=get_url(obj)) + descricao=descricao, endereco=get_url(obj), + eh_stub=eh_stub) link.save() @@ -250,6 +244,7 @@ def get_fields_dict(model): class DataMigrator: def __init__(self): self.field_renames, self.model_renames = get_renames() + self.data_mudada = {} def populate_renamed_fields(self, new, old): renames = self.field_renames[type(new)] @@ -257,7 +252,7 @@ class DataMigrator: for field in new._meta.fields: old_field_name = renames.get(field.name) field_type = field.get_internal_type() - msg = ("Campo %s (%s) da model %s " % + msg = ("O valor do campo %s (%s) da model %s era inválido" % (field.name, field_type, field.model.__name__)) if old_field_name: old_value = getattr(old, old_field_name) @@ -278,18 +273,24 @@ class DataMigrator: combined_names = "(" + ")|(".join(names) + ")" matches = re.search('(ano_\w+)', combined_names) if not matches: + descricao = 'A data 0001-01-01 foi colocada no lugar' warn(msg + - '=> colocando valor 0000-01-01 para DateField') + '=> ' + descricao) value = '0001-01-01' + self.data_mudada['obj'] = new + self.data_mudada['descricao'] = descricao + self.data_mudada['problema'] = msg else: value = '%d-01-01' % getattr(old, matches.group(0)) + descricao = ('A data %s para foi colocada no lugar' + % value) + self.data_mudada['obj'] = new + self.data_mudada['descricao'] = descricao + self.data_mudada['problema'] = msg warn(msg + - "=> colocando %s para DateField não nulável" % - (value)) + '=> ' + descricao) if field_type == 'CharField' or field_type == 'TextField': if value is None: - warn(msg + "=> colocando string vazia para valor %s" % - (value)) value = '' setattr(new, field.name, value) @@ -359,26 +360,27 @@ class DataMigrator: if adjust: adjust(new, old) save(new, old) + if self.data_mudada: + save_relation(**self.data_mudada) + self.data_mudada.clear() if getattr(old, 'ind_excluido', False): self.to_delete.append(new) def delete_stubs(self): excluidos = 0 for obj in ProblemaMigracao.objects.all(): - if obj.content_object: + if obj.content_object and obj.eh_stub: original = obj.content_type.get_all_objects_for_this_type( - id=obj.object_id) + id=obj.object_id) if stub_desnecessario(original[0]): - # Se qtd_exclusoes for maior que 1, está deletando mais - # objetos do que deveria.. qtd_exclusoes, *_ = original.delete() assert qtd_exclusoes == 1 qtd_exclusoes, *_ = obj.delete() assert qtd_exclusoes == 1 excluidos = excluidos + 1 - else: - resultado, *_ = obj.delete() - assert resultado == 1 + elif not obj.content_object and not obj.eh_stub: + qtd_exclusoes, *_ = obj.delete() + assert qtd_exclusoes == 1 excluidos = excluidos + 1 return excluidos @@ -408,13 +410,14 @@ def adjust_participacao(new_participacao, old): def adjust_parlamentar(new_parlamentar, old): - value = new_parlamentar.unidade_deliberativa - # Field is defined as not null in legacy db, - # but data includes null values - # => transform None to False - if value is None: - warn('nulo convertido para falso') - new_parlamentar.unidade_deliberativa = False + if old.ind_unid_deliberativa: + value = new_parlamentar.unidade_deliberativa + # Field is defined as not null in legacy db, + # but data includes null values + # => transform None to False + if value is None: + warn('nulo convertido para falso') + new_parlamentar.unidade_deliberativa = False def adjust_sessaoplenaria(new, old): @@ -445,7 +448,6 @@ def check_app_no_ind_excluido(app): def make_with_log(model, _quantity=None, make_m2m=False, **attrs): - import ipdb; ipdb.set_trace() all_fields = model._meta.get_fields() fields_dict = get_fields_dict(model) stub = make(model, _quantity, make_m2m, **fields_dict) @@ -453,7 +455,7 @@ def make_with_log(model, _quantity=None, make_m2m=False, **attrs): descricao = 'Essa entrada é necessária para um dos stubs criados' ' anteriormente' warn(problema) - save_relation(obj=stub, problema=problema, descricao=descricao) + save_relation(stub, problema, descricao, eh_stub=True) return stub make_with_log.required = foreign_key_required diff --git a/parlamentares/migrations/0017_remove_parlamentar_unidade_deliberativa.py b/parlamentares/migrations/0017_remove_parlamentar_unidade_deliberativa.py new file mode 100644 index 000000000..111ad79ee --- /dev/null +++ b/parlamentares/migrations/0017_remove_parlamentar_unidade_deliberativa.py @@ -0,0 +1,19 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.5 on 2016-04-25 16:29 +from __future__ import unicode_literals + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('parlamentares', '0016_auto_20160404_1409'), + ] + + operations = [ + migrations.RemoveField( + model_name='parlamentar', + name='unidade_deliberativa', + ), + ]