From dd1351d70f752ab0231511913e80828400e061ed Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 14 Mar 2017 14:17:58 -0300 Subject: [PATCH 1/3] =?UTF-8?q?Adapta=20migra=C3=A7=C3=A3o=20para=20rodar?= =?UTF-8?q?=20em=20paralelo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../management/commands/migracao_25_31.py | 14 ++++++- sapl/legacy/migration.py | 42 ++++++++++--------- sapl/legacy/scripts/migra_dbs.sh | 5 +++ sapl/legacy/scripts/migra_um_db.sh | 25 +++++++++++ sapl/legacy/scripts/recria_dbs_postgres.sh | 5 +++ sapl/legacy/scripts/recria_um_db_postgres.sh | 5 +++ sapl/legacy_migration_settings.py | 12 ++++-- 7 files changed, 84 insertions(+), 24 deletions(-) create mode 100755 sapl/legacy/scripts/migra_dbs.sh create mode 100755 sapl/legacy/scripts/migra_um_db.sh create mode 100755 sapl/legacy/scripts/recria_dbs_postgres.sh create mode 100755 sapl/legacy/scripts/recria_um_db_postgres.sh diff --git a/sapl/legacy/management/commands/migracao_25_31.py b/sapl/legacy/management/commands/migracao_25_31.py index f01b8117a..5334ac7c0 100644 --- a/sapl/legacy/management/commands/migracao_25_31.py +++ b/sapl/legacy/management/commands/migracao_25_31.py @@ -4,7 +4,17 @@ from sapl.legacy import migration class Command(BaseCommand): - help = u'Faz a migração de dados do SAPL 2.5 para o SAPL 3.1' + + help = u'Migração de dados do SAPL 2.5 para o SAPL 3.1' + + def add_arguments(self, parser): + parser.add_argument( + '-f', + action='store_true', + default=False, + dest='force', + help='Não interativa: pula confirmação de exclusão dos dados', + ) def handle(self, *args, **options): - migration.migrate() + migration.migrate(interativo=not options['force']) diff --git a/sapl/legacy/migration.py b/sapl/legacy/migration.py index f4b6f00a6..c7bad24d9 100644 --- a/sapl/legacy/migration.py +++ b/sapl/legacy/migration.py @@ -11,11 +11,10 @@ from django.contrib.auth import get_user_model from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist from django.db import OperationalError, ProgrammingError, connections, models -from django.db.models import CharField, Max, ProtectedError, TextField, signals +from django.db.models import CharField, Max, ProtectedError, TextField from django.db.models.base import ModelBase from model_mommy import mommy from model_mommy.mommy import foreign_key_required, make -from reversion.models import Revision, Version from sapl.base.models import Autor, ProblemaMigracao from sapl.comissoes.models import Comissao, Composicao, Participacao @@ -28,6 +27,7 @@ from sapl.norma.models import (AssuntoNorma, NormaJuridica, from sapl.parlamentares.models import Parlamentar from sapl.protocoloadm.models import Protocolo, StatusTramitacaoAdministrativo from sapl.sessao.models import ExpedienteMateria, OrdemDia, SessaoPlenaria +from sapl.settings import PROJECT_DIR from sapl.utils import normalize # BASE ###################################################################### @@ -316,9 +316,9 @@ def fill_vinculo_norma_juridica(): ('L', 'Suspende integralmente a norma'), ('N', 'Julgada integralmente inconstitucional'), ('O', 'Julgada parcialmente inconstitucional')] - lista_objs = [VinculoNormaJuridica(sigla=item[0], descricao=item[1]) + lista_objs = [TipoVinculoNormaJuridica(sigla=item[0], descricao=item[1]) for item in lista] - VinculoNormaJuridica.objects.bulk_create(lista_objs) + TipoVinculoNormaJuridica.objects.bulk_create(lista_objs) class DataMigrator: @@ -394,24 +394,28 @@ class DataMigrator: value = None setattr(new, field.name, value) - def migrate(self, obj=appconfs): + def migrate(self, obj=appconfs, interativo=True): # warning: model/app migration order is of utmost importance - exec_sql_file('sapl/legacy/scripts/fix_tables.sql', 'legacy') + exec_sql_file(PROJECT_DIR.child( + 'sapl', 'legacy', 'scripts', 'fix_tables.sql'), 'legacy') self.to_delete = [] # excluindo database antigo. - info('Todos os dados do banco serão excluidos. ' - 'Recomendamos que faça backup do banco sapl antes de continuar.') - info('Deseja continuar? [s/n]') - resposta = input() - if resposta.lower() in ['s', 'sim', 'y', 'yes']: - pass - else: - info('Migração cancelada.') - return 0 + if interativo: + info('Todos os dados do banco serão excluidos. ' + 'Recomendamos que faça backup do banco sapl ' + 'antes de continuar.') + info('Deseja continuar? [s/n]') + resposta = input() + if resposta.lower() in ['s', 'sim', 'y', 'yes']: + pass + else: + info('Migração cancelada.') + return 0 info('Excluindo entradas antigas do banco.') - call(['./manage.py', 'flush', '--settings=sapl.settings', - '--database=default', '--no-input'], stdout=PIPE) + call([PROJECT_DIR.child('manage.py'), 'flush', + '--settings=sapl.settings', '--database=default', '--no-input'], + stdout=PIPE) info('Começando migração: %s...' % obj) self._do_migrate(obj) @@ -528,9 +532,9 @@ class DataMigrator: return excluidos -def migrate(obj=appconfs): +def migrate(obj=appconfs, interativo=True): dm = DataMigrator() - dm.migrate(obj) + dm.migrate(obj, interativo) # MIGRATION_ADJUSTMENTS ##################################################### diff --git a/sapl/legacy/scripts/migra_dbs.sh b/sapl/legacy/scripts/migra_dbs.sh new file mode 100755 index 000000000..c7529e015 --- /dev/null +++ b/sapl/legacy/scripts/migra_dbs.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +# rodar esse script na raiz do projeto + +parallel --verbose ./sapl/legacy/scripts/migra_um_db.sh :::: <(mysql -u root -padmin -e 'show databases;' | grep '^sapl_') diff --git a/sapl/legacy/scripts/migra_um_db.sh b/sapl/legacy/scripts/migra_um_db.sh new file mode 100755 index 000000000..26240d8d4 --- /dev/null +++ b/sapl/legacy/scripts/migra_um_db.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# rodar esse script na raiz do projeto + +DIR=~/logs_migracao +mkdir -p $DIR + +LOG="$DIR/$1.migracao.log" +rm -f $LOG + +echo "########################################" | tee -a $LOG +echo "MIGRANDO BANCO $1" | tee -a $LOG +echo "########################################" | tee -a $LOG +echo >> $LOG + + +echo "--- DJANGO MIGRATE ---" | tee -a $LOG +echo >> $LOG +DATABASE_NAME=$1 ./manage.py migrate --settings sapl.legacy_migration_settings +echo >> $LOG + +echo "--- MIGRACAO DE DADOS ---" | tee -a $LOG +echo >> $LOG +DATABASE_NAME=$1 ./manage.py migracao_25_31 -f --settings sapl.legacy_migration_settings |& tee -a $LOG +echo >> $LOG diff --git a/sapl/legacy/scripts/recria_dbs_postgres.sh b/sapl/legacy/scripts/recria_dbs_postgres.sh new file mode 100755 index 000000000..6fbf18794 --- /dev/null +++ b/sapl/legacy/scripts/recria_dbs_postgres.sh @@ -0,0 +1,5 @@ +# (Re)cria todos os bancos postgres para migração +# cria um banco postgres (de mesmo nome) para cada banco mysql cujo nome começa com "sapl_" + +mysql -u root -padmin -e 'show databases;' | grep '^sapl_' | xargs -I{} ./recria_um_db_postgres.sh {} + diff --git a/sapl/legacy/scripts/recria_um_db_postgres.sh b/sapl/legacy/scripts/recria_um_db_postgres.sh new file mode 100755 index 000000000..fd56a95ee --- /dev/null +++ b/sapl/legacy/scripts/recria_um_db_postgres.sh @@ -0,0 +1,5 @@ +# (Re)cria um db postgres +# uso: recria_um_db_postgres + +sudo -u postgres psql -c "drop DATABASE if exists $1" +sudo -u postgres psql -c "CREATE DATABASE $1 WITH OWNER = sapl ENCODING = 'UTF8' TABLESPACE = pg_default LC_COLLATE = 'pt_BR.UTF-8' LC_CTYPE = 'pt_BR.UTF-8' CONNECTION LIMIT = -1 TEMPLATE template0;" diff --git a/sapl/legacy_migration_settings.py b/sapl/legacy_migration_settings.py index d46c4b027..2b7366fc2 100644 --- a/sapl/legacy_migration_settings.py +++ b/sapl/legacy_migration_settings.py @@ -1,12 +1,11 @@ import os -from decouple import AutoConfig, Config, RepositoryEnv +from decouple import Config, RepositoryEnv from dj_database_url import parse as db_url from .settings import * # flake8: noqa -config = AutoConfig() -config.config = Config(RepositoryEnv(os.path.abspath('sapl/legacy/.env'))) +config = Config(RepositoryEnv(BASE_DIR.child('legacy', '.env'))) INSTALLED_APPS += ( @@ -15,6 +14,13 @@ INSTALLED_APPS += ( DATABASES['legacy'] = config('DATABASE_URL', cast=db_url,) +# Sobrescreve o nome dos bancos caso a variável de ambiente seja definida +# Útil para migração em lote de vários bancos +DATABASE_NAME_OVERRIDE = os.environ.get('DATABASE_NAME') +if DATABASE_NAME_OVERRIDE: + for db in DATABASES.values(): + db['NAME'] = DATABASE_NAME_OVERRIDE + DATABASE_ROUTERS = ['sapl.legacy.router.LegacyRouter', ] DEBUG = False From 3bfda6c3817704cc563c935e30ccd900b50cf319 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 14 Mar 2017 19:06:05 -0300 Subject: [PATCH 2/3] =?UTF-8?q?Adiciona=20script=20para=20rodar=20shell=5F?= =?UTF-8?q?plus=20de=20migra=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit o script usa um banco específico, passado como argumento --- sapl/legacy/scripts/shell_para_migracao.sh | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100755 sapl/legacy/scripts/shell_para_migracao.sh diff --git a/sapl/legacy/scripts/shell_para_migracao.sh b/sapl/legacy/scripts/shell_para_migracao.sh new file mode 100755 index 000000000..fb0f3c6de --- /dev/null +++ b/sapl/legacy/scripts/shell_para_migracao.sh @@ -0,0 +1,8 @@ +#!/bin/bash + +# Inicia um shell_plus com as configurações de migração usando um banco específico +# Uso: ./shell_para_migracao.sh + +# Rode esse script a partir da raiz do projeto + +DATABASE_NAME=$1 ./manage.py shell_plus --settings sapl.legacy_migration_settings From cedb6cc15979d03fd4e479401f47fa089d10f755 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 14 Mar 2017 19:41:02 -0300 Subject: [PATCH 3/3] Exclui legacy da busca por ContentType p TipoAutor MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bugfix para migrações de vários bancos --- sapl/legacy/migration.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sapl/legacy/migration.py b/sapl/legacy/migration.py index c7bad24d9..e42fd5fe4 100644 --- a/sapl/legacy/migration.py +++ b/sapl/legacy/migration.py @@ -386,12 +386,13 @@ class DataMigrator: setattr(new, field.name, value) elif field.model.__name__ == 'TipoAutor' and \ field.name == 'content_type': - try: - value = field.related_model.objects.get( - model=normalize(new.descricao.lower()).replace(' ', - '')) - except ObjectDoesNotExist: - value = None + + model = normalize(new.descricao.lower()).replace(' ', '') + content_types = field.related_model.objects.filter( + model=model).exclude(app_label='legacy') + assert len(content_types) <= 1 + + value = content_types[0] if content_types else None setattr(new, field.name, value) def migrate(self, obj=appconfs, interativo=True):