Merge pull request #937 from interlegis/932-migracao-em-paralelo

Fix #932 Migracao em paralelo

sapl/legacy/management/commands/migracao_25_31.py

@@ -4,7 +4,17 @@ from sapl.legacy import migration
 
 
 class Command(BaseCommand):
-    help = u'Faz a migração de dados do SAPL 2.5 para o SAPL 3.1'
+
+    help = u'Migração de dados do SAPL 2.5 para o SAPL 3.1'
+
+    def add_arguments(self, parser):
+        parser.add_argument(
+            '-f',
+            action='store_true',
+            default=False,
+            dest='force',
+            help='Não interativa: pula confirmação de exclusão dos dados',
+        )
 
     def handle(self, *args, **options):
-        migration.migrate()
+        migration.migrate(interativo=not options['force'])

sapl/legacy/migration.py

@@ -11,11 +11,10 @@ from django.contrib.auth import get_user_model
 from django.contrib.contenttypes.models import ContentType
 from django.core.exceptions import ObjectDoesNotExist
 from django.db import OperationalError, ProgrammingError, connections, models
-from django.db.models import CharField, Max, ProtectedError, TextField, signals
+from django.db.models import CharField, Max, ProtectedError, TextField
 from django.db.models.base import ModelBase
 from model_mommy import mommy
 from model_mommy.mommy import foreign_key_required, make
-from reversion.models import Revision, Version
 
 from sapl.base.models import Autor, ProblemaMigracao
 from sapl.comissoes.models import Comissao, Composicao, Participacao
@@ -28,6 +27,7 @@ from sapl.norma.models import (AssuntoNorma, NormaJuridica,
 from sapl.parlamentares.models import Parlamentar
 from sapl.protocoloadm.models import Protocolo, StatusTramitacaoAdministrativo
 from sapl.sessao.models import ExpedienteMateria, OrdemDia, SessaoPlenaria
+from sapl.settings import PROJECT_DIR
 from sapl.utils import normalize
 
 # BASE ######################################################################
@@ -316,9 +316,9 @@ def fill_vinculo_norma_juridica():
              ('L', 'Suspende integralmente a norma'),
              ('N', 'Julgada integralmente inconstitucional'),
              ('O', 'Julgada parcialmente inconstitucional')]
-    lista_objs = [VinculoNormaJuridica(sigla=item[0], descricao=item[1])
+    lista_objs = [TipoVinculoNormaJuridica(sigla=item[0], descricao=item[1])
                   for item in lista]
-    VinculoNormaJuridica.objects.bulk_create(lista_objs)
+    TipoVinculoNormaJuridica.objects.bulk_create(lista_objs)
 
 
 class DataMigrator:
@@ -386,32 +386,37 @@ class DataMigrator:
                 setattr(new, field.name, value)
             elif field.model.__name__ == 'TipoAutor' and \
                     field.name == 'content_type':
-                try:
+
-                    value = field.related_model.objects.get(
+                model = normalize(new.descricao.lower()).replace(' ', '')
-                        model=normalize(new.descricao.lower()).replace(' ',
+                content_types = field.related_model.objects.filter(
-                                                                       ''))
+                    model=model).exclude(app_label='legacy')
-                except ObjectDoesNotExist:
+                assert len(content_types) <= 1
-                    value = None
+
+                value = content_types[0] if content_types else None
                 setattr(new, field.name, value)
 
-    def migrate(self, obj=appconfs):
+    def migrate(self, obj=appconfs, interativo=True):
         # warning: model/app migration order is of utmost importance
-        exec_sql_file('sapl/legacy/scripts/fix_tables.sql', 'legacy')
+        exec_sql_file(PROJECT_DIR.child(
+            'sapl', 'legacy', 'scripts', 'fix_tables.sql'), 'legacy')
         self.to_delete = []
 
         # excluindo database antigo.
-        info('Todos os dados do banco serão excluidos. '
+        if interativo:
-             'Recomendamos que faça backup do banco sapl antes de continuar.')
+            info('Todos os dados do banco serão excluidos. '
-        info('Deseja continuar? [s/n]')
+                 'Recomendamos que faça backup do banco sapl '
-        resposta = input()
+                 'antes de continuar.')
-        if resposta.lower() in ['s', 'sim', 'y', 'yes']:
+            info('Deseja continuar? [s/n]')
-            pass
+            resposta = input()
-        else:
+            if resposta.lower() in ['s', 'sim', 'y', 'yes']:
-            info('Migração cancelada.')
+                pass
-            return 0
+            else:
+                info('Migração cancelada.')
+                return 0
         info('Excluindo entradas antigas do banco.')
-        call(['./manage.py', 'flush', '--settings=sapl.settings',
+        call([PROJECT_DIR.child('manage.py'), 'flush',
-              '--database=default', '--no-input'], stdout=PIPE)
+              '--settings=sapl.settings', '--database=default', '--no-input'],
+             stdout=PIPE)
 
         info('Começando migração: %s...' % obj)
         self._do_migrate(obj)
@@ -528,9 +533,9 @@ class DataMigrator:
         return excluidos
 
 
-def migrate(obj=appconfs):
+def migrate(obj=appconfs, interativo=True):
     dm = DataMigrator()
-    dm.migrate(obj)
+    dm.migrate(obj, interativo)
 
 
 # MIGRATION_ADJUSTMENTS #####################################################

sapl/legacy/scripts/migra_dbs.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+# rodar esse script na raiz do projeto
+
+parallel --verbose ./sapl/legacy/scripts/migra_um_db.sh :::: <(mysql -u root -padmin -e 'show databases;' | grep '^sapl_')

sapl/legacy/scripts/migra_um_db.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+# rodar esse script na raiz do projeto
+
+DIR=~/logs_migracao
+mkdir -p $DIR
+
+LOG="$DIR/$1.migracao.log"
+rm -f $LOG
+
+echo "########################################" | tee -a $LOG
+echo "MIGRANDO BANCO $1" | tee -a $LOG
+echo "########################################" | tee -a $LOG
+echo >> $LOG
+
+
+echo "--- DJANGO MIGRATE ---" | tee -a $LOG
+echo >> $LOG
+DATABASE_NAME=$1 ./manage.py migrate --settings sapl.legacy_migration_settings
+echo >> $LOG
+
+echo "--- MIGRACAO DE DADOS ---" | tee -a $LOG
+echo >> $LOG
+DATABASE_NAME=$1 ./manage.py migracao_25_31 -f --settings sapl.legacy_migration_settings |& tee -a $LOG
+echo >> $LOG

sapl/legacy/scripts/recria_dbs_postgres.sh

@@ -0,0 +1,5 @@
+# (Re)cria todos os bancos postgres para migração
+# cria um banco postgres (de mesmo nome) para cada banco mysql cujo nome começa com "sapl_"
+
+mysql -u root -padmin -e 'show databases;' | grep '^sapl_' | xargs -I{} ./recria_um_db_postgres.sh {}
+

sapl/legacy/scripts/recria_um_db_postgres.sh

@@ -0,0 +1,5 @@
+# (Re)cria um db postgres
+# uso:  recria_um_db_postgres <NOME DO BANCO>
+
+sudo -u postgres psql -c "drop DATABASE if exists $1"
+sudo -u postgres psql -c "CREATE DATABASE $1 WITH OWNER = sapl ENCODING = 'UTF8' TABLESPACE = pg_default LC_COLLATE = 'pt_BR.UTF-8' LC_CTYPE = 'pt_BR.UTF-8' CONNECTION LIMIT = -1 TEMPLATE template0;"

sapl/legacy/scripts/shell_para_migracao.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# Inicia um shell_plus com as configurações de migração usando um banco específico
+# Uso: ./shell_para_migracao.sh <NOME DO BANCO>
+
+# Rode esse script a partir da raiz do projeto
+
+DATABASE_NAME=$1 ./manage.py shell_plus --settings sapl.legacy_migration_settings

sapl/legacy_migration_settings.py

@@ -1,12 +1,11 @@
 import os
 
-from decouple import AutoConfig, Config, RepositoryEnv
+from decouple import Config, RepositoryEnv
 from dj_database_url import parse as db_url
 
 from .settings import *  # flake8: noqa
 
-config = AutoConfig()
+config = Config(RepositoryEnv(BASE_DIR.child('legacy', '.env')))
-config.config = Config(RepositoryEnv(os.path.abspath('sapl/legacy/.env')))
 
 
 INSTALLED_APPS += (
@@ -15,6 +14,13 @@ INSTALLED_APPS += (
 
 DATABASES['legacy'] = config('DATABASE_URL', cast=db_url,)
 
+# Sobrescreve o nome dos bancos caso a variável de ambiente seja definida
+# Útil para migração em lote de vários bancos
+DATABASE_NAME_OVERRIDE = os.environ.get('DATABASE_NAME')
+if DATABASE_NAME_OVERRIDE:
+    for db in DATABASES.values():
+        db['NAME'] = DATABASE_NAME_OVERRIDE
+
 DATABASE_ROUTERS = ['sapl.legacy.router.LegacyRouter', ]
 
 DEBUG = False