From 8e3977b1abd7d31faae0f9ef2d89cac43e732109 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Mon, 23 Apr 2018 14:50:01 -0300 Subject: [PATCH] =?UTF-8?q?Corrige=20normaliza=C3=A7=C3=A3o=20de=20dump=20?= =?UTF-8?q?mysql?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_dados.py | 21 +++++++---- sapl/legacy/scripts/normaliza_dump_mysql.py | 40 +++++++++++++++++++++ sapl/legacy/scripts/normaliza_dump_mysql.sh | 28 --------------- 3 files changed, 54 insertions(+), 35 deletions(-) create mode 100755 sapl/legacy/scripts/normaliza_dump_mysql.py delete mode 100755 sapl/legacy/scripts/normaliza_dump_mysql.sh diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 301b11458..3fc2fc2d9 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -12,9 +12,12 @@ from subprocess import PIPE, call import git import pkg_resources import pyaml +import yaml +from pyaml import UnsafePrettyYAMLDumper +from unipath import Path + import pytz import reversion -import yaml from django.apps import apps from django.contrib.auth import get_user_model from django.contrib.auth.models import Group @@ -22,9 +25,6 @@ from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist from django.db import connections, transaction from django.db.models import Max, Q -from pyaml import UnsafePrettyYAMLDumper -from unipath import Path - from sapl.base.models import AppConfig as AppConf from sapl.base.models import Autor, TipoAutor, cria_models_tipo_autor from sapl.comissoes.models import Comissao, Composicao, Participacao @@ -48,6 +48,7 @@ from sapl.sessao.models import (ExpedienteMateria, OrdemDia, RegistroVotacao, TipoResultadoVotacao) from sapl.utils import normalize +from .scripts.normaliza_dump_mysql import normaliza_dump_mysql from .timezonesbrasil import get_timezone # BASE ###################################################################### @@ -144,6 +145,7 @@ for nome_novo, nome_antigo in (('comissao', 'cod_comissao'), class CampoVirtual(namedtuple('CampoVirtual', 'model related_model')): null = True + CAMPOS_VIRTUAIS_PROPOSICAO = { TipoMateriaLegislativa: CampoVirtual(Proposicao, MateriaLegislativa), TipoDocumento: CampoVirtual(Proposicao, DocumentoAdministrativo) @@ -176,6 +178,7 @@ for related, campo_antigo in [(Parlamentar, 'cod_parlamentar'), def info(msg): print('INFO: ' + msg) + ocorrencias = defaultdict(list) @@ -738,6 +741,8 @@ REPO = git.Repo.init(DIR_REPO) def dict_representer(dumper, data): return dumper.represent_dict(data.items()) + + yaml.add_representer(OrderedDict, dict_representer) @@ -799,9 +804,7 @@ def migrar_dados(interativo=True): 'dumps_mysql', '{}.sql'.format(NOME_BANCO_LEGADO))) assert arq_dump.exists(), 'Dump do mysql faltando: {}'.format(arq_dump) info('Restaurando dump mysql de [{}]'.format(arq_dump)) - normaliza_dump_mysql = Path(scripts.__file__).parent.child( - 'normaliza_dump_mysql.sh') - roda_comando_shell('{} {}'.format(normaliza_dump_mysql, arq_dump)) + normaliza_dump_mysql(arq_dump) roda_comando_shell('mysql -uroot < {}'.format(arq_dump)) # executa ajustes pré-migração, se existirem @@ -1276,6 +1279,8 @@ TIME_FORMAT = '%H:%M:%S' # permite a gravação de tempos puros pelo pretty-yaml def time_representer(dumper, data): return dumper.represent_scalar('!time', data.strftime(TIME_FORMAT)) + + UnsafePrettyYAMLDumper.add_representer(datetime.time, time_representer) @@ -1283,6 +1288,8 @@ UnsafePrettyYAMLDumper.add_representer(datetime.time, time_representer) def time_constructor(loader, node): value = loader.construct_scalar(node) return datetime.datetime.strptime(value, TIME_FORMAT).time() + + yaml.add_constructor(u'!time', time_constructor) diff --git a/sapl/legacy/scripts/normaliza_dump_mysql.py b/sapl/legacy/scripts/normaliza_dump_mysql.py new file mode 100755 index 000000000..a56d74b3a --- /dev/null +++ b/sapl/legacy/scripts/normaliza_dump_mysql.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +import re +import sys + +from unipath import Path + +cabecalho = ''' +/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; +/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; +/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; +/*!40101 SET NAMES utf8 */; +/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; +/*!40103 SET TIME_ZONE='+00:00' */; +/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; +/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; +/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; +/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; + +/*!40000 DROP DATABASE IF EXISTS `{banco}`*/; + +CREATE DATABASE /*!32312 IF NOT EXISTS*/ `{banco}` /*!40100 DEFAULT CHARACTER SET latin1 */; + +USE `{banco}`; + +''' + + +def normaliza_dump_mysql(nome_arquivo): + arquivo = Path(nome_arquivo).expand() + banco = arquivo.stem + conteudo = arquivo.read_file() + inicio = re.finditer('--\n-- Table structure for table .*\n--\n', conteudo) + inicio = next(inicio).start() + conteudo = cabecalho.format(banco=banco) + conteudo[inicio:] + arquivo.write_file(conteudo) + + +if __name__ == "__main__": + nome_aquivo = sys.argv[1] + normaliza_dump_mysql(nome_aquivo) diff --git a/sapl/legacy/scripts/normaliza_dump_mysql.sh b/sapl/legacy/scripts/normaliza_dump_mysql.sh deleted file mode 100755 index 75bd5435f..000000000 --- a/sapl/legacy/scripts/normaliza_dump_mysql.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash - -ARQUIVO=$1 -BANCO=`basename $1 | cut -f1 -d.` -TMP=__tmp.sql - -cat << EOF > $TMP - -/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; -/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; -/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; -/*!40101 SET NAMES utf8 */; -/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; -/*!40103 SET TIME_ZONE='+00:00' */; -/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; -/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; -/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; -/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; - -/*!40000 DROP DATABASE IF EXISTS \`$BANCO\`*/; - -CREATE DATABASE /*!32312 IF NOT EXISTS*/ \`$BANCO\` /*!40100 DEFAULT CHARACTER SET latin1 */; - -USE \`$BANCO\`; -EOF - -sed 1,`grep -n '^USE ' $ARQUIVO |cut -f1 -d:`d $ARQUIVO >> $TMP -mv $TMP $ARQUIVO