From 5d7bdd5a5500cacf41fb019dbbd2cb5cf8a42be3 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Sat, 16 Jun 2018 15:36:53 -0300 Subject: [PATCH 01/19] Ajusta refs a XSLT ao migrar SDE --- sapl/legacy/migracao.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sapl/legacy/migracao.py b/sapl/legacy/migracao.py index 9ed09360f..dd54a103e 100644 --- a/sapl/legacy/migracao.py +++ b/sapl/legacy/migracao.py @@ -51,6 +51,9 @@ def salva_conteudo_do_sde(proposicao, conteudo): proposicao, 'proposicao_sde_{}.xml'.format(proposicao.pk)) caminho_absoluto = Path(REPO.working_dir, caminho_relativo) caminho_absoluto.parent.mkdir(parents=True) + # ajusta caminhos para folhas de estilo + conteudo = conteudo.replace(b'"XSLT/HTML', b'"/XSLT/HTML') + conteudo = conteudo.replace(b"'XSLT/HTML", b"'/XSLT/HTML") with open(caminho_absoluto, 'wb') as arq: arq.write(conteudo) proposicao.texto_original = caminho_relativo From f1519b9f0c7596a361c931108cdc940b45996a2f Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Sat, 16 Jun 2018 15:37:17 -0300 Subject: [PATCH 02/19] Adiciona backup do banco ao repo ao migrar --- sapl/legacy/migracao_dados.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 41c7b9934..895380493 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -1357,6 +1357,7 @@ def gravar_marco(): # salva mudanças REPO.git.add([dir_dados.name]) + REPO.git.add([arq_backup.name]) if 'master' not in REPO.heads or REPO.index.diff('HEAD'): # se de fato existe mudança REPO.index.commit('Grava marco') From de8d813ef20fe4a177eeed5ce0219e5739860666 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Sat, 16 Jun 2018 19:26:00 -0300 Subject: [PATCH 03/19] =?UTF-8?q?Adiciona=20verifica=C3=A7=C3=A3o=20de=20m?= =?UTF-8?q?time=20ao=20exportar=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scripts/exporta_zope/exporta_zope.py | 51 ++++++++++++------- 1 file changed, 34 insertions(+), 17 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 270c90583..33a007230 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -13,6 +13,7 @@ import sys from collections import defaultdict from contextlib import contextmanager from functools import partial +from os.path import exists import git import magic @@ -147,7 +148,7 @@ def enumerate_by_key_list(folder, key_list, type_key): for entry in folder.get(key_list, []): id, meta_type = entry['id'], entry[type_key] try: - obj = br(folder.get(id, None)) + obj = folder.get(id, None) except POSKeyError: print('#' * 80) print('#' * 80) @@ -170,7 +171,7 @@ def enumerate_btree(folder): tree = folder['_tree'] contagem_real = 0 # para o caso em que não haja itens for contagem_real, (id, obj) in enumerate(tree.iteritems(), start=1): - obj, meta_type = br(obj), type(obj).__name__ + meta_type = type(obj).__name__ yield id, obj, meta_type # verificação de consistência if contagem_esperada != contagem_real: @@ -197,10 +198,10 @@ def logando_nao_identificados(): print('#' * 80) -def dump_folder(folder, path, salvar, enum=enumerate_folder): +def dump_folder(folder, path, salvar, mtimes, enum=enumerate_folder): name = folder['id'] path = os.path.join(path, name) - if not os.path.exists(path): + if not exists(path): os.makedirs(path) for id, obj, meta_type in enum(folder): # pula pastas *_old (presentes em várias bases) @@ -210,8 +211,16 @@ def dump_folder(folder, path, salvar, enum=enumerate_folder): if dump == '?': nao_identificados[meta_type].append(path + '/' + id) elif dump: - id_interno = dump(obj, path, salvar) - assert id == id_interno + if isinstance(dump, partial) and dump.func == dump_folder: + dump(br(obj), path, salvar, mtimes) + else: + # se o objeto for mais recente que o da última exportação + mtime = obj._p_mtime + fullname = os.path.join(path, id) + if mtime > mtimes.get(fullname): + id_interno = dump(br(obj), path, salvar) + assert id == id_interno + mtimes[fullname] = mtime return name @@ -223,7 +232,7 @@ def read_sde(element): def read_properties(): for id, obj, meta_type in enumerate_properties(element): - yield id, decode_iso8859(obj) + yield id, decode_iso8859(br(obj)) def read_children(): for id, obj, meta_type in enumerate_folder(element): @@ -237,7 +246,7 @@ def read_sde(element): # ignoramos os scrips python de eventos dos templates yield {'id': id, 'meta_type': meta_type, - 'dados': read_sde(obj)} + 'dados': read_sde(br(obj))} data = dict(read_properties()) children = list(read_children()) @@ -335,9 +344,12 @@ def dump_usuarios(sapl, path, salvar): save_as_yaml(path, 'usuarios.yaml', users, salvar) -def _dump_sapl(data_fs_path, documentos_fs_path, destino, salvar): - assert Path(data_fs_path).exists() - assert Path(documentos_fs_path).exists() +def _dump_sapl(data_fs_path, documentos_fs_path, destino, salvar, mtimes): + assert exists(data_fs_path) + assert exists(documentos_fs_path) + # precisamos trabalhar com strings e não Path's para as comparações de mtimes + data_fs_path, documentos_fs_path, destino = map(str, ( + data_fs_path, documentos_fs_path, destino)) app, close_db = get_app(data_fs_path) try: @@ -352,12 +364,12 @@ def _dump_sapl(data_fs_path, documentos_fs_path, destino, salvar): sapl = find_sapl(app) # extrai folhas XSLT if 'XSLT' in sapl: - dump_folder(br(sapl['XSLT']), destino, salvar) + dump_folder(br(sapl['XSLT']), destino, salvar, mtimes) # extrai documentos docs = br(sapl['sapl_documentos']) with logando_nao_identificados(): - dump_folder(docs, destino, salvar) + dump_folder(docs, destino, salvar, mtimes) dump_propriedades(docs, destino, salvar) finally: close_db() @@ -392,7 +404,7 @@ def build_salvar(repo): print('- hash encontrado - {}'.format(fullname)) else: fullname = ajusta_extensao(fullname, conteudo) - if os.path.exists(fullname): + if exists(fullname): # destrava arquivo pré-existente (o conteúdo mudou) repo_execute(repo, 'git annex unlock', fullname) with open(fullname, 'w') as arq: @@ -409,8 +421,8 @@ def dump_sapl(sigla): 'datafs', '{}_cm_{}.fs'.format(prefixo, sigla)) for prefixo in ('Data', 'DocumentosSapl')] - assert data_fs_path.exists(), 'Origem não existe: {}'.format(data_fs_path) - if not documentos_fs_path.exists(): + assert exists(data_fs_path), 'Origem não existe: {}'.format(data_fs_path) + if not exists(documentos_fs_path): documentos_fs_path = data_fs_path nome_banco_legado = 'sapl_cm_{}'.format(sigla) @@ -427,12 +439,17 @@ def dump_sapl(sigla): salvar = build_salvar(repo) try: finalizado = False - _dump_sapl(data_fs_path, documentos_fs_path, destino, salvar) + arq_mtimes = Path(repo.working_dir, 'mtimes.yaml') + mtimes = yaml.load( + arq_mtimes.read_file()) if arq_mtimes.exists() else {} + _dump_sapl(data_fs_path, documentos_fs_path, destino, salvar, mtimes) finalizado = True finally: # grava mundaças repo_execute(repo, 'git annex add sapl_documentos') repo.git.add(A=True) + arq_mtimes.write_file(yaml.safe_dump(mtimes, allow_unicode=True)) + # atualiza repo if 'master' not in repo.heads or repo.index.diff('HEAD'): # se de fato existe mudança status = 'completa' if finalizado else 'parcial' From a1b2d3bc7efe09da444388fc32eb805a8e7179c1 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Sat, 16 Jun 2018 19:33:06 -0300 Subject: [PATCH 04/19] =?UTF-8?q?Retira=20verifica=C3=A7=C3=A3o=20de=20has?= =?UTF-8?q?hes=20do=20annex=20ao=20exportar=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Já estamos usando verificando o mtime --- .../scripts/exporta_zope/exporta_zope.py | 28 +++++-------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 33a007230..5639a7665 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -379,12 +379,6 @@ def repo_execute(repo, cmd, *args): return repo.git.execute(cmd.split() + list(args)) -def get_annex_hashes(repo): - hashes = repo_execute( - repo, 'git annex find', '--format=${keyname}\n', '--include=*') - return {os.path.splitext(h)[0] for h in hashes.splitlines()} - - def ajusta_extensao(fullname, conteudo): base, extensao = os.path.splitext(fullname) if extensao not in ['.xsl', '.xslt', '.yaml', '.css']: @@ -393,23 +387,15 @@ def ajusta_extensao(fullname, conteudo): def build_salvar(repo): - """Constroi função salvar que pula arquivos que já estão no annex - """ - hashes = get_annex_hashes(repo) def salvar(fullname, conteudo): - sha = hashlib.sha256() - sha.update(conteudo) - if sha.hexdigest() in hashes: - print('- hash encontrado - {}'.format(fullname)) - else: - fullname = ajusta_extensao(fullname, conteudo) - if exists(fullname): - # destrava arquivo pré-existente (o conteúdo mudou) - repo_execute(repo, 'git annex unlock', fullname) - with open(fullname, 'w') as arq: - arq.write(conteudo) - print(fullname) + fullname = ajusta_extensao(fullname, conteudo) + if exists(fullname): + # destrava arquivo pré-existente (o conteúdo mudou) + repo_execute(repo, 'git annex unlock', fullname) + with open(fullname, 'w') as arq: + arq.write(conteudo) + print(fullname) return salvar From a62fe294570f49968c4385efa4aff1c4070fe33b Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Sun, 17 Jun 2018 16:18:24 -0300 Subject: [PATCH 05/19] =?UTF-8?q?Ajusta=20compara=C3=A7=C3=A3o=20de=20mtim?= =?UTF-8?q?es=20na=20exporta=C3=A7=C3=A3o=20de=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 5639a7665..8f78b060f 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -217,7 +217,7 @@ def dump_folder(folder, path, salvar, mtimes, enum=enumerate_folder): # se o objeto for mais recente que o da última exportação mtime = obj._p_mtime fullname = os.path.join(path, id) - if mtime > mtimes.get(fullname): + if mtime > mtimes.get(fullname, 0): id_interno = dump(br(obj), path, salvar) assert id == id_interno mtimes[fullname] = mtime From ad9f0bd846805ac11f4b03d304167c0837ae52fa Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 19 Jun 2018 12:02:34 -0300 Subject: [PATCH 06/19] Inclui arq mtimes.yaml no versionamento ao exportar docs --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 8f78b060f..ca5044db1 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -433,8 +433,8 @@ def dump_sapl(sigla): finally: # grava mundaças repo_execute(repo, 'git annex add sapl_documentos') - repo.git.add(A=True) arq_mtimes.write_file(yaml.safe_dump(mtimes, allow_unicode=True)) + repo.git.add(A=True) # atualiza repo if 'master' not in repo.heads or repo.index.diff('HEAD'): # se de fato existe mudança From 84aa257b7087a0ded12a2dfa97e32212c85be978 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Wed, 20 Jun 2018 19:07:59 -0300 Subject: [PATCH 07/19] =?UTF-8?q?Limita=20buffer=20para=20magic=20na=20exp?= =?UTF-8?q?orta=C3=A7=C3=A3o=20de=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index ca5044db1..568535d7f 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -79,7 +79,7 @@ def br(obj): def guess_extension(fullname, buffer): - mime = magic.from_buffer(buffer, mime=True) + mime = magic.from_buffer(buffer[:1024], mime=True) extensao = EXTENSOES.get(mime) if extensao is not None: return extensao From 97bf1347d317b409c331668dbf0be58d1f501fc8 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 21 Jun 2018 14:14:08 -0300 Subject: [PATCH 08/19] =?UTF-8?q?Corrige=20ordem=20de=20propaga=C3=A7?= =?UTF-8?q?=C3=A3o=20de=20exclus=C3=B5es?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_dados.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 895380493..5a75afb26 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -567,8 +567,8 @@ def propaga_exclusoes(): def uniformiza_banco(): exec_legado('SET SESSION sql_mode = "";') # desliga checagens do mysql - checa_registros_votacao_ambiguos_e_remove_nao_usados() propaga_exclusoes() + checa_registros_votacao_ambiguos_e_remove_nao_usados() garante_coluna_no_legado('proposicao', 'num_proposicao int(11) NULL') From 7de5654166a25f2374343784e2d32947bcbafd4f Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 21 Jun 2018 14:14:40 -0300 Subject: [PATCH 09/19] Unifica autores por username --- sapl/legacy/migracao_dados.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 5a75afb26..8d3489439 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -649,6 +649,7 @@ sessao_plenaria_presenca | dat_sessao = NULL | dat_sessao = 0 unifica_autores_repetidos_no_legado('cod_parlamentar') unifica_autores_repetidos_no_legado('cod_comissao') + unifica_autores_repetidos_no_legado('col_username') # é importante reverter a exclusão de autores somente depois, para que a # unificação possa dar prioridade às informações dos autores não excluídos From 5c0f4a05544f159146e14dabeca20742cac978e6 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 21 Jun 2018 14:14:58 -0300 Subject: [PATCH 10/19] =?UTF-8?q?Corrige=20verifica=C3=A7=C3=A3o=20de=20re?= =?UTF-8?q?uni=C3=B5es=20no=20legado?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_dados.py | 7 ++++++- sapl/legacy/migracao_documentos.py | 6 +++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 8d3489439..2df65c51b 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -855,12 +855,17 @@ def move_para_depois_de(lista, movido, referencias): return lista +TABELAS_LEGADO = [t for (t,) in exec_legado('show tables')] +EXISTE_REUNIAO_NO_LEGADO = 'reuniao_comissao' in TABELAS_LEGADO + + def get_models_a_migrar(): models = [model for app in appconfs for model in app.models.values() if model in field_renames] # retira reuniões quando não existe na base legada # (só existe no sapl 3.0) - if 'reuniao_comissao' not in list(exec_legado('show tables')): + tabelas_legado = [t for (t,) in exec_legado('show tables')] + if not EXISTE_REUNIAO_NO_LEGADO: models.remove(Reuniao) # Devido à referência TipoProposicao.tipo_conteudo_related # a migração de TipoProposicao precisa ser feita diff --git a/sapl/legacy/migracao_documentos.py b/sapl/legacy/migracao_documentos.py index fd692aa47..d7fbd89c2 100644 --- a/sapl/legacy/migracao_documentos.py +++ b/sapl/legacy/migracao_documentos.py @@ -9,7 +9,7 @@ from image_cropping.fields import ImageCropField from sapl.base.models import CasaLegislativa from sapl.comissoes.models import Reuniao -from sapl.legacy.migracao_dados import exec_legado +from sapl.legacy.migracao_dados import EXISTE_REUNIAO_NO_LEGADO, exec_legado from sapl.materia.models import (DocumentoAcessorio, MateriaLegislativa, Proposicao) from sapl.norma.models import NormaJuridica @@ -36,9 +36,9 @@ DOCS = { } # acrescenta reuniões (que só existem no sapl 3.0) -if 'reuniao_comissao' in set(exec_legado('show tables')): +if EXISTE_REUNIAO_NO_LEGADO: DOCS[Reuniao] = [('upload_pauta', 'reuniao_comissao/{}_pauta'), - ('upload_ata', 'reuniao_comissao/{}_ata')], + ('upload_ata', 'reuniao_comissao/{}_ata')] DOCS = {model: [(campo, join('sapl_documentos', origem)) From f2985f9af9432cf0e54b6db0d2e8efb3c47589f3 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 22 Jun 2018 10:53:20 -0300 Subject: [PATCH 11/19] =?UTF-8?q?Otimiza=20migra=C3=A7=C3=A3o=20de=20docum?= =?UTF-8?q?entos?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_documentos.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sapl/legacy/migracao_documentos.py b/sapl/legacy/migracao_documentos.py index d7fbd89c2..44e52ffef 100644 --- a/sapl/legacy/migracao_documentos.py +++ b/sapl/legacy/migracao_documentos.py @@ -53,7 +53,7 @@ def mover_documento(repo, origem, destino, ignora_origem_ausente=False): print('Origem ignorada ao mover documento: {}'.format(origem)) return os.makedirs(os.path.dirname(destino), exist_ok=True) - repo.git.mv(origem, destino) + os.rename(origem, destino) def migrar_logotipo(repo, casa, propriedades): @@ -167,6 +167,10 @@ def migrar_documentos(repo): for model in DOCS: migrar_docs_por_ids(repo, model) + # versiona modificações + repo.git.add('-A', '.') + repo.index.commit('Migração dos documentos completa') + sobrando = [join(dir, file) for (dir, _, files) in os.walk(join(repo.working_dir, 'sapl_documentos')) From b3aa5d77902f7430ea1c3d43c1c56e85eee12bbb Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Sat, 23 Jun 2018 10:52:09 -0300 Subject: [PATCH 12/19] Resiste a POSKeyError ao enumerar btree --- .../scripts/exporta_zope/exporta_zope.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 568535d7f..c2b8e7973 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -144,17 +144,21 @@ def get_conteudo_dtml_method(doc): return doc['raw'] +def print_msg_poskeyerror(id): + print('#' * 80) + print('#' * 80) + print('ATENÇÃO: DIRETÓRIO corrompido: {}'.format(id)) + print('#' * 80) + print('#' * 80) + + def enumerate_by_key_list(folder, key_list, type_key): for entry in folder.get(key_list, []): id, meta_type = entry['id'], entry[type_key] try: obj = folder.get(id, None) except POSKeyError: - print('#' * 80) - print('#' * 80) - print('ATENÇÃO: DIRETÓRIO corrompido: {}'.format(id)) - print('#' * 80) - print('#' * 80) + print_msg_poskeyerror(id) else: yield id, obj, meta_type @@ -170,9 +174,12 @@ def enumerate_btree(folder): contagem_esperada = folder['_count'].value tree = folder['_tree'] contagem_real = 0 # para o caso em que não haja itens - for contagem_real, (id, obj) in enumerate(tree.iteritems(), start=1): - meta_type = type(obj).__name__ - yield id, obj, meta_type + try: + for contagem_real, (id, obj) in enumerate(tree.iteritems(), start=1): + meta_type = type(obj).__name__ + yield id, obj, meta_type + except POSKeyError: + print_msg_poskeyerror(folder['id']) # verificação de consistência if contagem_esperada != contagem_real: print('ATENÇÃO: contagens diferentes na btree: ' From 82c18f5c95b97d56fcc5a786ad5a68519bf288bd Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Sat, 23 Jun 2018 11:16:56 -0300 Subject: [PATCH 13/19] Preserva ordem ao ler yaml no python 3.5 --- sapl/legacy/migracao_dados.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 2df65c51b..99d656504 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -53,6 +53,22 @@ from sapl.utils import normalize from .scripts.normaliza_dump_mysql import normaliza_dump_mysql from .timezonesbrasil import get_timezone + + +# YAML SETUP ############################################################### +def dict_representer(dumper, data): + return dumper.represent_dict(data.items()) + +yaml.add_representer(OrderedDict, dict_representer) + + +# importante para preservar a ordem ao ler yaml no python 3.5 +def dict_constructor(loader, node): + return OrderedDict(loader.construct_pairs(node)) + +yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + dict_constructor) + # BASE ###################################################################### # apps to be migrated, in app dependency order (very important) appconfs = [apps.get_app_config(n) for n in [ @@ -740,13 +756,6 @@ def reinicia_sequence(model, id): REPO = git.Repo.init(DIR_REPO) -def dict_representer(dumper, data): - return dumper.represent_dict(data.items()) - - -yaml.add_representer(OrderedDict, dict_representer) - - # configura timezone de migração match = re.match('sapl_cm_(.*)', NOME_BANCO_LEGADO) sigla_casa = match.group(1) From 8fbc5aee5cb676e3c664509ff8484e9135a871ea Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Sat, 23 Jun 2018 12:35:07 -0300 Subject: [PATCH 14/19] =?UTF-8?q?Corrige=20git=20annex=20fix=20na=20migra?= =?UTF-8?q?=C3=A7=C3=A3o=20de=20imagens?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_documentos.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sapl/legacy/migracao_documentos.py b/sapl/legacy/migracao_documentos.py index 44e52ffef..b246cd467 100644 --- a/sapl/legacy/migracao_documentos.py +++ b/sapl/legacy/migracao_documentos.py @@ -140,6 +140,7 @@ def migrar_docs_por_ids(repo, model): if tem_cropping: # conserta link do git annex (antes do commit) # pois o conteúdo das imagens é acessado pelo cropping + repo.git.add(destino) repo.git.execute('git annex fix'.split() + [destino]) obj.save() else: From 943ce8046993c8b7841ebc68a8590f8111944921 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Sat, 23 Jun 2018 12:44:37 -0300 Subject: [PATCH 15/19] Verifica se dir parlamentar existe antes de git annex get --- sapl/legacy/migracao_documentos.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sapl/legacy/migracao_documentos.py b/sapl/legacy/migracao_documentos.py index b246cd467..66ada19fc 100644 --- a/sapl/legacy/migracao_documentos.py +++ b/sapl/legacy/migracao_documentos.py @@ -163,7 +163,9 @@ def migrar_documentos(repo): # garante que o conteúdo das fotos dos parlamentares esteja presente # (necessário para o cropping de imagem) - repo.git.execute('git annex get sapl_documentos/parlamentar'.split()) + if os.path.exists( + os.path.join(repo.working_dir, 'sapl_documentos/parlamentar')): + repo.git.execute('git annex get sapl_documentos/parlamentar'.split()) for model in DOCS: migrar_docs_por_ids(repo, model) From 78bf262f41c6ea574a39cc8d506c8c264bfb6203 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Mon, 25 Jun 2018 10:35:06 -0300 Subject: [PATCH 16/19] Resiste a POSKeyError em mais um ponto --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index c2b8e7973..51be333af 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -219,7 +219,11 @@ def dump_folder(folder, path, salvar, mtimes, enum=enumerate_folder): nao_identificados[meta_type].append(path + '/' + id) elif dump: if isinstance(dump, partial) and dump.func == dump_folder: - dump(br(obj), path, salvar, mtimes) + try: + dump(br(obj), path, salvar, mtimes) + except POSKeyError as e: + print_msg_poskeyerror(id) + continue else: # se o objeto for mais recente que o da última exportação mtime = obj._p_mtime From 1cc3d1a19ab1069242b67c7a4227d4ced44b1f74 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Wed, 27 Jun 2018 13:52:00 -0300 Subject: [PATCH 17/19] =?UTF-8?q?Corrige=20detec=C3=A7=C3=A3o=20.docx=20na?= =?UTF-8?q?=20exporta=C3=A7=C3=A3o=20do=20zope?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 51be333af..bf71d051c 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -79,7 +79,8 @@ def br(obj): def guess_extension(fullname, buffer): - mime = magic.from_buffer(buffer[:1024], mime=True) + # um corte de apenas 1024 impediu a detecção correta de .docx + mime = magic.from_buffer(buffer[:4096], mime=True) extensao = EXTENSOES.get(mime) if extensao is not None: return extensao From c599db3e9edb9c2b3e8e578533fe4cba10ad55ee Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 29 Jun 2018 14:27:17 -0300 Subject: [PATCH 18/19] Apaga banco antes de migrar flush direto --- sapl/legacy/migracao_dados.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 99d656504..786824880 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -22,6 +22,7 @@ from django.contrib.auth import get_user_model from django.contrib.auth.models import Group from django.contrib.contenttypes.models import ContentType from django.core.exceptions import ObjectDoesNotExist +from django.core.management.commands.flush import Command as FlushCommand from django.db import connections, transaction from django.db.models import Max, Q from pyaml import UnsafePrettyYAMLDumper @@ -54,7 +55,6 @@ from .scripts.normaliza_dump_mysql import normaliza_dump_mysql from .timezonesbrasil import get_timezone - # YAML SETUP ############################################################### def dict_representer(dumper, data): return dumper.represent_dict(data.items()) @@ -64,7 +64,7 @@ yaml.add_representer(OrderedDict, dict_representer) # importante para preservar a ordem ao ler yaml no python 3.5 def dict_constructor(loader, node): - return OrderedDict(loader.construct_pairs(node)) + return OrderedDict(loader.construct_pairs(node)) yaml.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, dict_constructor) @@ -830,8 +830,8 @@ def migrar_dados(): # excluindo database antigo. info('Excluindo entradas antigas do banco destino.') - call([PROJECT_DIR.child('manage.py'), 'flush', - '--database=default', '--no-input'], stdout=PIPE) + flush = FlushCommand() + flush.handle(database='default', interactive=False, verbosity=0) # apaga tipos de autor padrão (criados no flush acima) TipoAutor.objects.all().delete() From 6000ac3b20efd41dd58e7b5d590451e67ce297de Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 3 Jul 2018 12:56:21 -0300 Subject: [PATCH 19/19] =?UTF-8?q?Corrige=20decodifica=C3=A7=C3=A3o=20de=20?= =?UTF-8?q?assuntos=20de=20norma?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_dados.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 786824880..ed05b2e90 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -1195,8 +1195,8 @@ def adjust_normajuridica_depois_salvar(): for model in [AssuntoNorma, NormaJuridica]] def filtra_assuntos_migrados(cod_assunto): - return [a for a in map(int, cod_assunto.split(',')) - if a in assuntos_migrados] + cods = {int(a) for a in cod_assunto.split(',') if a} + return cods.intersection(assuntos_migrados) norma_para_assuntos = [ (norma, filtra_assuntos_migrados(cod_assunto)) @@ -1206,7 +1206,7 @@ def adjust_normajuridica_depois_salvar(): ligacao.objects.bulk_create( ligacao(normajuridica_id=norma, assuntonorma_id=assunto) for norma, assuntos in norma_para_assuntos - for assunto in assuntos) + for assunto in sorted(assuntos)) def adjust_autor(new, old):