From 11384f4ebadd0cbc26250c7cb1896310b99714f9 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 29 May 2018 10:05:29 -0300 Subject: [PATCH 01/20] =?UTF-8?q?Exec=20migrate=20s=C3=B3=20depois=20de=20?= =?UTF-8?q?testar=20tag=20marco?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/management/commands/migracao_25_31.py | 2 -- sapl/legacy/migracao.py | 2 ++ 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sapl/legacy/management/commands/migracao_25_31.py b/sapl/legacy/management/commands/migracao_25_31.py index 4541425c4..e3864ad0f 100644 --- a/sapl/legacy/management/commands/migracao_25_31.py +++ b/sapl/legacy/management/commands/migracao_25_31.py @@ -1,4 +1,3 @@ -from django.core import management from django.core.management.base import BaseCommand from sapl.legacy.migracao import migrar @@ -9,5 +8,4 @@ class Command(BaseCommand): help = 'Migração de dados do SAPL 2.5 para o SAPL 3.1' def handle(self, *args, **options): - management.call_command('migrate') migrar(interativo=False) diff --git a/sapl/legacy/migracao.py b/sapl/legacy/migracao.py index ffd0d2a18..c2d863b36 100644 --- a/sapl/legacy/migracao.py +++ b/sapl/legacy/migracao.py @@ -2,6 +2,7 @@ import subprocess from getpass import getpass import requests +from django.core import management from unipath import Path from sapl.legacy.migracao_dados import (REPO, TAG_MARCO, gravar_marco, info, @@ -24,6 +25,7 @@ def migrar(interativo=False): assert TAG_ZOPE in REPO.tags, adornar_msg( 'Antes de migrar ' 'é necessário fazer a exportação de documentos do zope') + management.call_command('migrate') migrar_dados(interativo=interativo) migrar_usuarios(REPO.working_dir) migrar_documentos(REPO) From e71f549d94374e1ff27c198262cdd59922c63c94 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 29 May 2018 15:28:29 -0300 Subject: [PATCH 02/20] =?UTF-8?q?Adapta=20exporta=C3=A7=C3=A3o=20do=20zope?= =?UTF-8?q?=20para=20DocumentosSapl?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit e encoding flexível --- .../scripts/exporta_zope/exporta_zope.py | 59 +++++++++++++++---- 1 file changed, 46 insertions(+), 13 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 20c7d21e7..424d0c7f4 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -275,35 +275,62 @@ def find_sapl(app): id, meta_type = obj['id'], obj['meta_type'] if id.startswith('cm_') and meta_type == 'Folder': cm_zzz = br(app[id]) - sapl = br(cm_zzz.get('sapl', None)) - if sapl and 'sapl_documentos' in sapl and 'acl_users' in sapl: - return sapl + return find_sapl(cm_zzz) + elif id == 'sapl' and meta_type in ['SAPL', 'Folder']: + sapl = br(app['sapl']) + return sapl -def dump_propriedades(docs, path, salvar, encoding='iso-8859-1'): +def detectar_encoding(fonte): + desc = magic.from_buffer(fonte) + for termo, enc in [('ISO-8859', 'latin1'), ('UTF-8', 'utf-8')]: + if termo in desc: + return enc + return None + + +def autodecode(fonte): + if isinstance(fonte, str): + enc = detectar_encoding(fonte) + return fonte.decode(enc) if enc else fonte + else: + return fonte + + +def dump_propriedades(docs, path, salvar): props_sapl = br(docs['props_sapl']) ids = [p['id'] for p in props_sapl['_properties']] props = {id: props_sapl[id] for id in ids} - props = {id: p.decode(encoding) if isinstance(p, str) else p - for id, p in props.items()} + props = {id: autodecode(p) for id, p in props.items()} save_as_yaml(path, 'sapl_documentos/propriedades.yaml', props, salvar) def dump_usuarios(sapl, path, salvar): users = br(br(sapl['acl_users'])['data']) - users = {k: br(v) for k, v in users['data'].items()} + users = {autodecode(k): br(v) for k, v in users['data'].items()} save_as_yaml(path, 'usuarios.yaml', users, salvar) -def _dump_sapl(data_fs_path, destino, salvar): +def _dump_sapl(data_fs_path, documentos_fs_path, destino, salvar): assert Path(data_fs_path).exists() + assert Path(documentos_fs_path).exists() + app, close_db = get_app(data_fs_path) try: sapl = find_sapl(app) - # extrai folhas XSLT - dump_folder(br(sapl['XSLT']), destino, salvar) # extrai usuários com suas senhas e perfis dump_usuarios(sapl, destino, salvar) + # garantindo que a pasta XSLT não está aqui + assert 'XSLT' not in sapl + finally: + close_db() + + app, close_db = get_app(documentos_fs_path) + try: + sapl = find_sapl(app) + # extrai folhas XSLT + if 'XSLT' in sapl: + dump_folder(br(sapl['XSLT']), destino, salvar) # extrai documentos docs = br(sapl['sapl_documentos']) @@ -355,9 +382,15 @@ def build_salvar(repo): def dump_sapl(sigla): sigla = sigla[-3:] # ignora prefixo (por ex. 'sapl_cm_') - data_fs_path = DIR_DADOS_MIGRACAO.child('datafs', - 'Data_cm_{}.fs'.format(sigla)) + data_fs_path, documentos_fs_path = [ + DIR_DADOS_MIGRACAO.child( + 'datafs', '{}_cm_{}.fs'.format(prefixo, sigla)) + for prefixo in ('Data', 'DocumentosSapl')] + assert data_fs_path.exists(), 'Origem não existe: {}'.format(data_fs_path) + if not documentos_fs_path.exists(): + documentos_fs_path = data_fs_path + nome_banco_legado = 'sapl_cm_{}'.format(sigla) destino = DIR_DADOS_MIGRACAO.child('repos', nome_banco_legado) destino.mkdir(parents=True) @@ -372,7 +405,7 @@ def dump_sapl(sigla): salvar = build_salvar(repo) try: finalizado = False - _dump_sapl(data_fs_path, destino, salvar) + _dump_sapl(data_fs_path, documentos_fs_path, destino, salvar) finalizado = True finally: # grava mundaças From 695206b8a47f143ab7608e94109f90459b01a5d3 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Wed, 30 May 2018 11:05:12 -0300 Subject: [PATCH 03/20] =?UTF-8?q?Corrige=20dump=20de=20XSLT=20na=20exporta?= =?UTF-8?q?=C3=A7=C3=A3o=20do=20zope?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 424d0c7f4..7507ffb87 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -320,8 +320,6 @@ def _dump_sapl(data_fs_path, documentos_fs_path, destino, salvar): sapl = find_sapl(app) # extrai usuários com suas senhas e perfis dump_usuarios(sapl, destino, salvar) - # garantindo que a pasta XSLT não está aqui - assert 'XSLT' not in sapl finally: close_db() From 76824dcc7a1af590bea5418304e33e7e2879ad73 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Wed, 30 May 2018 11:32:50 -0300 Subject: [PATCH 04/20] =?UTF-8?q?Retira=20restri=C3=A7=C3=A3o=20de=20num?= =?UTF-8?q?=20de=20superusers=20ao=20migrar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_usuarios.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/sapl/legacy/migracao_usuarios.py b/sapl/legacy/migracao_usuarios.py index be0478c82..51b3348c8 100644 --- a/sapl/legacy/migracao_usuarios.py +++ b/sapl/legacy/migracao_usuarios.py @@ -95,16 +95,7 @@ def migrar_usuarios(dir_repo): usuario.groups.add(PERFIL_LEGADO_PARA_NOVO[perfil]) usuario.save() - # restringe e configura administradores - if len(admins) > 2: - admins = ( - # ususários com admin no nome - [u for u in admins if 'admin' in u.username] - # senão, o usuário saploper, apenas - or [u for u in admins if 'saploper' == u.username] - # senão, simplesmente até os dois primeiros da lista - or admins[:2] - ) + # configura administradores for admin in admins: admin.is_superuser = True admin.save() From da44d3c7c9e5c105fca962fa2b6c248d7fd152ae Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Wed, 30 May 2018 14:29:00 -0300 Subject: [PATCH 05/20] =?UTF-8?q?Autodecodifica=20nomes=20de=20usu=C3=A1ri?= =?UTF-8?q?os=20ao=20exportar=20zope?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 7507ffb87..a29a0b83b 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -17,12 +17,12 @@ from functools import partial import git import magic import yaml -import ZODB.DB -import ZODB.FileStorage from unipath import Path -from ZODB.broken import Broken +import ZODB.DB +import ZODB.FileStorage from variaveis_comuns import DIR_DADOS_MIGRACAO, TAG_ZOPE +from ZODB.broken import Broken EXTENSOES = { 'application/msword': '.doc', @@ -308,6 +308,8 @@ def dump_propriedades(docs, path, salvar): def dump_usuarios(sapl, path, salvar): users = br(br(sapl['acl_users'])['data']) users = {autodecode(k): br(v) for k, v in users['data'].items()} + for dados in users.values(): + dados['name'] = autodecode(dados['name']) save_as_yaml(path, 'usuarios.yaml', users, salvar) From 3de1e2b99d6dadf77661dffb87bf567ca5c7722c Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Sat, 2 Jun 2018 12:23:14 -0300 Subject: [PATCH 06/20] =?UTF-8?q?Pula=20pastas=20*=5Fold=20na=20exporta?= =?UTF-8?q?=C3=A7=C3=A3o=20do=20zope?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 3 +++ 1 file changed, 3 insertions(+) mode change 100755 => 100644 sapl/legacy/scripts/exporta_zope/exporta_zope.py diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py old mode 100755 new mode 100644 index a29a0b83b..75c21b033 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -186,6 +186,9 @@ def dump_folder(folder, path, salvar, enum=enumerate_folder): if not os.path.exists(path): os.makedirs(path) for id, obj, meta_type in enum(folder): + # pula pastas *_old (presentes em várias bases) + if id.endswith('_old') and meta_type in ['Folder', 'BTreeFolder2']: + continue dump = DUMP_FUNCTIONS.get(meta_type, '?') if dump == '?': nao_identificados[meta_type].append(path + '/' + id) From 37720a91ce1c6e3defab104a9914994185ae36b8 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Sat, 2 Jun 2018 14:08:44 -0300 Subject: [PATCH 07/20] =?UTF-8?q?Reconhece=20arquivos=20corrompidos=20na?= =?UTF-8?q?=20exporta=C3=A7=C3=A3o=20do=20zope?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scripts/exporta_zope/exporta_zope.py | 37 ++++++++++++------- 1 file changed, 23 insertions(+), 14 deletions(-) mode change 100644 => 100755 sapl/legacy/scripts/exporta_zope/exporta_zope.py diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py old mode 100644 new mode 100755 index 75c21b033..12819f512 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -17,12 +17,13 @@ from functools import partial import git import magic import yaml -from unipath import Path - import ZODB.DB import ZODB.FileStorage -from variaveis_comuns import DIR_DADOS_MIGRACAO, TAG_ZOPE +from unipath import Path from ZODB.broken import Broken +from ZODB.POSException import POSKeyError + +from variaveis_comuns import DIR_DADOS_MIGRACAO, TAG_ZOPE EXTENSOES = { 'application/msword': '.doc', @@ -95,6 +96,9 @@ def guess_extension(fullname, buffer): return '.DESCONHECIDO.{}'.format(mime.replace('/', '__')) +CONTEUDO_ARQUIVO_CORROMPIDO = 'ARQUIVO CORROMPIDO' + + def get_conteudo_file(doc): # A partir daqui usamos dict.pop('...') nos __Broken_state__ # para contornar um "vazamento" de memória que ocorre @@ -105,25 +109,30 @@ def get_conteudo_file(doc): # # Essa medida descarta quase todos os dados retornados # e só funciona na primeira passagem + try: + pdata = br(doc.pop('data')) + if isinstance(pdata, str): + # Retrocedemos se pdata ja eh uma str (necessario em Images) + doc['data'] = pdata + pdata = doc - pdata = br(doc.pop('data')) - if isinstance(pdata, str): - # Retrocedemos se pdata ja eh uma str (necessario em Images) - doc['data'] = pdata - pdata = doc - - output = cStringIO.StringIO() - while pdata: - output.write(pdata.pop('data')) - pdata = br(pdata.pop('next', None)) + output = cStringIO.StringIO() + while pdata: + output.write(pdata.pop('data')) + pdata = br(pdata.pop('next', None)) - return output.getvalue() + return output.getvalue() + except POSKeyError: + return CONTEUDO_ARQUIVO_CORROMPIDO def dump_file(doc, path, salvar, get_conteudo=get_conteudo_file): name = doc['__name__'] fullname = os.path.join(path, name) conteudo = get_conteudo(doc) + if conteudo == CONTEUDO_ARQUIVO_CORROMPIDO: + fullname = fullname + '.CORROMPIDO' + print('ATENÇÃO: arquivo corrompido: {}'.format(fullname)) if conteudo: # pula arquivos vazios salvar(fullname, conteudo) From e3317eebe7e25abd0c975c6e0a5765879617e274 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Mon, 11 Jun 2018 11:15:57 -0300 Subject: [PATCH 08/20] =?UTF-8?q?Ajusta=20nome=20de=20arquivo=20corrompido?= =?UTF-8?q?=20na=20exporta=C3=A7=C3=A3o=20do=20zope?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 12819f512..f1a4b791f 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -131,7 +131,7 @@ def dump_file(doc, path, salvar, get_conteudo=get_conteudo_file): fullname = os.path.join(path, name) conteudo = get_conteudo(doc) if conteudo == CONTEUDO_ARQUIVO_CORROMPIDO: - fullname = fullname + '.CORROMPIDO' + fullname = fullname + '_CORROMPIDO' print('ATENÇÃO: arquivo corrompido: {}'.format(fullname)) if conteudo: # pula arquivos vazios From 8d16524d858427d4d02df823d5ff58c037451b88 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Mon, 11 Jun 2018 11:20:38 -0300 Subject: [PATCH 09/20] =?UTF-8?q?Adiciona=20tag=20zope=20ao=20exportar=20z?= =?UTF-8?q?ope=20mesmo=20sem=20mudan=C3=A7a?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index f1a4b791f..d6af2c535 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -17,14 +17,14 @@ from functools import partial import git import magic import yaml +from unipath import Path + import ZODB.DB import ZODB.FileStorage -from unipath import Path +from variaveis_comuns import DIR_DADOS_MIGRACAO, TAG_ZOPE from ZODB.broken import Broken from ZODB.POSException import POSKeyError -from variaveis_comuns import DIR_DADOS_MIGRACAO, TAG_ZOPE - EXTENSOES = { 'application/msword': '.doc', 'application/pdf': '.pdf', @@ -427,8 +427,8 @@ def dump_sapl(sigla): # se de fato existe mudança status = 'completa' if finalizado else 'parcial' repo.index.commit(u'Exportação do zope {}'.format(status)) - if finalizado: - repo.git.execute('git tag -f'.split() + [TAG_ZOPE]) + if finalizado: + repo.git.execute('git tag -f'.split() + [TAG_ZOPE]) if __name__ == "__main__": From a3d5321e097642c3c288c1fc27fb478d6262c7ed Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 12 Jun 2018 13:37:37 -0300 Subject: [PATCH 10/20] Ajusta sessao_plenaria_presenca.dat_sessao antes de migrar --- sapl/legacy/migracao_dados.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 88758c8b1..87646cd07 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -626,12 +626,13 @@ def uniformiza_banco(): ''') update_specs = ''' -vinculo_norma_juridica| ind_excluido = '' | trim(ind_excluido) = '0' -unidade_tramitacao | cod_parlamentar = NULL | cod_parlamentar = 0 -parlamentar | cod_nivel_instrucao = NULL | cod_nivel_instrucao = 0 -parlamentar | tip_situacao_militar = NULL | tip_situacao_militar = 0 -mandato | tip_afastamento = NULL | tip_afastamento = 0 -relatoria | tip_fim_relatoria = NULL | tip_fim_relatoria = 0 +vinculo_norma_juridica | ind_excluido = '' | trim(ind_excluido) = '0' +unidade_tramitacao | cod_parlamentar = NULL | cod_parlamentar = 0 +parlamentar | cod_nivel_instrucao = NULL | cod_nivel_instrucao = 0 +parlamentar | tip_situacao_militar = NULL | tip_situacao_militar = 0 +mandato | tip_afastamento = NULL | tip_afastamento = 0 +relatoria | tip_fim_relatoria = NULL | tip_fim_relatoria = 0 +sessao_plenaria_presenca | dat_sessao = NULL | dat_sessao = 0 '''.strip().splitlines() for spec in update_specs: @@ -794,7 +795,7 @@ def roda_comando_shell(cmd): assert res == 0, 'O comando falhou: {}'.format(cmd) -def migrar_dados(interativo=True): +def migrar_dados(interativo=False): # restaura dump arq_dump = Path(DIR_DADOS_MIGRACAO.child( From f5246f1d4c594d4b522db14851b60dbfa19383b9 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Wed, 13 Jun 2018 09:16:05 -0300 Subject: [PATCH 11/20] =?UTF-8?q?Retira=20c=C3=B3digo=20desnecess=C3=A1rio?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao.py | 2 +- sapl/legacy/migracao_dados.py | 13 +------------ 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/sapl/legacy/migracao.py b/sapl/legacy/migracao.py index c2d863b36..746dff6f8 100644 --- a/sapl/legacy/migracao.py +++ b/sapl/legacy/migracao.py @@ -26,7 +26,7 @@ def migrar(interativo=False): 'Antes de migrar ' 'é necessário fazer a exportação de documentos do zope') management.call_command('migrate') - migrar_dados(interativo=interativo) + migrar_dados() migrar_usuarios(REPO.working_dir) migrar_documentos(REPO) gravar_marco() diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 87646cd07..fb5020437 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -795,7 +795,7 @@ def roda_comando_shell(cmd): assert res == 0, 'O comando falhou: {}'.format(cmd) -def migrar_dados(interativo=False): +def migrar_dados(): # restaura dump arq_dump = Path(DIR_DADOS_MIGRACAO.child( @@ -814,17 +814,6 @@ def migrar_dados(interativo=False): uniformiza_banco() # excluindo database antigo. - if interativo: - info('Todos os dados do banco serão excluidos. ' - 'Recomendamos que faça backup do banco sapl ' - 'antes de continuar.') - info('Deseja continuar? [s/n]') - resposta = input() - if resposta.lower() in ['s', 'sim', 'y', 'yes']: - pass - else: - info('Migração cancelada.') - return 0 info('Excluindo entradas antigas do banco destino.') call([PROJECT_DIR.child('manage.py'), 'flush', '--database=default', '--no-input'], stdout=PIPE) From 5a27d9127b19addb637070501bb8209f8c0a0c4e Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Wed, 13 Jun 2018 09:38:20 -0300 Subject: [PATCH 12/20] =?UTF-8?q?Loga=20ocorrencias=20em=20toda=20a=20migr?= =?UTF-8?q?a=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_dados.py | 56 +++++++++++++++++------------------ 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index fb5020437..c6ca892f0 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -796,36 +796,36 @@ def roda_comando_shell(cmd): def migrar_dados(): - - # restaura dump - arq_dump = Path(DIR_DADOS_MIGRACAO.child( - 'dumps_mysql', '{}.sql'.format(NOME_BANCO_LEGADO))) - assert arq_dump.exists(), 'Dump do mysql faltando: {}'.format(arq_dump) - info('Restaurando dump mysql de [{}]'.format(arq_dump)) - normaliza_dump_mysql(arq_dump) - roda_comando_shell('mysql -uroot < {}'.format(arq_dump)) - - # executa ajustes pré-migração, se existirem - arq_ajustes_pre_migracao = DIR_DADOS_MIGRACAO.child( - 'ajustes_pre_migracao', '{}.sql'.format(sigla_casa)) - if arq_ajustes_pre_migracao.exists(): - exec_legado(arq_ajustes_pre_migracao.read_file()) - - uniformiza_banco() - - # excluindo database antigo. - info('Excluindo entradas antigas do banco destino.') - call([PROJECT_DIR.child('manage.py'), 'flush', - '--database=default', '--no-input'], stdout=PIPE) - - # apaga tipos de autor padrão (criados no flush acima) - TipoAutor.objects.all().delete() - - fill_vinculo_norma_juridica() - fill_dados_basicos() - info('Começando migração: ...') try: ocorrencias.clear() + + # restaura dump + arq_dump = Path(DIR_DADOS_MIGRACAO.child( + 'dumps_mysql', '{}.sql'.format(NOME_BANCO_LEGADO))) + assert arq_dump.exists(), 'Dump do mysql faltando: {}'.format(arq_dump) + info('Restaurando dump mysql de [{}]'.format(arq_dump)) + normaliza_dump_mysql(arq_dump) + roda_comando_shell('mysql -uroot < {}'.format(arq_dump)) + + # executa ajustes pré-migração, se existirem + arq_ajustes_pre_migracao = DIR_DADOS_MIGRACAO.child( + 'ajustes_pre_migracao', '{}.sql'.format(sigla_casa)) + if arq_ajustes_pre_migracao.exists(): + exec_legado(arq_ajustes_pre_migracao.read_file()) + + uniformiza_banco() + + # excluindo database antigo. + info('Excluindo entradas antigas do banco destino.') + call([PROJECT_DIR.child('manage.py'), 'flush', + '--database=default', '--no-input'], stdout=PIPE) + + # apaga tipos de autor padrão (criados no flush acima) + TipoAutor.objects.all().delete() + + fill_vinculo_norma_juridica() + fill_dados_basicos() + info('Começando migração: ...') migrar_todos_os_models() except Exception as e: ocorrencias['traceback'] = str(traceback.format_exc()) From c7f69e09ca341606feda30ac663c7f059931ba6f Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Wed, 13 Jun 2018 19:26:00 -0300 Subject: [PATCH 13/20] =?UTF-8?q?Pula=20diret=C3=B3rios=20corrompidos=20ao?= =?UTF-8?q?=20exportar=20docs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index d6af2c535..7494dca17 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -146,8 +146,16 @@ def get_conteudo_dtml_method(doc): def enumerate_by_key_list(folder, key_list, type_key): for entry in folder.get(key_list, []): id, meta_type = entry['id'], entry[type_key] - obj = br(folder.get(id, None)) - yield id, obj, meta_type + try: + obj = br(folder.get(id, None)) + except POSKeyError: + print('#' * 80) + print('#' * 80) + print('ATENÇÃO: DIRETÓRIO corrompido: {}'.format(id)) + print('#' * 80) + print('#' * 80) + else: + yield id, obj, meta_type enumerate_folder = partial(enumerate_by_key_list, From f81b678bac859a5cd9625146ef4c8328ecf23db9 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Wed, 13 Jun 2018 19:27:22 -0300 Subject: [PATCH 14/20] =?UTF-8?q?Suspende=20compacta=C3=A7=C3=A3o=20de=20m?= =?UTF-8?q?edia=20ao=20migrar?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sapl/legacy/migracao.py b/sapl/legacy/migracao.py index 746dff6f8..9ed09360f 100644 --- a/sapl/legacy/migracao.py +++ b/sapl/legacy/migracao.py @@ -30,7 +30,7 @@ def migrar(interativo=False): migrar_usuarios(REPO.working_dir) migrar_documentos(REPO) gravar_marco() - compactar_media() + # compactar_media() def compactar_media(): From 6a3f2c77a4df9de4a09bc3c0ee7e2ed84540ca03 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Wed, 13 Jun 2018 21:33:32 -0300 Subject: [PATCH 15/20] =?UTF-8?q?Migra=20dados=20de=20reuni=C3=A3o=20de=20?= =?UTF-8?q?comiss=C3=A3o=20(do=20sapl=203.0)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/comissoes/legacy.yaml | 7 ++++ .../migrations/0015_auto_20180613_2023.py | 25 ++++++++++++++ .../migrations/0016_auto_20180613_2121.py | 21 ++++++++++++ sapl/comissoes/models.py | 11 ++++-- sapl/comissoes/views.py | 34 +++++++++++-------- sapl/legacy/migracao_dados.py | 22 ++++++++++-- sapl/legacy/models.py | 14 ++++++++ 7 files changed, 115 insertions(+), 19 deletions(-) create mode 100644 sapl/comissoes/migrations/0015_auto_20180613_2023.py create mode 100644 sapl/comissoes/migrations/0016_auto_20180613_2121.py diff --git a/sapl/comissoes/legacy.yaml b/sapl/comissoes/legacy.yaml index 66ed8b060..8e1040510 100644 --- a/sapl/comissoes/legacy.yaml +++ b/sapl/comissoes/legacy.yaml @@ -42,3 +42,10 @@ Participacao (ComposicaoComissao): observacao: obs_composicao parlamentar: cod_parlamentar titular: ind_titular + +Reuniao (ReuniaoComissao): + comissao: cod_comissao + numero: num_reuniao + data: dat_inicio_reuniao + observacao: txt_observacao + diff --git a/sapl/comissoes/migrations/0015_auto_20180613_2023.py b/sapl/comissoes/migrations/0015_auto_20180613_2023.py new file mode 100644 index 000000000..b6602cfc3 --- /dev/null +++ b/sapl/comissoes/migrations/0015_auto_20180613_2023.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.13 on 2018-06-13 23:23 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('comissoes', '0014_auto_20180503_1055'), + ] + + operations = [ + migrations.AlterField( + model_name='reuniao', + name='hora_fim', + field=models.TimeField(null=True, verbose_name='Horário de Término (hh:mm)'), + ), + migrations.AlterField( + model_name='reuniao', + name='hora_inicio', + field=models.TimeField(null=True, verbose_name='Horário de Início (hh:mm)'), + ), + ] diff --git a/sapl/comissoes/migrations/0016_auto_20180613_2121.py b/sapl/comissoes/migrations/0016_auto_20180613_2121.py new file mode 100644 index 000000000..77c1894f8 --- /dev/null +++ b/sapl/comissoes/migrations/0016_auto_20180613_2121.py @@ -0,0 +1,21 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.9.13 on 2018-06-14 00:21 +from __future__ import unicode_literals + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('comissoes', '0015_auto_20180613_2023'), + ] + + operations = [ + migrations.AlterField( + model_name='reuniao', + name='periodo', + field=models.ForeignKey(null=True, on_delete=django.db.models.deletion.PROTECT, to='comissoes.Periodo', verbose_name='Periodo da Composicão da Comissão'), + ), + ] diff --git a/sapl/comissoes/models.py b/sapl/comissoes/models.py index 0c3240fc6..a2477ef1d 100644 --- a/sapl/comissoes/models.py +++ b/sapl/comissoes/models.py @@ -184,13 +184,16 @@ class Participacao(models.Model): # ComposicaoComissao def get_comissao_media_path(instance, subpath, filename): return './sapl/comissao/%s/%s/%s' % (instance.numero, subpath, filename) + def pauta_upload_path(instance, filename): return texto_upload_path(instance, filename, subpath='pauta', pk_first=True) + def ata_upload_path(instance, filename): return texto_upload_path(instance, filename, subpath='ata', pk_first=True) + def anexo_upload_path(instance, filename): return texto_upload_path(instance, filename, subpath='anexo', pk_first=True) @@ -198,6 +201,7 @@ def anexo_upload_path(instance, filename): class Reuniao(models.Model): periodo = models. ForeignKey( Periodo, + null=True, on_delete=models.PROTECT, verbose_name=_('Periodo da Composicão da Comissão')) comissao = models.ForeignKey( @@ -211,8 +215,10 @@ class Reuniao(models.Model): max_length=150, blank=True, verbose_name=_('Tema da Reunião')) data = models.DateField(verbose_name=_('Data')) hora_inicio = models.TimeField( + null=True, verbose_name=_('Horário de Início (hh:mm)')) hora_fim = models.TimeField( + null=True, verbose_name=_('Horário de Término (hh:mm)')) local_reuniao = models.CharField( max_length=100, blank=True, verbose_name=_('Local da Reunião')) @@ -287,12 +293,13 @@ class Reuniao(models.Model): @reversion.register() class DocumentoAcessorio(models.Model): - reuniao = models.ForeignKey(Reuniao, + reuniao = models.ForeignKey(Reuniao, related_name='documentoacessorio_set', on_delete=models.PROTECT) nome = models.CharField(max_length=50, verbose_name=_('Nome')) - data = models.DateField(blank=True, null=True, default=None, verbose_name=_('Data')) + data = models.DateField(blank=True, null=True, + default=None, verbose_name=_('Data')) autor = models.CharField( max_length=100, verbose_name=_('Autor')) ementa = models.TextField(blank=True, verbose_name=_('Ementa')) diff --git a/sapl/comissoes/views.py b/sapl/comissoes/views.py index c5870389e..78e639168 100644 --- a/sapl/comissoes/views.py +++ b/sapl/comissoes/views.py @@ -8,20 +8,20 @@ from django.views.generic.base import RedirectView from django.views.generic.detail import DetailView from django.views.generic.edit import FormMixin - from sapl.base.models import AppConfig as AppsAppConfig -from sapl.crud.base import (RP_DETAIL, RP_LIST, Crud, - CrudAux, MasterDetailCrud, - PermissionRequiredForAppCrudMixin) -from sapl.comissoes.forms import (ComissaoForm, ComposicaoForm, DocumentoAcessorioCreateForm, - DocumentoAcessorioEditForm, ParticipacaoCreateForm, - ParticipacaoEditForm, ReuniaoForm, PeriodoForm) +from sapl.comissoes.apps import AppConfig +from sapl.comissoes.forms import (ComissaoForm, ComposicaoForm, + DocumentoAcessorioCreateForm, + DocumentoAcessorioEditForm, + ParticipacaoCreateForm, ParticipacaoEditForm, + PeriodoForm, ReuniaoForm) +from sapl.crud.base import (RP_DETAIL, RP_LIST, Crud, CrudAux, + MasterDetailCrud, + PermissionRequiredForAppCrudMixin) from sapl.materia.models import MateriaLegislativa, Tramitacao - from .models import (CargoComissao, Comissao, Composicao, DocumentoAcessorio, - Participacao, Periodo, TipoComissao, Reuniao) -from sapl.comissoes.apps import AppConfig + Participacao, Periodo, Reuniao, TipoComissao) def pegar_url_composicao(pk): @@ -30,6 +30,7 @@ def pegar_url_composicao(pk): url = reverse('sapl.comissoes:composicao_detail', kwargs={'pk': comp_pk}) return url + def pegar_url_reuniao(pk): documentoacessorio = DocumentoAcessorio.objects.get(id=pk) r_pk = documentoacessorio.reuniao.pk @@ -42,6 +43,7 @@ TipoComissaoCrud = CrudAux.build( TipoComissao, 'tipo_comissao', list_field_names=[ 'sigla', 'nome', 'natureza', 'dispositivo_regimental']) + class PeriodoComposicaoCrud(CrudAux): model = Periodo @@ -77,6 +79,7 @@ class ParticipacaoCrud(MasterDetailCrud): form_class = ParticipacaoEditForm class DeleteView(MasterDetailCrud.DeleteView): + def get_success_url(self): composicao_comissao_pk = self.object.composicao.comissao.pk composicao_pk = self.object.composicao.pk @@ -93,12 +96,11 @@ class ComposicaoCrud(MasterDetailCrud): class CreateView(MasterDetailCrud.CreateView): form_class = ComposicaoForm - + def get_initial(self): comissao = Comissao.objects.get(id=self.kwargs['pk']) return {'comissao': comissao} - class ListView(MasterDetailCrud.ListView): template_name = "comissoes/composicao_list.html" paginate_by = None @@ -180,6 +182,7 @@ class MateriasTramitacaoListView(ListView): context['object'] = Comissao.objects.get(id=self.kwargs['pk']) return context + class ReuniaoCrud(MasterDetailCrud): model = Reuniao parent_field = 'comissao' @@ -187,7 +190,7 @@ class ReuniaoCrud(MasterDetailCrud): public = [RP_LIST, RP_DETAIL, ] class BaseMixin(MasterDetailCrud.BaseMixin): - list_field_names = [ 'nome', 'tema', 'data'] + list_field_names = ['data', 'nome', 'tema'] class ListView(MasterDetailCrud.ListView): paginate_by = 10 @@ -228,9 +231,9 @@ class ReuniaoCrud(MasterDetailCrud): form_class = ReuniaoForm def get_initial(self): - comissao = Comissao.objects.get(id=self.kwargs['pk']) + comissao = Comissao.objects.get(id=self.kwargs['pk']) - return {'comissao': comissao} + return {'comissao': comissao} class DocumentoAcessorioCrud(MasterDetailCrud): @@ -256,6 +259,7 @@ class DocumentoAcessorioCrud(MasterDetailCrud): form_class = DocumentoAcessorioEditForm class DeleteView(MasterDetailCrud.DeleteView): + def delete(self, *args, **kwargs): obj = self.get_object() obj.delete() diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index c6ca892f0..96bb0905f 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -29,7 +29,7 @@ from unipath import Path from sapl.base.models import AppConfig as AppConf from sapl.base.models import Autor, TipoAutor, cria_models_tipo_autor -from sapl.comissoes.models import Comissao, Composicao, Participacao +from sapl.comissoes.models import Comissao, Composicao, Participacao, Reuniao from sapl.legacy import scripts from sapl.legacy.models import NormaJuridica as OldNormaJuridica from sapl.legacy.models import TipoNumeracaoProtocolo @@ -58,6 +58,8 @@ from .timezonesbrasil import get_timezone appconfs = [apps.get_app_config(n) for n in [ 'parlamentares', 'comissoes', + # base precisa vir depois dos apps parlamentares e comissoes + # pois Autor os referencia 'base', 'materia', 'norma', @@ -99,7 +101,7 @@ def get_renames(): model_name, old_name = match.groups() else: old_name = None - model = getattr(app.models_module, model_name) + model = app.get_model(model_name) if old_name: model_renames[model] = old_name field_renames[model] = renames @@ -212,6 +214,10 @@ class ForeignKeyFaltando(ObjectDoesNotExist): 'Uma FK aponta para um registro inexistente' def __init__(self, field, valor, old): + if (field.related_model.__name__ == 'Comissao' + and old.__class__.__name__ == 'ReuniaoComissao' + and valor == 1): + __import__('pdb').set_trace() self.field = field self.valor = valor self.old = old @@ -1237,6 +1243,17 @@ def adjust_tiporesultadovotacao(new, old): {'pk': new.pk, 'nome': new.nome}) +def str_to_time(fonte): + if not fonte.strip(): + return None + tempo = datetime.datetime.strptime(fonte, '%H:%M') + return tempo.time() if tempo else None + + +def adjust_reuniao_comissao(new, old): + new.hora_inicio = str_to_time(old.hr_inicio_reuniao) + + def remove_style(conteudo): if 'style' not in conteudo: return conteudo # atalho que acelera muito os casos sem style @@ -1274,6 +1291,7 @@ AJUSTE_ANTES_SALVAR = { Tramitacao: adjust_tramitacao, TipoResultadoVotacao: adjust_tiporesultadovotacao, ExpedienteSessao: adjust_expediente_sessao, + Reuniao: adjust_reuniao_comissao, } AJUSTE_DEPOIS_SALVAR = { diff --git a/sapl/legacy/models.py b/sapl/legacy/models.py index 7d341624d..cd6d433df 100644 --- a/sapl/legacy/models.py +++ b/sapl/legacy/models.py @@ -779,6 +779,20 @@ class Relatoria(models.Model): db_table = 'relatoria' +class ReuniaoComissao(models.Model): + cod_reuniao = models.AutoField(primary_key=True) + cod_comissao = models.IntegerField() + num_reuniao = models.IntegerField() + dat_inicio_reuniao = models.DateField() + hr_inicio_reuniao = models.CharField(max_length=5, blank=True, null=True) + txt_observacao = models.TextField(blank=True, null=True) + ind_excluido = models.IntegerField() + + class Meta: + managed = False + db_table = 'reuniao_comissao' + + class SessaoLegislativa(models.Model): cod_sessao_leg = models.AutoField(primary_key=True) num_legislatura = models.IntegerField() From e9f38dd062232c5ddc876ff806f8249e83b5b678 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 14 Jun 2018 10:58:55 -0300 Subject: [PATCH 16/20] Corrige mapeamento dos campos legados de models abstratos --- sapl/legacy/migracao_dados.py | 22 +++++++++------------- sapl/sessao/legacy.yaml | 12 ++++++------ 2 files changed, 15 insertions(+), 19 deletions(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 96bb0905f..24ee536e0 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -87,15 +87,24 @@ for a1, s1 in name_sets: # RENAMES ################################################################### MODEL_RENAME_PATTERN = re.compile('(.+) \((.+)\)') +MODEL_RENAME_INCLUDE_PATTERN = re.compile('<(.+)>') def get_renames(): field_renames = {} model_renames = {} + includes = {} for app in appconfs: app_rename_data = yaml.load( pkg_resources.resource_string(app.module.__name__, 'legacy.yaml')) for model_name, renames in app_rename_data.items(): + # armazena ou substitui includes + if MODEL_RENAME_INCLUDE_PATTERN.match(model_name): + includes[model_name] = renames + continue + elif isinstance(renames, str): + renames = includes[renames] + # detecta mudança de nome match = MODEL_RENAME_PATTERN.match(model_name) if match: model_name, old_name = match.groups() @@ -106,19 +115,6 @@ def get_renames(): model_renames[model] = old_name field_renames[model] = renames - # collect renames from parent classes - for model, renames in field_renames.items(): - if any(parent in field_renames for parent in model.__mro__[1:]): - renames = {} - for parent in reversed(model.__mro__): - if parent in field_renames: - renames.update(field_renames[parent]) - field_renames[model] = renames - - # remove abstract classes - field_renames = {m: r for m, r in field_renames.items() - if not m._meta.abstract} - return field_renames, model_renames diff --git a/sapl/sessao/legacy.yaml b/sapl/sessao/legacy.yaml index e7b55095a..6df4dd90d 100644 --- a/sapl/sessao/legacy.yaml +++ b/sapl/sessao/legacy.yaml @@ -15,7 +15,7 @@ SessaoPlenaria: url_audio: url_audio url_video: url_video -AbstractOrdemDia: +: data_ordem: dat_ordem materia: cod_materia numero_ordem: num_ordem @@ -24,7 +24,7 @@ AbstractOrdemDia: sessao_plenaria: cod_sessao_plen tipo_votacao: tip_votacao -ExpedienteMateria: {} +ExpedienteMateria: TipoExpediente: nome: nom_expediente @@ -39,17 +39,17 @@ IntegranteMesa (MesaSessaoPlenaria): parlamentar: cod_parlamentar sessao_plenaria: cod_sessao_plen -AbstractOrador: +: numero_ordem: num_ordem parlamentar: cod_parlamentar sessao_plenaria: cod_sessao_plen url_discurso: url_discurso -Orador (Oradores): {} +Orador (Oradores): -OradorExpediente (OradoresExpediente): {} +OradorExpediente (OradoresExpediente): -OrdemDia: {} +OrdemDia: PresencaOrdemDia (OrdemDiaPresenca): parlamentar: cod_parlamentar From 392dd92df6b2966b476d1c365036c228bcee62f4 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 14 Jun 2018 11:05:46 -0300 Subject: [PATCH 17/20] =?UTF-8?q?Congela=20ocorr=C3=AAncias=20ao=20final?= =?UTF-8?q?=20da=20migra=C3=A7=C3=A3o=20de=20dados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_dados.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index 24ee536e0..da0ca270a 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -800,6 +800,7 @@ def roda_comando_shell(cmd): def migrar_dados(): try: ocorrencias.clear() + ocorrencias.default_factory = list # restaura dump arq_dump = Path(DIR_DADOS_MIGRACAO.child( @@ -833,7 +834,8 @@ def migrar_dados(): ocorrencias['traceback'] = str(traceback.format_exc()) raise e finally: - # grava ocorrências + # congela e grava ocorrências + ocorrencias.default_factory = None arq_ocorrencias = Path(REPO.working_dir, 'ocorrencias.yaml') with open(arq_ocorrencias, 'w') as arq: pyaml.dump(ocorrencias, arq, vspacing=1) From 6b9d6d11da17537f58015ebf576902f9e8d9fb53 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 14 Jun 2018 14:33:31 -0300 Subject: [PATCH 18/20] =?UTF-8?q?Vincula=20docs=20de=20reuni=C3=B5es=20aos?= =?UTF-8?q?=20dados?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/migracao_documentos.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/sapl/legacy/migracao_documentos.py b/sapl/legacy/migracao_documentos.py index 2bb17c53e..a759e3667 100644 --- a/sapl/legacy/migracao_documentos.py +++ b/sapl/legacy/migracao_documentos.py @@ -8,6 +8,7 @@ from django.db import transaction from image_cropping.fields import ImageCropField from sapl.base.models import CasaLegislativa +from sapl.comissoes.models import Reuniao from sapl.legacy.migracao_dados import exec_legado from sapl.materia.models import (DocumentoAcessorio, MateriaLegislativa, Proposicao) @@ -26,6 +27,8 @@ DOCS = { MateriaLegislativa: [('texto_original', 'materia/{}_texto_integral')], DocumentoAcessorio: [('arquivo', 'materia/{}')], NormaJuridica: [('texto_integral', 'norma_juridica/{}_texto_integral')], + Reuniao: [('upload_pauta', 'reuniao_comissao/{}_pauta'), + ('upload_ata', 'reuniao_comissao/{}_ata')], SessaoPlenaria: [('upload_pauta', 'pauta_sessao/{}_pauta_sessao'), ('upload_ata', 'ata_sessao/{}_ata_sessao'), ('upload_anexo', 'anexo_sessao/{}_texto_anexado')], @@ -40,9 +43,12 @@ DOCS = {model: [(campo, join('sapl_documentos', origem)) for model, campos in DOCS.items()} -def mover_documento(repo, origem, destino): +def mover_documento(repo, origem, destino, ignora_origem_ausente=False): origem, destino = [join(repo.working_dir, c) if not os.path.isabs(c) else c for c in (origem, destino)] + if ignora_origem_ausente and not os.path.exists(origem): + print('Origem ignorada ao mover documento: {}'.format(origem)) + return os.makedirs(os.path.dirname(destino), exist_ok=True) repo.git.mv(origem, destino) @@ -52,7 +58,8 @@ def migrar_logotipo(repo, casa, propriedades): [(campo, origem)] = DOCS[CasaLegislativa] # a extensão do logo pode ter sido ajustada pelo tipo real do arquivo nome_nas_propriedades = os.path.splitext(propriedades['id_logo'])[0] - arquivos = glob(join(repo.working_dir, origem.format(nome_nas_propriedades))) + arquivos = glob( + join(repo.working_dir, origem.format(nome_nas_propriedades))) if arquivos: assert len(arquivos) == 1, 'Há mais de um logotipo para a casa' [logo] = arquivos @@ -145,7 +152,8 @@ def migrar_documentos(repo): # Isto significa que para rodar novamente esta função é preciso # restaurar o repo ao estado anterior - mover_documento(repo, 'XSLT', 'sapl/public/XSLT') + mover_documento(repo, 'XSLT', 'sapl/public/XSLT', + ignora_origem_ausente=True) migrar_propriedades_da_casa(repo) From f257c13dd145699b9477f14ffd43d59b36234dfb Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 14 Jun 2018 20:33:31 -0300 Subject: [PATCH 19/20] =?UTF-8?q?Corrige=20busca=20do=20sapl=20dentro=20do?= =?UTF-8?q?=20Data.fs=20na=20exporta=C3=A7=C3=A3o?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 7494dca17..270c90583 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -17,14 +17,14 @@ from functools import partial import git import magic import yaml -from unipath import Path - import ZODB.DB import ZODB.FileStorage -from variaveis_comuns import DIR_DADOS_MIGRACAO, TAG_ZOPE +from unipath import Path from ZODB.broken import Broken from ZODB.POSException import POSKeyError +from variaveis_comuns import DIR_DADOS_MIGRACAO, TAG_ZOPE + EXTENSOES = { 'application/msword': '.doc', 'application/pdf': '.pdf', @@ -291,8 +291,10 @@ def get_app(data_fs_path): def find_sapl(app): - for obj in app['_objects']: - id, meta_type = obj['id'], obj['meta_type'] + ids_meta_types = [(obj['id'], obj['meta_type']) for obj in app['_objects']] + # estar ordenado é muito importante para que a busca dê prioridade + # a um id "cm_zzz" antes do id "sapl" + for id, meta_type in sorted(ids_meta_types): if id.startswith('cm_') and meta_type == 'Folder': cm_zzz = br(app[id]) return find_sapl(cm_zzz) From bec2532f9029fb8d32169bc613d681a4a7449238 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 14 Jun 2018 20:34:41 -0300 Subject: [PATCH 20/20] =?UTF-8?q?Migra=20reuni=C3=B5es=20apenas=20quando?= =?UTF-8?q?=20existirem=20no=20legado?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Só acontece em bases de sapl 3.0 --- sapl/legacy/migracao_dados.py | 4 ++++ sapl/legacy/migracao_documentos.py | 26 ++++++++++---------------- 2 files changed, 14 insertions(+), 16 deletions(-) diff --git a/sapl/legacy/migracao_dados.py b/sapl/legacy/migracao_dados.py index da0ca270a..41c7b9934 100644 --- a/sapl/legacy/migracao_dados.py +++ b/sapl/legacy/migracao_dados.py @@ -857,6 +857,10 @@ def move_para_depois_de(lista, movido, referencias): def get_models_a_migrar(): models = [model for app in appconfs for model in app.models.values() if model in field_renames] + # retira reuniões quando não existe na base legada + # (só existe no sapl 3.0) + if 'reuniao_comissao' not in list(exec_legado('show tables')): + models.remove(Reuniao) # Devido à referência TipoProposicao.tipo_conteudo_related # a migração de TipoProposicao precisa ser feita # após TipoMateriaLegislativa e TipoDocumento diff --git a/sapl/legacy/migracao_documentos.py b/sapl/legacy/migracao_documentos.py index a759e3667..fd692aa47 100644 --- a/sapl/legacy/migracao_documentos.py +++ b/sapl/legacy/migracao_documentos.py @@ -22,13 +22,10 @@ from sapl.sessao.models import SessaoPlenaria DOCS = { - CasaLegislativa: [('logotipo', 'props_sapl/{}.*')], Parlamentar: [('fotografia', 'parlamentar/fotos/{}_foto_parlamentar')], MateriaLegislativa: [('texto_original', 'materia/{}_texto_integral')], DocumentoAcessorio: [('arquivo', 'materia/{}')], NormaJuridica: [('texto_integral', 'norma_juridica/{}_texto_integral')], - Reuniao: [('upload_pauta', 'reuniao_comissao/{}_pauta'), - ('upload_ata', 'reuniao_comissao/{}_ata')], SessaoPlenaria: [('upload_pauta', 'pauta_sessao/{}_pauta_sessao'), ('upload_ata', 'ata_sessao/{}_ata_sessao'), ('upload_anexo', 'anexo_sessao/{}_texto_anexado')], @@ -38,8 +35,14 @@ DOCS = { DocumentoAcessorioAdministrativo: [('arquivo', 'administrativo/{}')], } +# acrescenta reuniões (que só existem no sapl 3.0) +if 'reuniao_comissao' in set(exec_legado('show tables')): + DOCS[Reuniao] = [('upload_pauta', 'reuniao_comissao/{}_pauta'), + ('upload_ata', 'reuniao_comissao/{}_ata')], + + DOCS = {model: [(campo, join('sapl_documentos', origem)) - for campo, origem, in campos] + for campo, origem in campos] for model, campos in DOCS.items()} @@ -55,7 +58,7 @@ def mover_documento(repo, origem, destino, ignora_origem_ausente=False): def migrar_logotipo(repo, casa, propriedades): print('.... Migrando logotipo da casa ....') - [(campo, origem)] = DOCS[CasaLegislativa] + campo, origem = 'logotipo', 'sapl_documentos/props_sapl/{}.*' # a extensão do logo pode ter sido ajustada pelo tipo real do arquivo nome_nas_propriedades = os.path.splitext(propriedades['id_logo'])[0] arquivos = glob( @@ -102,10 +105,10 @@ def migrar_propriedades_da_casa(repo): migrar_logotipo(repo, casa, propriedades) casa.save() - repo.git.rm(caminho) def migrar_docs_por_ids(repo, model): + for campo, base_origem in DOCS[model]: print('#### Migrando {} de {} ####'.format(campo, model.__name__)) @@ -161,16 +164,7 @@ def migrar_documentos(repo): # (necessário para o cropping de imagem) repo.git.execute('git annex get sapl_documentos/parlamentar'.split()) - for model in [ - Parlamentar, - MateriaLegislativa, - DocumentoAcessorio, - NormaJuridica, - SessaoPlenaria, - Proposicao, - DocumentoAdministrativo, - DocumentoAcessorioAdministrativo, - ]: + for model in DOCS: migrar_docs_por_ids(repo, model) sobrando = [join(dir, file)