From ce1c75860062da34a70753a1e845530fb218c494 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 10 Nov 2017 11:58:53 -0200 Subject: [PATCH 01/12] =?UTF-8?q?Adiciona=20script=20de=20exporta=C3=A7?= =?UTF-8?q?=C3=A3o=20de=20docs=20do=20zope?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/.gitignore | 2 + .../scripts/exporta_zope/exporta_zope.py | 151 ++++++++++++++++++ .../scripts/exporta_zope/requirements.txt | 2 + 3 files changed, 155 insertions(+) create mode 100644 sapl/legacy/scripts/exporta_zope/.gitignore create mode 100644 sapl/legacy/scripts/exporta_zope/exporta_zope.py create mode 100644 sapl/legacy/scripts/exporta_zope/requirements.txt diff --git a/sapl/legacy/scripts/exporta_zope/.gitignore b/sapl/legacy/scripts/exporta_zope/.gitignore new file mode 100644 index 000000000..98561c589 --- /dev/null +++ b/sapl/legacy/scripts/exporta_zope/.gitignore @@ -0,0 +1,2 @@ +Data*.fs* +sapl_documentos diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py new file mode 100644 index 000000000..618c9ec5c --- /dev/null +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -0,0 +1,151 @@ +# -*- coding: utf-8 -*- + +# IMPORTANTE: +# Esse script precisa rodar em python 2 +# e depende apenas do descrito no arquivo requiments.txt + +import os.path +from collections import defaultdict +from functools import partial + +import ZODB.DB +import ZODB.FileStorage +from ZODB.broken import Broken + +EXTENSOES = { + 'application/msword': '.doc', + 'application/pdf': '.pdf', + 'application/vnd.oasis.opendocument.text': '.odt', + 'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx', # noqa + 'application/xml': '.xml', + 'text/xml': '.xml', + 'application/zip': '.zip', + 'image/jpeg': '.jpeg', + 'image/png': '.png', + 'image/gif': '.gif', + 'text/html': '.html', + 'text/rtf': '.rtf', + 'text/x-python': '.py', + 'text/plain': '.txt', + 'SDE-Document': 'xml', + + # TODO rever... + 'text/richtext': '.rtf', + + # sem extensao + 'application/octet-stream': '', # binario + 'inode/x-empty': '', # vazio + 'text/x-unknown-content-type': '', +} + + +def br(obj): + if isinstance(obj, Broken): + return obj.__Broken_state__ + else: + return obj + + +def dump_file(doc, path): + name = doc['__name__'] + extension = EXTENSOES[doc['content_type']] + fullname = os.path.join(path, name + extension) + print(fullname) + + pdata = br(doc['data']) + if isinstance(pdata, str): + # Retrocedemos se pdata ja eh uma str (necessario em Images) + pdata = doc + + with open(fullname, 'w') as arq: + while pdata: + arq.write(pdata['data']) + pdata = br(pdata.get('next', None)) + return name + + +nao_identificados = defaultdict(list) + + +def enumerate_folder(folder): + # folder vazio nao tem _objects + for entry in folder.get('_objects', []): + id, meta_type = entry['id'], entry['meta_type'] + obj = br(folder[id]) + yield id, obj, meta_type + + +def enumerate_btree(folder): + tree = folder['_tree'] + for id, obj in tree.iteritems(): + obj, meta_type = br(obj), type(obj).__name__ + yield id, obj, meta_type + + +def dump_folder(folder, path='', enum=enumerate_folder): + name = folder['id'] + path = os.path.join(path, name) + if not os.path.exists(path): + os.makedirs(path) + for id, obj, meta_type in enum(folder): + dump = DUMP_FUNCTIONS.get(meta_type, '?') + if dump == '?': + nao_identificados[meta_type].append(path + '/' + id) + elif dump: + id_interno = dump(obj, path) + assert id == id_interno + return name + + +DUMP_FUNCTIONS = { + 'File': dump_file, + 'Image': dump_file, + 'Folder': partial(dump_folder, enum=enumerate_folder), + 'BTreeFolder2': partial(dump_folder, enum=enumerate_btree), + + # explicitamente ignorados + 'ZCatalog': None, + 'Dumper': None, +} + + +def get_app(data_fs_path): + storage = ZODB.FileStorage.FileStorage(data_fs_path) + db = ZODB.DB(storage) + connection = db.open() + root = connection.root() + app = br(root['Application']) + + def close_db(): + db.close() + + return app, close_db + + +def find_sapl(app): + [id] = [e['id'] for e in app['_objects'] + if e['id'].startswith('cm_') + and e['meta_type'] == 'Folder'] + cm_zzz = br(app[id]) + sapl = br(cm_zzz['sapl']) + return sapl + + +def dump_sapl(data_fs_path): + app, close_db = get_app(data_fs_path) + try: + sapl = find_sapl(app) + docs = br(sapl['sapl_documentos']) + + nao_identificados.clear() + dump_folder(docs) + if nao_identificados: + print('#' * 80) + print('#' * 80) + print(u'FORAM ENCONTRADOS ARQUIVOS DE FORMATO NÃO IDENTIFICADO!!!') + print(u'REFAÇA A EXPORTAÇÃO\n') + print(nao_identificados) + print('#' * 80) + print('#' * 80) + finally: + close_db() diff --git a/sapl/legacy/scripts/exporta_zope/requirements.txt b/sapl/legacy/scripts/exporta_zope/requirements.txt new file mode 100644 index 000000000..9b1afa6fb --- /dev/null +++ b/sapl/legacy/scripts/exporta_zope/requirements.txt @@ -0,0 +1,2 @@ +# ZODB version 3.7.4 +git+git://github.com/zopefoundation/ZODB.git@d6f3c3ce5e6df1060de7a3#egg=ZODB3 From 9c63bd688617ee95e420808f1089f5584649bb2a Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Sat, 11 Nov 2017 17:34:27 -0200 Subject: [PATCH 02/12] Confere tamanho da btree exportada --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 618c9ec5c..3ebe40b4a 100644 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -76,10 +76,13 @@ def enumerate_folder(folder): def enumerate_btree(folder): + contagem_esperada = folder['_count'].value tree = folder['_tree'] - for id, obj in tree.iteritems(): + for contagem_real, (id, obj) in enumerate(tree.iteritems(), start=1): obj, meta_type = br(obj), type(obj).__name__ yield id, obj, meta_type + # verificação de consistência + assert contagem_esperada == contagem_real def dump_folder(folder, path='', enum=enumerate_folder): From 80d20175e7f1d435726f2f33c0b5aa050af3bd7d Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Mon, 13 Nov 2017 11:22:17 -0200 Subject: [PATCH 03/12] Adiciona dump de propriedades do site --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 10 ++++++++++ sapl/legacy/scripts/exporta_zope/requirements.txt | 1 + 2 files changed, 11 insertions(+) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 3ebe40b4a..1a39247bb 100644 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -152,3 +152,13 @@ def dump_sapl(data_fs_path): print('#' * 80) finally: close_db() + + +def dump_propriedades(docs): + props_sapl = br(docs['props_sapl']) + ids = [p['id'] for p in props_sapl['_properties']] + props = {id: props_sapl[id] for id in ids} + props = {id: p.decode('iso-8859-1') if isinstance(p, str) else p + for id, p in props.items()} + with open('sapl_documentos/propriedades.yaml', 'w') as f: + f.write(yaml.safe_dump(props)) diff --git a/sapl/legacy/scripts/exporta_zope/requirements.txt b/sapl/legacy/scripts/exporta_zope/requirements.txt index 9b1afa6fb..1770426da 100644 --- a/sapl/legacy/scripts/exporta_zope/requirements.txt +++ b/sapl/legacy/scripts/exporta_zope/requirements.txt @@ -1,2 +1,3 @@ # ZODB version 3.7.4 git+git://github.com/zopefoundation/ZODB.git@d6f3c3ce5e6df1060de7a3#egg=ZODB3 +PyYAML==3.12 From ffc34c3ed347a03ffa0d4ce8fcdded46e8e62518 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 14 Nov 2017 11:26:13 -0200 Subject: [PATCH 04/12] Especifica melhor o objeto sapl dentro do zodb --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 1a39247bb..9d43ce4f2 100644 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -126,12 +126,13 @@ def get_app(data_fs_path): def find_sapl(app): - [id] = [e['id'] for e in app['_objects'] - if e['id'].startswith('cm_') - and e['meta_type'] == 'Folder'] - cm_zzz = br(app[id]) - sapl = br(cm_zzz['sapl']) - return sapl + for obj in app['_objects']: + id, meta_type = obj['id'], obj['meta_type'] + if id.startswith('cm_') and meta_type == 'Folder': + cm_zzz = br(app[id]) + sapl = br(cm_zzz.get('sapl', None)) + if sapl and 'sapl_documentos' in sapl and 'acl_users' in sapl: + return sapl def dump_sapl(data_fs_path): From 82de14e686df5ed8078aaa05de939cd7e62f39ea Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 14 Nov 2017 11:36:11 -0200 Subject: [PATCH 05/12] Estrutura dump de docs como script --- .../scripts/exporta_zope/exporta_zope.py | 32 +++++++++++++------ 1 file changed, 23 insertions(+), 9 deletions(-) mode change 100644 => 100755 sapl/legacy/scripts/exporta_zope/exporta_zope.py diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py old mode 100644 new mode 100755 index 9d43ce4f2..2ac6eaad2 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -1,3 +1,4 @@ +#!/usr/bin/env python # -*- coding: utf-8 -*- # IMPORTANTE: @@ -5,9 +6,11 @@ # e depende apenas do descrito no arquivo requiments.txt import os.path +import sys from collections import defaultdict from functools import partial +import yaml import ZODB.DB import ZODB.FileStorage from ZODB.broken import Broken @@ -64,7 +67,6 @@ def dump_file(doc, path): return name -nao_identificados = defaultdict(list) def enumerate_folder(folder): @@ -85,6 +87,9 @@ def enumerate_btree(folder): assert contagem_esperada == contagem_real +nao_identificados = defaultdict(list) + + def dump_folder(folder, path='', enum=enumerate_folder): name = folder['id'] path = os.path.join(path, name) @@ -135,6 +140,16 @@ def find_sapl(app): return sapl +def dump_propriedades(docs): + props_sapl = br(docs['props_sapl']) + ids = [p['id'] for p in props_sapl['_properties']] + props = {id: props_sapl[id] for id in ids} + props = {id: p.decode('iso-8859-1') if isinstance(p, str) else p + for id, p in props.items()} + with open('sapl_documentos/propriedades.yaml', 'w') as f: + f.write(yaml.safe_dump(props)) + + def dump_sapl(data_fs_path): app, close_db = get_app(data_fs_path) try: @@ -143,6 +158,7 @@ def dump_sapl(data_fs_path): nao_identificados.clear() dump_folder(docs) + dump_propriedades(docs) if nao_identificados: print('#' * 80) print('#' * 80) @@ -155,11 +171,9 @@ def dump_sapl(data_fs_path): close_db() -def dump_propriedades(docs): - props_sapl = br(docs['props_sapl']) - ids = [p['id'] for p in props_sapl['_properties']] - props = {id: props_sapl[id] for id in ids} - props = {id: p.decode('iso-8859-1') if isinstance(p, str) else p - for id, p in props.items()} - with open('sapl_documentos/propriedades.yaml', 'w') as f: - f.write(yaml.safe_dump(props)) +if __name__ == "__main__": + if len(sys.argv) == 2: + data_fs_path = sys.argv[1] + dump_sapl(data_fs_path) + else: + print('Uso: python exporta_zope ') From d0c62ea8b841e7c4dbd42b2a385dade8bd47f501 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 14 Nov 2017 12:04:54 -0200 Subject: [PATCH 06/12] =?UTF-8?q?Contorna=20vazamento=20de=20mem=C3=B3ria?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../scripts/exporta_zope/exporta_zope.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 2ac6eaad2..d8498266a 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -55,18 +55,28 @@ def dump_file(doc, path): fullname = os.path.join(path, name + extension) print(fullname) - pdata = br(doc['data']) + # A partir daqui usamos dict.pop('...') nos __Broken_state__ + # para contornar um "vazamento" de memória que ocorre + # ao percorrer a árvore de objetos + # + # Imaginamos que, internamente, o ZODB está guardando referências + # para os objetos Broken criados e não conseguimos identificar como. + # + # Essa medida descarta quase todos os dados retornados + # e só funciona na primeira passagem + + pdata = br(doc.pop('data')) if isinstance(pdata, str): # Retrocedemos se pdata ja eh uma str (necessario em Images) + doc['data'] = pdata pdata = doc with open(fullname, 'w') as arq: while pdata: - arq.write(pdata['data']) - pdata = br(pdata.get('next', None)) - return name - + arq.write(pdata.pop('data')) + pdata = br(pdata.pop('next', None)) + return name def enumerate_folder(folder): From 37f83d0a4131390a566565a49184223c8f2ac878 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 14 Nov 2017 12:40:36 -0200 Subject: [PATCH 07/12] =?UTF-8?q?Registra=20extens=C3=B5es=20desconhecidas?= =?UTF-8?q?=20p=20inspe=C3=A7=C3=A3o=20posterior?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index d8498266a..f6ebb5748 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -49,12 +49,20 @@ def br(obj): return obj +extensoes_desconhecidas = defaultdict(list) + + def dump_file(doc, path): name = doc['__name__'] - extension = EXTENSOES[doc['content_type']] + content_type = doc['content_type'] + extension = EXTENSOES.get(content_type, 'ZZZZ') + fullname = os.path.join(path, name + extension) print(fullname) + if extension == 'ZZZZ': + extensoes_desconhecidas[content_type].append(fullname) + # A partir daqui usamos dict.pop('...') nos __Broken_state__ # para contornar um "vazamento" de memória que ocorre # ao percorrer a árvore de objetos From 09124f70ad4d76911886184566f103420e047eb3 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 14 Nov 2017 12:57:34 -0200 Subject: [PATCH 08/12] =?UTF-8?q?Usa=20=C3=BAltima=20vers=C3=A3o=20do=20ZO?= =?UTF-8?q?DB?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sapl/legacy/scripts/exporta_zope/requirements.txt b/sapl/legacy/scripts/exporta_zope/requirements.txt index 1770426da..4794267ae 100644 --- a/sapl/legacy/scripts/exporta_zope/requirements.txt +++ b/sapl/legacy/scripts/exporta_zope/requirements.txt @@ -1,3 +1,3 @@ # ZODB version 3.7.4 -git+git://github.com/zopefoundation/ZODB.git@d6f3c3ce5e6df1060de7a3#egg=ZODB3 PyYAML==3.12 +ZODB==5.3.0 From 614f92fce7faea485503329b58e3c61ecc6afefc Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 21 Nov 2017 13:12:59 -0200 Subject: [PATCH 09/12] =?UTF-8?q?Mant=C3=A9m=20extens=C3=A3o=20original=20?= =?UTF-8?q?dos=20arquivos=20que=20tiverem?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- sapl/legacy/scripts/exporta_zope/exporta_zope.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index f6ebb5748..b09d2e812 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -9,6 +9,7 @@ import os.path import sys from collections import defaultdict from functools import partial +from os.path import splitext import yaml import ZODB.DB @@ -53,9 +54,10 @@ extensoes_desconhecidas = defaultdict(list) def dump_file(doc, path): - name = doc['__name__'] + id = doc['__name__'] + name, extension = splitext(id) content_type = doc['content_type'] - extension = EXTENSOES.get(content_type, 'ZZZZ') + extension = extension or EXTENSOES.get(content_type, 'ZZZZ') fullname = os.path.join(path, name + extension) print(fullname) @@ -84,7 +86,7 @@ def dump_file(doc, path): arq.write(pdata.pop('data')) pdata = br(pdata.pop('next', None)) - return name + return id def enumerate_folder(folder): From 22de81d16ae9b6f82367e31ee523eda10b9a63d2 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Thu, 23 Nov 2017 15:24:25 -0200 Subject: [PATCH 10/12] Adiciona dump de documentos do SDE --- sapl/legacy/scripts/exporta_zope/.gitignore | 1 + .../scripts/exporta_zope/exporta_zope.py | 54 +++++++++++++++++-- 2 files changed, 50 insertions(+), 5 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/.gitignore b/sapl/legacy/scripts/exporta_zope/.gitignore index 98561c589..11c2f1d6b 100644 --- a/sapl/legacy/scripts/exporta_zope/.gitignore +++ b/sapl/legacy/scripts/exporta_zope/.gitignore @@ -1,2 +1,3 @@ Data*.fs* sapl_documentos +XSLT diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index b09d2e812..e0ae23f7a 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -89,14 +89,20 @@ def dump_file(doc, path): return id -def enumerate_folder(folder): - # folder vazio nao tem _objects - for entry in folder.get('_objects', []): - id, meta_type = entry['id'], entry['meta_type'] - obj = br(folder[id]) +def enumerate_by_key_list(folder, key_list, type_key): + for entry in folder.get(key_list, []): + id, meta_type = entry['id'], entry[type_key] + obj = br(folder.get(id, None)) yield id, obj, meta_type +enumerate_folder = partial(enumerate_by_key_list, + key_list='_objects', type_key='meta_type') + +enumerate_properties = partial(enumerate_by_key_list, + key_list='_properties', type_key='type') + + def enumerate_btree(folder): contagem_esperada = folder['_count'].value tree = folder['_tree'] @@ -125,11 +131,49 @@ def dump_folder(folder, path='', enum=enumerate_folder): return name +def decode_iso8859(obj): + return obj.decode('iso8859-1') if isinstance(obj, str) else obj + + +def read_sde(element): + + def read_properties(): + for id, obj, meta_type in enumerate_properties(element): + assert meta_type == 'string' + if id == 'title': + assert not obj + else: + yield id, decode_iso8859(obj) + + def read_children(): + for id, obj, meta_type in enumerate_folder(element): + assert meta_type == 'SDE-Document-Element' + yield id, read_sde(obj) + + data = dict(read_properties()) + children = list(read_children()) + if children: + assert all(k.startswith('SDE') for k, v in children) + data['children'] = children + return data + + +def dump_sde(strdoc, path): + id = strdoc['id'] + fullname = os.path.join(path, id + '.sde.yaml') + print(fullname) + sde = read_sde(strdoc) + with open(fullname, 'w') as arquivo: + yaml.safe_dump(sde, arquivo) + return id + + DUMP_FUNCTIONS = { 'File': dump_file, 'Image': dump_file, 'Folder': partial(dump_folder, enum=enumerate_folder), 'BTreeFolder2': partial(dump_folder, enum=enumerate_btree), + 'SDE-Document': dump_sde, # explicitamente ignorados 'ZCatalog': None, From 203d7493b4c45bcfa9d2ac9e5e0237c62ae2c310 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Fri, 24 Nov 2017 12:02:21 -0200 Subject: [PATCH 11/12] Adiciona dump de templates do SDE --- .../scripts/exporta_zope/exporta_zope.py | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index e0ae23f7a..8faf30c34 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -139,28 +139,30 @@ def read_sde(element): def read_properties(): for id, obj, meta_type in enumerate_properties(element): - assert meta_type == 'string' - if id == 'title': - assert not obj - else: - yield id, decode_iso8859(obj) + yield id, decode_iso8859(obj) def read_children(): for id, obj, meta_type in enumerate_folder(element): - assert meta_type == 'SDE-Document-Element' - yield id, read_sde(obj) + assert meta_type in ['SDE-Document-Element', + 'SDE-Template-Element', + 'SDE-Template-Link', + 'SDE-Template-Attribute', + 'Script (Python)', + ] + if meta_type != 'Script (Python)': + # ignoramos os scrips python de eventos dos templates + yield id, read_sde(obj) data = dict(read_properties()) children = list(read_children()) if children: - assert all(k.startswith('SDE') for k, v in children) data['children'] = children return data -def dump_sde(strdoc, path): +def dump_sde(strdoc, path, tipo): id = strdoc['id'] - fullname = os.path.join(path, id + '.sde.yaml') + fullname = os.path.join(path, '{}.{}.yaml'.format(id, tipo)) print(fullname) sde = read_sde(strdoc) with open(fullname, 'w') as arquivo: @@ -173,7 +175,8 @@ DUMP_FUNCTIONS = { 'Image': dump_file, 'Folder': partial(dump_folder, enum=enumerate_folder), 'BTreeFolder2': partial(dump_folder, enum=enumerate_btree), - 'SDE-Document': dump_sde, + 'SDE-Document': partial(dump_sde, tipo='sde.document'), + 'SDE-Template': partial(dump_sde, tipo='sde.template'), # explicitamente ignorados 'ZCatalog': None, From 47412eb1019c4b348ced6577ce0c82c39e080b79 Mon Sep 17 00:00:00 2001 From: Marcio Mazza Date: Tue, 5 Dec 2017 16:30:16 -0200 Subject: [PATCH 12/12] Adiciona dump de usuarios e consolida dump_sapl --- .../scripts/exporta_zope/exporta_zope.py | 37 +++++++++++++------ 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/sapl/legacy/scripts/exporta_zope/exporta_zope.py b/sapl/legacy/scripts/exporta_zope/exporta_zope.py index 8faf30c34..d1dcc4736 100755 --- a/sapl/legacy/scripts/exporta_zope/exporta_zope.py +++ b/sapl/legacy/scripts/exporta_zope/exporta_zope.py @@ -160,13 +160,18 @@ def read_sde(element): return data +def save_as_yaml(path, name, obj): + fullname = os.path.join(path, name) + with open(fullname, 'w') as arquivo: + yaml.safe_dump(obj, arquivo) + print(fullname) + return fullname + + def dump_sde(strdoc, path, tipo): id = strdoc['id'] - fullname = os.path.join(path, '{}.{}.yaml'.format(id, tipo)) - print(fullname) sde = read_sde(strdoc) - with open(fullname, 'w') as arquivo: - yaml.safe_dump(sde, arquivo) + save_as_yaml(path, '{}.{}.yaml'.format(id, tipo), sde) return id @@ -207,25 +212,35 @@ def find_sapl(app): return sapl -def dump_propriedades(docs): +def dump_propriedades(docs, path): props_sapl = br(docs['props_sapl']) ids = [p['id'] for p in props_sapl['_properties']] props = {id: props_sapl[id] for id in ids} props = {id: p.decode('iso-8859-1') if isinstance(p, str) else p for id, p in props.items()} - with open('sapl_documentos/propriedades.yaml', 'w') as f: - f.write(yaml.safe_dump(props)) + save_as_yaml(path, 'sapl_documentos/propriedades.yaml', props) + + +def dump_usuarios(sapl, path): + users = br(br(sapl['acl_users'])['data']) + users = {k: br(v) for k, v in users['data'].items()} + save_as_yaml(path, 'usuarios.yaml', users) -def dump_sapl(data_fs_path): +def dump_sapl(data_fs_path, destino='../../../../media'): app, close_db = get_app(data_fs_path) try: sapl = find_sapl(app) - docs = br(sapl['sapl_documentos']) + # extrai folhas XSLT + dump_folder(br(sapl['XSLT']), destino) + # extrai usuários com suas senhas e perfis + dump_usuarios(sapl, destino) + # extrai documentos + docs = br(sapl['sapl_documentos']) nao_identificados.clear() - dump_folder(docs) - dump_propriedades(docs) + dump_folder(docs, destino) + dump_propriedades(docs, destino) if nao_identificados: print('#' * 80) print('#' * 80)