From 5b506ff8374338348ba397b6c6f1c5fc84a8c87a Mon Sep 17 00:00:00 2001 From: Victor Fabre Date: Wed, 11 Apr 2018 18:31:11 -0300 Subject: [PATCH 1/7] Fix #1851 (#1853) * Fix #1851 * Adciona unrtf para indexar .rtf * Altera sintaxe --- Dockerfile | 2 +- sapl/base/search_indexes.py | 19 ++++++++++++++----- sapl/utils.py | 22 +++++++++++++++++++--- 3 files changed, 34 insertions(+), 9 deletions(-) diff --git a/Dockerfile b/Dockerfile index 693230025..fab4204d8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.5 ENV BUILD_PACKAGES postgresql-dev graphviz-dev graphviz build-base git pkgconfig \ python3-dev libxml2-dev jpeg-dev libressl-dev libffi-dev libxslt-dev nodejs py3-lxml \ -py3-magic postgresql-client poppler-utils antiword vim +py3-magic postgresql-client poppler-utils antiword vim unrtf RUN apk --update add fontconfig ttf-dejavu && fc-cache -fv diff --git a/sapl/base/search_indexes.py b/sapl/base/search_indexes.py index f685cb002..ed811cebb 100644 --- a/sapl/base/search_indexes.py +++ b/sapl/base/search_indexes.py @@ -21,6 +21,7 @@ from sapl.compilacao.models import (STATUS_TA_IMMUTABLE_PUBLIC, from sapl.materia.models import DocumentoAcessorio, MateriaLegislativa from sapl.norma.models import NormaJuridica from sapl.settings import BASE_DIR, SOLR_URL +from sapl.utils import RemoveTag logger = logging.getLogger(BASE_DIR.name) @@ -51,10 +52,17 @@ class TextExtractField(CharField): return extracted_data def whoosh_extraction(self, arquivo): - return textract.process( - arquivo.path, - language='pt-br').decode('utf-8').replace('\n', ' ').replace( - '\t', ' ') + + if arquivo.path.endswith('html') or arquivo.path.endswith('xml'): + with open(arquivo.path, 'r', encoding="utf8", errors='ignore') as f: + content = ' '.join(f.read()) + return RemoveTag(content) + + else: + return textract.process( + arquivo.path, + language='pt-br').decode('utf-8').replace('\n', ' ').replace( + '\t', ' ') def print_error(self, arquivo): msg = 'Erro inesperado processando arquivo: %s' % ( @@ -82,7 +90,8 @@ class TextExtractField(CharField): except ExtensionNotSupported as e: print(str(e)) logger.error(str(e)) - except Exception: + except Exception as e2: + print(str(e2)) self.print_error(arquivo) return '' diff --git a/sapl/utils.py b/sapl/utils.py index 196e5b5cf..0fa20f706 100644 --- a/sapl/utils.py +++ b/sapl/utils.py @@ -707,12 +707,28 @@ def TrocaTag(texto, startTag, endTag, sizeStart, sizeEnd, styleName): if (texto[i:i + sizeEnd] == endTag): textoSaida += 'blockTable>' insideTag = 0 - i = i + sizeEnd + i += sizeEnd else: textoSaida += texto[i] - i = i + 1 + i += 1 else: textoSaida += texto[i] - i = i + 1 + i += 1 + + return textoSaida + + +def RemoveTag(texto): + textoSaida = '' + i = 0 + + while (i < len(texto)): + + if (texto[i] == '<'): + i = ExtraiTag(texto, i) + + else: + textoSaida += texto[i] + i += 1 return textoSaida From 9a7e048ac9aec82abc3f801c611f8500f4d728f5 Mon Sep 17 00:00:00 2001 From: Talitha Pumar Date: Wed, 11 Apr 2018 18:28:23 -0300 Subject: [PATCH 2/7] Fix #1845 (#1852) --- sapl/templates/materia/materialegislativa_filter.html | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sapl/templates/materia/materialegislativa_filter.html b/sapl/templates/materia/materialegislativa_filter.html index 2d54016ba..f44d8788b 100644 --- a/sapl/templates/materia/materialegislativa_filter.html +++ b/sapl/templates/materia/materialegislativa_filter.html @@ -135,8 +135,9 @@ {% endfor %} {% endif %}

- - Acompanhar Matéria + {% if m.em_tramitacao %} + Acompanhar Matéria + {% endif %} {% endfor %} From 740755886f7a0c2708171e9b17e8b1536967aef0 Mon Sep 17 00:00:00 2001 From: Mariana Mendes Date: Wed, 11 Apr 2018 12:49:06 -0300 Subject: [PATCH 3/7] FIX #1842 (#1848) --- sapl/relatorios/templates/pdf_sessao_plenaria_gerar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sapl/relatorios/templates/pdf_sessao_plenaria_gerar.py b/sapl/relatorios/templates/pdf_sessao_plenaria_gerar.py index 42c74b796..2632fe94c 100644 --- a/sapl/relatorios/templates/pdf_sessao_plenaria_gerar.py +++ b/sapl/relatorios/templates/pdf_sessao_plenaria_gerar.py @@ -120,7 +120,7 @@ def inf_basicas(inf_basicas_dic): data_fim_sessao = dat_fim_sessao + ' - ' if dat_fim_sessao else '' tmp += '\t\tEncerramento: ' + \ - dat_fim_sessao + hr_fim_sessao + '\n' + dat_fim_sessao + ' - ' + hr_fim_sessao + '\n' return tmp From fa36b8af73a4e56881cfec364f4c2def9d0b1842 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Wed, 11 Apr 2018 18:36:08 -0300 Subject: [PATCH 4/7] Release: 3.1.71 --- docker-compose.yml | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7b301395c..36df3e501 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,7 +11,7 @@ sapldb: ports: - "5432:5432" sapl: - image: interlegis/sapl:3.1.70 + image: interlegis/sapl:3.1.71 restart: always environment: ADMIN_PASSWORD: interlegis diff --git a/setup.py b/setup.py index cbe8558aa..7b3240fbf 100644 --- a/setup.py +++ b/setup.py @@ -49,7 +49,7 @@ install_requires = [ ] setup( name='interlegis-sapl', - version='3.1.70', + version='3.1.71', packages=find_packages(), include_package_data=True, license='GNU GENERAL PUBLIC LICENSE Version 3, 29 June 2007', From 566af94dd83c9b3665de3672fcd8dd71bb0361b3 Mon Sep 17 00:00:00 2001 From: Victor Fabre Date: Wed, 11 Apr 2018 12:49:30 -0300 Subject: [PATCH 5/7] Fix #1836 (#1837) --- requirements/requirements.txt | 1 + sapl/settings.py | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 4e8c9e452..2f6b5a8db 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -35,3 +35,4 @@ gunicorn==19.6.0 django-reversion==2.0.8 WeasyPrint==0.42 whoosh==2.7.4 +git+git://github.com/interlegis/django-speedinfo.git diff --git a/sapl/settings.py b/sapl/settings.py index 2e6b7c7ff..e79df2d61 100644 --- a/sapl/settings.py +++ b/sapl/settings.py @@ -87,6 +87,7 @@ INSTALLED_APPS = ( 'rest_framework', 'reversion', 'whoosh', + 'speedinfo', ) + SAPL_APPS @@ -125,8 +126,16 @@ MIDDLEWARE_CLASSES = ( 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'django.middleware.security.SecurityMiddleware', + 'speedinfo.middleware.ProfilerMiddleware', ) +CACHES = { + 'default': { + 'BACKEND': 'speedinfo.backends.proxy_cache', + 'CACHE_BACKEND': 'django.core.cache.backends.filebased.FileBasedCache', + 'LOCATION': '/var/tmp/django_cache', + } +} REST_FRAMEWORK = { "UNICODE_JSON": False, From dfabe1195ff5a586b0dd08c58fbc867c029600b8 Mon Sep 17 00:00:00 2001 From: VictorFabreF Date: Wed, 11 Apr 2018 18:10:51 -0300 Subject: [PATCH 6/7] HOT-FIX: atualiza django-speedinfo nos requirements --- requirements/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 2f6b5a8db..fe2e139c8 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -35,4 +35,4 @@ gunicorn==19.6.0 django-reversion==2.0.8 WeasyPrint==0.42 whoosh==2.7.4 -git+git://github.com/interlegis/django-speedinfo.git +django-speedinfo==1.3.5 From 761ef98b7c948e82e38d0fffc2634be20728c9a4 Mon Sep 17 00:00:00 2001 From: Edward Ribeiro Date: Thu, 12 Apr 2018 07:49:30 -0300 Subject: [PATCH 7/7] HOT-FIX: remove unrtf do Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index fab4204d8..693230025 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM alpine:3.5 ENV BUILD_PACKAGES postgresql-dev graphviz-dev graphviz build-base git pkgconfig \ python3-dev libxml2-dev jpeg-dev libressl-dev libffi-dev libxslt-dev nodejs py3-lxml \ -py3-magic postgresql-client poppler-utils antiword vim unrtf +py3-magic postgresql-client poppler-utils antiword vim RUN apk --update add fontconfig ttf-dejavu && fc-cache -fv