diff --git a/dist/bin/upload_configset.sh b/dist/bin/upload_configset.sh old mode 100644 new mode 100755 diff --git a/docker/Dockerfile b/docker/Dockerfile index 460422def..a40bd91fe 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,12 +1,10 @@ -FROM python:3.7-slim-buster +FROM python:3.9-slim-buster # Setup env ENV LANG C.UTF-8 ENV LC_ALL C.UTF-8 ENV PYTHONDONTWRITEBYTECODE 1 - -#ENV PYTHONFAULTHANDLER 1 - +ENV PYTHONUNBUFFERED=1 ENV DEBIAN_FRONTEND noninteractive ENV BUILD_PACKAGES apt-utils apt-file libpq-dev graphviz-dev build-essential git pkg-config \ @@ -34,12 +32,13 @@ RUN apt-get update && \ SUDO_FORCE_REMOVE=yes apt-get purge -y --auto-remove $BUILD_PACKAGES && \ apt-get autoremove && apt-get clean && rm -rf /var/lib/apt/lists/* -ENV HOME=/var/interlegis/sapl +WORKDIR /var/interlegis/sapl/ +ADD . /var/interlegis/sapl/ COPY docker/start.sh $HOME -COPY docker/check_solr.sh $HOME -COPY docker/solr_api.py $HOME -COPY docker/busy-wait.sh $HOME +COPY docker/solr_cli.py $HOME +COPY docker/wait-for-pg.sh $HOME +COPY docker/wait-for-solr.sh $HOME COPY docker/create_admin.py $HOME COPY docker/genkey.py $HOME COPY docker/gunicorn_start.sh $HOME @@ -55,7 +54,8 @@ RUN rm -rf /var/interlegis/sapl/sapl/.env && \ rm -rf /var/interlegis/sapl/sapl.db RUN chmod +x /var/interlegis/sapl/start.sh && \ - chmod +x /var/interlegis/sapl/check_solr.sh && \ + chmod +x /var/interlegis/sapl/wait-for-solr.sh && \ + chmod +x /var/interlegis/sapl/wait-for-pg.sh && \ ln -sf /dev/stdout /var/log/nginx/access.log && \ ln -sf /dev/stderr /var/log/nginx/error.log && \ mkdir /var/log/sapl/ && touch /var/interlegis/sapl/sapl.log && \ diff --git a/docker/docker-compose.yaml b/docker/docker-compose.yaml new file mode 100644 index 000000000..b5c210201 --- /dev/null +++ b/docker/docker-compose.yaml @@ -0,0 +1,77 @@ +version: "3.7" +services: + sapldb: + image: postgres:10.5-alpine + restart: always + container_name: postgres + labels: + NAME: "postgres" + environment: + POSTGRES_PASSWORD: sapl + POSTGRES_USER: sapl + POSTGRES_DB: sapl + PGDATA : /var/lib/postgresql/data/ + volumes: + - sapldb_data:/var/lib/postgresql/data/ + ports: + - "5433:5432" + networks: + - sapl-net + saplsolr: + image: solr:8.11 + restart: always + command: bin/solr start -c -f + container_name: solr + labels: + NAME: "solr" + volumes: + - solr_data:/opt/solr/server/solr + - solr_configsets:/opt/solr/server/solr/configsets + ports: + - "8983:8983" + networks: + - sapl-net + sapl: +# image: interlegis/sapl:3.1.162-RC13 + build: + context: ../ + dockerfile: ./docker/Dockerfile + container_name: sapl + labels: + NAME: "sapl" + restart: always + environment: + ADMIN_PASSWORD: interlegis + ADMIN_EMAIL: email@dominio.net + DEBUG: 'False' + EMAIL_PORT: 587 + EMAIL_USE_TLS: 'False' + EMAIL_HOST: smtp.dominio.net + EMAIL_HOST_USER: usuariosmtp + EMAIL_SEND_USER: usuariosmtp + EMAIL_HOST_PASSWORD: senhasmtp + USE_SOLR: 'True' + SOLR_COLLECTION: sapl + SOLR_URL: http://solr:solr@saplsolr:8983 + IS_ZK_EMBEDDED: 'True' + TZ: America/Sao_Paulo + volumes: + - sapl_data:/var/interlegis/sapl/data + - sapl_media:/var/interlegis/sapl/media + depends_on: + - sapldb + - saplsolr + ports: + - "80:80" + networks: + - sapl-net +networks: + sapl-net: + name: sapl-net + driver: bridge +volumes: + sapldb_data: + sapl_data: + sapl_media: + solr_data: + solr_configsets: \ No newline at end of file diff --git a/docker/solr_api.py b/docker/solr_cli.py similarity index 69% rename from docker/solr_api.py rename to docker/solr_cli.py index 3ca06bba7..d452d1fe9 100755 --- a/docker/solr_api.py +++ b/docker/solr_cli.py @@ -1,18 +1,102 @@ -from io import BytesIO +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- import argparse -import os -import requests +import logging +import re +import secrets import subprocess import sys import zipfile +from base64 import b64encode, b64decode +from hashlib import sha256 +from io import BytesIO from pathlib import Path -## -## Este módulo deve ser executado na raiz do projeto -## +import requests +from kazoo.client import KazooClient + +# +# Este módulo deve ser executado na raiz do projeto +# + +logging.basicConfig() + +SECURITY_FILE_TEMPLATE = """ + { + "authentication":{ + "blockUnknown": true, + "class":"solr.BasicAuthPlugin", + "credentials":{"%s":"%s %s"}, + "forwardCredentials": false, + "realm": "Solr Login" + }, + "authorization":{ + "class":"solr.RuleBasedAuthorizationPlugin", + "permissions":[{"name":"security-edit", "role":"admin"}], + "user-role":{"%s":"admin"} + } + } +""" + +URL_PATTERN = 'https?://(([a-zA-Z0-9]+):([a-zA-Z0-9]+)@)?([a-zA-Z0-9.-]+)(:[0-9]{4})?' + + +def solr_hash_password(password: str, salt: str = None): + """ + Generates a password and salt to be used in Basic Auth Solr + + password: clean text password string + salt (optional): base64 salt string + returns: sha256 hash of password and salt (both base64 strings) + """ + m = sha256() + if salt is None: + salt = secrets.token_bytes(32) + else: + salt = b64decode(salt) + m.update(salt + password.encode('utf-8')) + digest = m.digest() -class SolrClient: + m = sha256() + m.update(digest) + digest = m.digest() + + cypher = b64encode(digest).decode('utf-8') + salt = b64encode(salt).decode('utf-8') + return cypher, salt + + +def create_security_file(username, password): + print("Creating security.json file...") + with open("security.json", "w") as f: + cypher, salt = solr_hash_password(password) + f.write(SECURITY_FILE_TEMPLATE % (username, cypher, salt, username)) + print("file created!") + + +def upload_security_file(zk_host): + zk_port = 9983 # embedded ZK port + print(f"Uploading security file to Solr, ZK server={zk_host}:{zk_port}...") + try: + with open('security.json', 'r') as f: + data = f.read() + zk = KazooClient(hosts=f"{zk_host}:{zk_port}") + zk.start() + print("Uploading security.json file...") + if zk.exists('/security.json'): + zk.set("/security.json", str.encode(data)) + else: + zk.create("/security.json", str.encode(data)) + data, stat = zk.get('/security.json') + print("file uploaded!") + print(data.decode('utf-8')) + zk.stop() + except Exception as e: + print(e) + sys.exit(-1) + +class SolrClient: LIST_CONFIGSETS = "{}/solr/admin/configs?action=LIST&omitHeader=true&wt=json" UPLOAD_CONFIGSET = "{}/solr/admin/configs?action=UPLOAD&name={}&wt=json" LIST_COLLECTIONS = "{}/solr/admin/collections?action=LIST&wt=json" @@ -129,7 +213,7 @@ class SolrClient: print(res.content) return False return True - + def delete_collection(self, collection_name): if collection_name == '*': collections = self.list_collections() @@ -160,6 +244,22 @@ class SolrClient: print("Num docs: %s" % num_docs) +def setup_embedded_zk(solr_url): + match = re.match(URL_PATTERN, solr_url) + if match: + _, solr_user, solr_pwd, solr_host, solr_port = match.groups() + + if solr_user and solr_pwd and solr_host: + create_security_file(solr_user, solr_pwd) + upload_security_file(solr_host) + else: + print(f"Missing Solr's username, password, and host: {solr_user}/{solr_pwd}/{solr_host}") + sys.exit(-1) + else: + print(f"Solr URL path doesn't match the required format: {solr_url}") + sys.exit(-1) + + if __name__ == '__main__': parser = argparse.ArgumentParser(description='Cria uma collection no Solr') @@ -178,6 +278,9 @@ if __name__ == '__main__': parser.add_argument('-ms', type=int, dest='max_shards_per_node', nargs='?', help='Max shards per node (default=1)', default=1) + parser.add_argument("--embedded_zk", default=False, action="store_true", + help="Embedded ZooKeeper") + try: args = parser.parse_args() except IOError as msg: @@ -185,10 +288,17 @@ if __name__ == '__main__': sys.exit(-1) url = args.url.pop() - collection = args.collection.pop() + if args.embedded_zk: + print("Setup embedded ZooKeeper...") + setup_embedded_zk(url) + + collection = args.collection.pop() client = SolrClient(url=url) + ## Add --force to force upload security.json, configset upload and collection recreation + ## it will clean the solr server before proceeding + ## Add --clean option to clean uploadconfig and collection if not client.exists_collection(collection): print("Collection '%s' doesn't exists. Creating a new one..." % collection) created = client.create_collection(collection, @@ -200,6 +310,7 @@ if __name__ == '__main__': else: print("Collection '%s' exists." % collection) + ## Add --disable-index to disable auto index num_docs = client.get_num_docs(collection) if num_docs == 0: print("Performing a full reindex of '%s' collection..." % collection) diff --git a/docker/start.sh b/docker/start.sh index 868942bfd..57b83b484 100755 --- a/docker/start.sh +++ b/docker/start.sh @@ -22,7 +22,6 @@ create_env() { touch $FILENAME - # explicitly use '>' to erase any previous content echo "SECRET_KEY="$KEY > $FILENAME # now only appends @@ -39,14 +38,14 @@ create_env() { echo "USE_SOLR = ""${USE_SOLR-False}" >> $FILENAME echo "SOLR_COLLECTION = ""${SOLR_COLLECTION-sapl}" >> $FILENAME echo "SOLR_URL = ""${SOLR_URL-http://localhost:8983}" >> $FILENAME + echo "IS_ZK_EMBEDDED = ""${IS_ZK_EMBEDDED-False}" >> $FILENAME - echo "[ENV FILE] done." } create_env -/bin/bash busy-wait.sh $DATABASE_URL +/bin/bash wait-for-pg.sh $DATABASE_URL yes yes | python3 manage.py migrate @@ -55,39 +54,46 @@ yes yes | python3 manage.py migrate USE_SOLR="${USE_SOLR:=False}" SOLR_URL="${SOLR_URL:=http://localhost:8983}" SOLR_COLLECTION="${SOLR_COLLECTION:=sapl}" - NUM_SHARDS=${NUM_SHARDS:=1} RF=${RF:=1} MAX_SHARDS_PER_NODE=${MAX_SHARDS_PER_NODE:=1} +IS_ZK_EMBEDDED="${IS_ZK_EMBEDDED:=False}" if [ "${USE_SOLR-False}" == "True" ] || [ "${USE_SOLR-False}" == "true" ]; then - echo "SOLR configurations" + echo "Solr configurations" echo "===================" echo "URL: $SOLR_URL" echo "COLLECTION: $SOLR_COLLECTION" echo "NUM_SHARDS: $NUM_SHARDS" echo "REPLICATION FACTOR: $RF" echo "MAX SHARDS PER NODE: $MAX_SHARDS_PER_NODE" + echo "ASSUME ZK EMBEDDED: $IS_ZK_EMBEDDED" echo "=========================================" - echo "running solr script" - /bin/bash check_solr.sh $SOLR_URL + echo "running Solr script" + /bin/bash wait-for-solr.sh $SOLR_URL CHECK_SOLR_RETURN=$? if [ $CHECK_SOLR_RETURN == 1 ]; then - echo "Connecting to solr..." - python3 solr_api.py -u $SOLR_URL -c $SOLR_COLLECTION -s $NUM_SHARDS -rf $RF -ms $MAX_SHARDS_PER_NODE & - # python3 manage.py rebuild_index --noinput & + echo "Connecting to Solr..." + + + if [ "${IS_ZK_EMBEDDED-False}" == "True" ] || [ "${IS_ZK_EMBEDDED-False}" == "true" ]; then + ZK_EMBEDDED="--embedded_zk" + echo "Assuming embedded ZooKeeper instalation..." + fi + + python3 solr_cli.py -u $SOLR_URL -c $SOLR_COLLECTION -s $NUM_SHARDS -rf $RF -ms $MAX_SHARDS_PER_NODE $ZK_EMBEDDED & else echo "Solr is offline, not possible to connect." fi else - echo "Suporte a SOLR não inicializado." + echo "Solr support is not initialized." fi -echo "Criando usuário admin..." +echo "Creating admin user..." user_created=$(python3 create_admin.py 2>&1) diff --git a/docker/busy-wait.sh b/docker/wait-for-pg.sh similarity index 100% rename from docker/busy-wait.sh rename to docker/wait-for-pg.sh diff --git a/docker/check_solr.sh b/docker/wait-for-solr.sh similarity index 82% rename from docker/check_solr.sh rename to docker/wait-for-solr.sh index 370b4adcf..df1fab5bb 100644 --- a/docker/check_solr.sh +++ b/docker/wait-for-solr.sh @@ -4,10 +4,10 @@ SOLR_URL=$1 -RETRY_COUNT=1 -RETRY_LIMIT=4 +RETRY_COUNT=0 +RETRY_LIMIT=60 # wait until 1 min -echo "Waiting for solr connection at $SOLR_URL ..." +echo "Waiting for Solr connection at $SOLR_URL ..." while [[ $RETRY_COUNT < $RETRY_LIMIT ]]; do echo "Attempt to connect to solr: $RETRY_COUNT of $RETRY_LIMIT" let RETRY_COUNT=RETRY_COUNT+1; @@ -18,7 +18,7 @@ while [[ $RETRY_COUNT < $RETRY_LIMIT ]]; do echo "Solr server is up!" exit 1 else - sleep 3 + sleep 1 fi done echo "Solr connection failed." diff --git a/requirements/requirements.txt b/requirements/requirements.txt index 40d3c4fdd..82241606b 100644 --- a/requirements/requirements.txt +++ b/requirements/requirements.txt @@ -33,6 +33,7 @@ PyPDF4==1.27.0 pyoai==2.5.0 Unidecode==1.1.1 whitenoise==5.1.0 +kazoo==2.8.0 git+https://github.com/interlegis/trml2pdf git+https://github.com/interlegis/django-admin-bootstrapped diff --git a/scripts/solr_password.py b/solr/bin/solr_password.py similarity index 97% rename from scripts/solr_password.py rename to solr/bin/solr_password.py index 7a558292d..050633444 100755 --- a/scripts/solr_password.py +++ b/solr/bin/solr_password.py @@ -18,7 +18,7 @@ from base64 import b64encode, b64decode ## -def solr_hash_password(password: str, salt: str = None) -> (str, str): +def solr_hash_password(password: str, salt: str = None): """ Generates a password and salt to be used in Basic Auth Solr