diff --git a/.dockerignore b/.dockerignore index 8f4c8f147..14b4481ad 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,4 +1,18 @@ -media -collected_static +*.log +*.swp +*~ +whoosh_index/ +whoosh/ +media/ +collected_static/ +bower +bower_components +postgres-data/ +data/ +solr-*/ +.cache .git -whoosh +.github +.idea +.project +.env \ No newline at end of file diff --git a/.gitignore b/.gitignore index e59540379..67e285e2e 100644 --- a/.gitignore +++ b/.gitignore @@ -105,3 +105,4 @@ solr-*/ # ignora tudo dentro de media, mas cria a pasta no checkout media/* !media/.gitkeep + diff --git a/.travis.yml b/.travis.yml index 80532c789..4910fa099 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,25 +1,39 @@ -language: python - -python: - - 3.5 +sudo: required services: - - postgresql + - docker -install: - - pip install -r requirements/test-requirements.txt +env: + global: + - DOCKER_COMPOSE_VERSION=1.24.1 -before_script: - - cp sapl/.env_test sapl/.env - - psql -c "CREATE USER sapl WITH PASSWORD 'sapl'" -U postgres; - - psql -c "CREATE DATABASE sapl OWNER sapl;" -U postgres - - ./scripts/django/check_migrations.sh +before_install: + # Install latest versions of docker and docker-compose + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add - + - sudo add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" + - sudo apt-get update + - sudo apt-get -y -o Dpkg::Options::="--force-confnew" install docker-ce + - sudo rm /usr/local/bin/docker-compose + - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + - chmod +x docker-compose + - sudo mv docker-compose /usr/local/bin + - docker --version + - docker-compose --version + +install: + - sudo service postgresql stop || true script: - - ./manage.py migrate - - py.test --create-db - # - ./scripts/django/test_and_check_qa.sh + - cd $TRAVIS_BUILD_DIR/docker + # Build master images and run the webserver (tests end to end) + #- docker-compose -f docker-compose.test.yml build + #- docker-compose up -d + #- docker logs sapl + #- docker-compose rm -f -s -v + - docker-compose -f docker-compose.test.yml run --workdir="/var/interlegis/sapl" sapl bash -c "cp docker/config/env_test .env && py.test --create-db" -addons: - hosts: - - 127.0.0.1 sapldb +after_script: + # Cleanup docker containers, images, and volumes + - docker-compose rm -f -s -v + - docker system prune -a -f + - docker volume prune -f diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 5eb35418d..000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,33 +0,0 @@ -sapldb: - image: postgres:10.5-alpine - restart: always - environment: - POSTGRES_PASSWORD: sapl - POSTGRES_USER: sapl - POSTGRES_DB: sapl - PGDATA : /var/lib/postgresql/data/ - volumes: - - sapldb_data:/var/lib/postgresql/data/ - ports: - - "5432:5432" -sapl: - image: interlegis/sapl:3.1.160-RC1 - restart: always - environment: - ADMIN_PASSWORD: interlegis - ADMIN_EMAIL: email@dominio.net - DEBUG: 'False' - EMAIL_PORT: 587 - EMAIL_USE_TLS: 'False' - EMAIL_HOST: smtp.dominio.net - EMAIL_HOST_USER: usuariosmtp - EMAIL_SEND_USER: usuariosmtp - EMAIL_HOST_PASSWORD: senhasmtp - TZ: America/Sao_Paulo - volumes: - - sapl_data:/var/interlegis/sapl/data - - sapl_media:/var/interlegis/sapl/media - links: - - sapldb - ports: - - "80:80" \ No newline at end of file diff --git a/Dockerfile b/docker/Dockerfile similarity index 65% rename from Dockerfile rename to docker/Dockerfile index 9c33fcd0f..535406d25 100644 --- a/Dockerfile +++ b/docker/Dockerfile @@ -5,7 +5,8 @@ ENV BUILD_PACKAGES postgresql-dev graphviz-dev graphviz build-base git pkgconfig nodejs py3-lxml py3-magic postgresql-client poppler-utils antiword \ curl jq openssh-client vim bash -RUN apk update --update-cache && apk upgrade +RUN apk update --update-cache && apk upgrade && \ + apk add --update --no-cache $BUILD_PACKAGES RUN apk --update add fontconfig ttf-dejavu && fc-cache -fv @@ -16,21 +17,27 @@ RUN apk add --no-cache python3 nginx tzdata && \ rm -r /root/.cache && \ rm -f /etc/nginx/conf.d/* -RUN mkdir -p /var/interlegis/sapl && \ - apk add --update --no-cache $BUILD_PACKAGES - +RUN mkdir -p /var/interlegis/sapl && mkdir /var/log/sapl WORKDIR /var/interlegis/sapl/ +COPY . /var/interlegis/sapl/ +RUN pip install -r /var/interlegis/sapl/requirements/dev-requirements.txt --upgrade setuptools && \ + rm -r /root/.cache -ADD . /var/interlegis/sapl/ +WORKDIR /var/interlegis/sapl/ -COPY start.sh /var/interlegis/sapl/ -COPY config/nginx/sapl.conf /etc/nginx/conf.d -COPY config/nginx/nginx.conf /etc/nginx/nginx.conf +ENV HOME=/var/interlegis/sapl -RUN pip install -r /var/interlegis/sapl/requirements/dev-requirements.txt --upgrade setuptools && \ - rm -r /root/.cache +COPY docker/start.sh $HOME +COPY docker/check_solr.sh $HOME +COPY docker/solr_api.py $HOME +COPY docker/busy-wait.sh $HOME +COPY docker/create_admin.py $HOME +COPY docker/genkey.py $HOME +COPY docker/gunicorn_start.sh $HOME -COPY config/env_dockerfile /var/interlegis/sapl/sapl/.env +COPY docker/config/nginx/sapl.conf /etc/nginx/conf.d +COPY docker/config/nginx/nginx.conf /etc/nginx/nginx.conf +COPY docker/config/env_dockerfile /var/interlegis/sapl/sapl/.env RUN python3 manage.py collectstatic --noinput --clear @@ -38,11 +45,10 @@ RUN python3 manage.py collectstatic --noinput --clear RUN rm -rf /var/interlegis/sapl/sapl/.env && \ rm -rf /var/interlegis/sapl/sapl.db -RUN chmod +x /var/interlegis/sapl/start.sh && \ - chmod +x /var/interlegis/sapl/check_solr.sh && \ +RUN chmod +x /var/interlegis/sapl/*.sh && \ ln -sf /dev/stdout /var/log/nginx/access.log && \ ln -sf /dev/stderr /var/log/nginx/error.log && \ - mkdir /var/log/sapl/ && touch /var/interlegis/sapl/sapl.log && \ + touch /var/interlegis/sapl/sapl.log && \ ln -s /var/interlegis/sapl/sapl.log /var/log/sapl/sapl.log VOLUME ["/var/interlegis/sapl/data", "/var/interlegis/sapl/media"] diff --git a/docker/Dockerfile.multistage b/docker/Dockerfile.multistage new file mode 100644 index 000000000..1879da32f --- /dev/null +++ b/docker/Dockerfile.multistage @@ -0,0 +1,62 @@ +# SAPL BASE IMAGE +FROM alpine:3.8 AS sapl-base + +ENV BUILD_PACKAGES postgresql-dev graphviz-dev graphviz build-base git pkgconfig \ + python3-dev libxml2-dev jpeg-dev libressl-dev libffi-dev libxslt-dev \ + nodejs py3-lxml py3-magic postgresql-client poppler-utils antiword \ + curl jq openssh-client vim bash + +RUN apk update --update-cache && apk upgrade && \ + apk add --update --no-cache $BUILD_PACKAGES + +RUN apk --update add fontconfig ttf-dejavu && fc-cache -fv + +RUN apk add --no-cache python3 nginx tzdata && \ + python3 -m ensurepip && \ + rm -r /usr/lib/python*/ensurepip && \ + pip3 install --upgrade pip setuptools && \ + rm -r /root/.cache && \ + rm -f /etc/nginx/conf.d/* + +# TODO: move to sapl production image? +RUN mkdir -p /var/interlegis/sapl && mkdir /var/log/sapl +WORKDIR /var/interlegis/sapl/ +COPY . /var/interlegis/sapl/ +RUN pip install -r /var/interlegis/sapl/requirements/dev-requirements.txt --upgrade setuptools && \ + rm -r /root/.cache + + +# SAPL PRODUCTION IMAGE +FROM sapl-base AS sapl + +WORKDIR /var/interlegis/sapl/ + +ENV HOME=/var/interlegis/sapl + +COPY docker/start.sh $HOME +COPY docker/check_solr.sh $HOME +COPY docker/solr_api.py $HOME +COPY docker/busy-wait.sh $HOME +COPY docker/create_admin.py $HOME +COPY docker/genkey.py $HOME +COPY docker/gunicorn_start.sh $HOME + +COPY docker/config/nginx/sapl.conf /etc/nginx/conf.d +COPY docker/config/nginx/nginx.conf /etc/nginx/nginx.conf +COPY docker/config/env_dockerfile /var/interlegis/sapl/sapl/.env + +RUN python3 manage.py collectstatic --noinput --clear + +# Remove .env(fake) e sapl.db da imagem +RUN rm -rf /var/interlegis/sapl/sapl/.env && \ + rm -rf /var/interlegis/sapl/sapl.db + +RUN chmod +x /var/interlegis/sapl/*.sh && \ + ln -sf /dev/stdout /var/log/nginx/access.log && \ + ln -sf /dev/stderr /var/log/nginx/error.log && \ + touch /var/interlegis/sapl/sapl.log && \ + ln -s /var/interlegis/sapl/sapl.log /var/log/sapl/sapl.log + +VOLUME ["/var/interlegis/sapl/data", "/var/interlegis/sapl/media"] + +CMD ["/var/interlegis/sapl/start.sh"] diff --git a/busy-wait.sh b/docker/busy-wait.sh similarity index 80% rename from busy-wait.sh rename to docker/busy-wait.sh index 73fde5c6b..fd00d7744 100644 --- a/busy-wait.sh +++ b/docker/busy-wait.sh @@ -5,6 +5,6 @@ while true; do if ! [ "$COUNT_PG" -eq "0" ]; then break fi - echo "Esperando Database Setup" + echo "Esperando conexão com BD" sleep 10 done diff --git a/check_solr.sh b/docker/check_solr.sh similarity index 100% rename from check_solr.sh rename to docker/check_solr.sh diff --git a/config/env-sample b/docker/config/env-sample similarity index 100% rename from config/env-sample rename to docker/config/env-sample diff --git a/config/env_dockerfile b/docker/config/env_dockerfile similarity index 71% rename from config/env_dockerfile rename to docker/config/env_dockerfile index 134beb274..4b564ee6c 100644 --- a/config/env_dockerfile +++ b/docker/config/env_dockerfile @@ -1,5 +1,5 @@ DATABASE_URL = sqlite:///sapl.db -SECRET_KEY = 'Dockerfile_Key' +SECRET_KEY = 'mzp++@i1y-6y8ez_=^sfbr!dzuyry#^@v(3g^2d1k9%f=+mhlb' DEBUG = False EMAIL_USE_TLS = True EMAIL_PORT = 587 diff --git a/docker/config/env_test b/docker/config/env_test new file mode 100644 index 000000000..4621842cd --- /dev/null +++ b/docker/config/env_test @@ -0,0 +1,9 @@ +DATABASE_URL = postgresql://sapl:sapl@sapldb:5432/sapl +SECRET_KEY=test++@i1y-6y8ez_=^sfbr!dzuyry#^@v(3g^2d1k9%f=+mhlb +DEBUG = True +EMAIL_USE_TLS = True +EMAIL_PORT = 587 +EMAIL_HOST = '' +EMAIL_HOST_USER = '' +EMAIL_SEND_USER = '' +EMAIL_HOST_PASSWORD = '' diff --git a/config/nginx/nginx.conf b/docker/config/nginx/nginx.conf similarity index 100% rename from config/nginx/nginx.conf rename to docker/config/nginx/nginx.conf diff --git a/config/nginx/sapl.conf b/docker/config/nginx/sapl.conf similarity index 100% rename from config/nginx/sapl.conf rename to docker/config/nginx/sapl.conf diff --git a/create_admin.py b/docker/create_admin.py similarity index 100% rename from create_admin.py rename to docker/create_admin.py diff --git a/solr/docker-compose.yml b/docker/docker-compose.test.yml similarity index 51% rename from solr/docker-compose.yml rename to docker/docker-compose.test.yml index 71e25c76c..d2262885a 100644 --- a/solr/docker-compose.yml +++ b/docker/docker-compose.test.yml @@ -1,7 +1,21 @@ -version: '2' +## +## +## ATENÇÃO: A ATUALIZAÇÃO DESTE ARQUIVO EM AMBIENTES DE PRODUÇÃO PODE CAUSAR PERDA DE DADOS +## +## Esta atualização da versão e formato do arquivo docker-compose.yml pode POTENCIALMENTE causar +## a perda de dados. Portanto, aconselha-se a fazer o BACKUP da base de dados do PostgreSQL e +## da pasta 'media', assim como outros arquivos que julgar necessários, mas não expresso neste +## disclaimer por PRECAUÇÃO. +## +## Adicionalmente, temos um novo container (saplsolr) que possui seus volumes que devem ser +## mapeados para um diretório no host da máquina. +## +## +version: "3.5" services: sapldb: image: postgres:10.5-alpine + container_name: postgres restart: always environment: POSTGRES_PASSWORD: sapl @@ -11,10 +25,13 @@ services: volumes: - sapldb_data:/var/lib/postgresql/data/ ports: - - "5432:5432" + - "5433:5432" + networks: + - sapl-net saplsolr: image: solr:7.4-alpine + container_name: solr restart: always command: bin/solr start -c -f volumes: @@ -22,10 +39,18 @@ services: - solr_configsets:/opt/solr/server/solr/configsets ports: - "8983:8983" + networks: + - sapl-net sapl: - image: interlegis/sapl:3.1.152 - # build: . + # image: sapl + build: + context: ../ + dockerfile: ./docker/Dockerfile + # target: sapl + container_name: sapl + labels: + NAME: "sapl" restart: always environment: ADMIN_PASSWORD: interlegis @@ -35,6 +60,7 @@ services: EMAIL_USE_TLS: 'False' EMAIL_HOST: smtp.dominio.net EMAIL_HOST_USER: usuariosmtp + EMAIL_SEND_USER: usuariosmtp EMAIL_HOST_PASSWORD: senhasmtp USE_SOLR: 'True' SOLR_COLLECTION: sapl @@ -43,18 +69,23 @@ services: volumes: - sapl_data:/var/interlegis/sapl/data - sapl_media:/var/interlegis/sapl/media - - sapl_root:/var/interlegis/sapl - volumes_from: + links: + - sapldb - saplsolr depends_on: - sapldb - saplsolr + networks: + - sapl-net ports: - "80:80" volumes: - sapldb_data: - sapl_data: - sapl_media: - sapl_root: - solr_data: - solr_configsets: + sapl_data: + sapl_media: + sapldb_data: + solr_data: + solr_configsets: + +networks: + sapl-net: + driver: bridge diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 000000000..2452c795a --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,91 @@ +## +## +## ATENÇÃO: A ATUALIZAÇÃO DESTE ARQUIVO EM AMBIENTES DE PRODUÇÃO PODE CAUSAR PERDA DE DADOS +## +## Esta atualização da versão e formato do arquivo docker-compose.yml pode POTENCIALMENTE causar +## a perda de dados. Portanto, aconselha-se a fazer o BACKUP da base de dados do PostgreSQL e +## da pasta 'media', assim como outros arquivos que julgar necessários, mas não expresso neste +## disclaimer por PRECAUÇÃO. +## +## Adicionalmente, temos um novo container (saplsolr) que possui seus volumes que devem ser +## mapeados para um diretório no host da máquina. +## +## +version: "3.5" +services: + sapldb: + image: postgres:10.5-alpine + container_name: postgres + restart: always + environment: + POSTGRES_PASSWORD: sapl + POSTGRES_USER: sapl + POSTGRES_DB: sapl + PGDATA : /var/lib/postgresql/data/ + volumes: + - sapldb_data:/var/lib/postgresql/data/ + ports: + - "5433:5432" + networks: + - sapl-net + + saplsolr: + image: solr:7.4-alpine + container_name: solr + restart: always + command: bin/solr start -c -f + volumes: + - solr_data:/opt/solr/server/solr + - solr_configsets:/opt/solr/server/solr/configsets + ports: + - "8983:8983" + networks: + - sapl-net + + sapl: + #image: interlegis/sapl:3.1.159 + build: + context: ../ + dockerfile: ./docker/Dockerfile + # target: sapl + container_name: sapl + labels: + NAME: "sapl" + restart: always + environment: + ADMIN_PASSWORD: interlegis + ADMIN_EMAIL: email@dominio.net + DEBUG: 'False' + EMAIL_PORT: 587 + EMAIL_USE_TLS: 'False' + EMAIL_HOST: smtp.dominio.net + EMAIL_HOST_USER: usuariosmtp + EMAIL_SEND_USER: usuariosmtp + EMAIL_HOST_PASSWORD: senhasmtp + USE_SOLR: 'True' + SOLR_COLLECTION: sapl + SOLR_URL: http://saplsolr:8983 + TZ: America/Sao_Paulo + volumes: + - sapl_data:/var/interlegis/sapl/data + - sapl_media:/var/interlegis/sapl/media + links: + - sapldb + - saplsolr + depends_on: + - sapldb + - saplsolr + networks: + - sapl-net + ports: + - "80:80" +volumes: + sapl_data: + sapl_media: + sapldb_data: + solr_data: + solr_configsets: + +networks: + sapl-net: + driver: bridge diff --git a/docker-env.sh b/docker/docker-env.sh similarity index 100% rename from docker-env.sh rename to docker/docker-env.sh diff --git a/genkey.py b/docker/genkey.py similarity index 100% rename from genkey.py rename to docker/genkey.py diff --git a/gunicorn_start.sh b/docker/gunicorn_start.sh similarity index 100% rename from gunicorn_start.sh rename to docker/gunicorn_start.sh diff --git a/docker/scripts_docker/backup-sapl.sh b/docker/scripts_docker/backup-sapl.sh new file mode 100755 index 000000000..c95945e1d --- /dev/null +++ b/docker/scripts_docker/backup-sapl.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +### +### NÃO ESTÁ BACKUP E RESTAURANDO CORRETAMENTE O BANCO!!!! +### + + +BACKUP_DIR=sapl-backup-`date -u +%Y-%m-%d-%H-%M` +mkdir -p ./$BACKUP_DIR + +# BACKUP DO BANCO DE DADOS +sudo docker exec -it postgres bash -c 'pg_dump -U sapl -d sapl -Fc -v > /tmp/database.backup' +sudo docker cp postgres:/tmp/database.backup ./$BACKUP_DIR + +# BACKUP DA PASTA MEDIA +sudo docker exec -it sapl bash -c 'tar -cvzf media.tar.gz media' +sudo docker cp sapl:/var/interlegis/sapl/media.tar.gz ./$BACKUP_DIR diff --git a/scripts_docker/remove-all-containers.sh b/docker/scripts_docker/remove-all-containers.sh similarity index 100% rename from scripts_docker/remove-all-containers.sh rename to docker/scripts_docker/remove-all-containers.sh diff --git a/scripts_docker/remove-db.sh b/docker/scripts_docker/remove-db.sh similarity index 100% rename from scripts_docker/remove-db.sh rename to docker/scripts_docker/remove-db.sh diff --git a/scripts_docker/restore-db.sh b/docker/scripts_docker/restore-db.sh similarity index 100% rename from scripts_docker/restore-db.sh rename to docker/scripts_docker/restore-db.sh diff --git a/docker/scripts_docker/restore-sapl.sh b/docker/scripts_docker/restore-sapl.sh new file mode 100755 index 000000000..4f5adbf1f --- /dev/null +++ b/docker/scripts_docker/restore-sapl.sh @@ -0,0 +1,26 @@ +#!/bin/bash + + +### +### NÃO ESTÁ RESTAURANDO CORRETAMENTE O BANCO!!!! +### + +BACKUP_DIR=$1 +if [ -z $BACKUP_DIR ]; then + echo "Diretório de backup não informado!" + echo "uso: $./restore-sapl.sh " + exit 1 +fi + +# RESTORE DO BANCO DE DADOS +sudo docker cp ./$BACKUP_DIR/database.backup postgres:/tmp/ +sudo docker exec -it postgres bash -c 'ls -lah /tmp/database.backup' + +# --clean --data-only --disable-trigger ???? +sudo docker exec -it postgres bash -c 'pg_restore --disable-triggers --data-only -Fc -v -U sapl -d sapl /tmp/database.backup' + +# RESTORE DA PASTA MEDIA +sudo docker cp ./$BACKUP_DIR/media.tar.gz sapl:/var/interlegis/sapl +sudo docker exec -it sapl bash -c 'ls -lah /var/interlegis/sapl/media.tar.gz' +#sudo docker exec -it sapl bash -c 'cd /var/interlegis/sapl && tar -zxvf media.tar.gz .' + diff --git a/scripts_docker/shell_sapl.sh b/docker/scripts_docker/shell_sapl.sh similarity index 100% rename from scripts_docker/shell_sapl.sh rename to docker/scripts_docker/shell_sapl.sh diff --git a/simple_gunicorn.sh b/docker/simple_gunicorn.sh similarity index 100% rename from simple_gunicorn.sh rename to docker/simple_gunicorn.sh diff --git a/solr_api.py b/docker/solr_api.py similarity index 100% rename from solr_api.py rename to docker/solr_api.py diff --git a/start.sh b/docker/start.sh similarity index 99% rename from start.sh rename to docker/start.sh index bea1eb959..ce259497c 100755 --- a/start.sh +++ b/docker/start.sh @@ -46,7 +46,7 @@ create_env() { create_env -/bin/bash busy-wait.sh $DATABASE_URL +/bin/bash wait-for-it.sh yes yes | python3 manage.py migrate diff --git a/docker/wait-for-it.sh b/docker/wait-for-it.sh new file mode 100644 index 000000000..998b757f7 --- /dev/null +++ b/docker/wait-for-it.sh @@ -0,0 +1,20 @@ + +#!/bin/sh +# wait until MySQL is really available +maxcounter=${MAX_DB_CONN_ATTEMPTS:-10} +echo "Trying to connect to PostgreSQL, max attempts="$maxcounter + +counter=1 +while ! mysql --host="$DATABASE_URL" --protocol TCP -u"$POSTGRES_USER" -p"$POSTGRES_PASSWORD" -e "show databases;" > /dev/null 2>&1; do + sleep 1 + counter=`expr $counter + 1` + if [ $counter -gt $maxcounter ]; then + >&2 echo "We have been waiting for PostgreSQL too long already; failing." + exit 1 + fi; +done +echo "-=------------------------------------------------------" +echo "-=------------------------------------------------------" +echo "Connected to PostgreSQL!" +echo "-=------------------------------------------------------" +echo "-=------------------------------------------------------" diff --git a/release.sh b/release.sh index b99dbf05c..574d35048 100755 --- a/release.sh +++ b/release.sh @@ -30,11 +30,11 @@ FINAL_VERSION= function change_files { - OLD_VERSION=$(grep -E 'interlegis/sapl:'$VERSION_PATTERN docker-compose.yml | cut -d':' -f3) + OLD_VERSION=$(grep -E 'interlegis/sapl:'$VERSION_PATTERN docker/docker-compose.yml | cut -d':' -f3) echo "Atualizando de "$OLD_VERSION" para "$FINAL_VERSION - sed -E -i "s|$OLD_VERSION|$FINAL_VERSION|g" docker-compose.yml + sed -E -i "s|$OLD_VERSION|$FINAL_VERSION|g" docker/docker-compose.yml sed -E -i "s|$OLD_VERSION|$FINAL_VERSION|g" setup.py