diff --git a/docker/solr_api.py b/docker/solr_api.py index 7713619a2..3ca06bba7 100755 --- a/docker/solr_api.py +++ b/docker/solr_api.py @@ -1,26 +1,37 @@ - +from io import BytesIO +import argparse +import os import requests import subprocess import sys -import argparse +import zipfile +from pathlib import Path +## +## Este módulo deve ser executado na raiz do projeto +## class SolrClient: LIST_CONFIGSETS = "{}/solr/admin/configs?action=LIST&omitHeader=true&wt=json" UPLOAD_CONFIGSET = "{}/solr/admin/configs?action=UPLOAD&name={}&wt=json" LIST_COLLECTIONS = "{}/solr/admin/collections?action=LIST&wt=json" - STATUS_COLLECTION = "{}/solr/admin/collections?action=CLUSTERSTATUS&collection={}&wt=json" + STATUS_COLLECTION = "{}/solr/admin/collections?action=CLUSTERSTATUS" \ + "&collection={}&wt=json" STATUS_CORE = "{}/admin/cores?action=STATUS&name={}" EXISTS_COLLECTION = "{}/solr/{}/admin/ping?wt=json" OPTIMIZE_COLLECTION = "{}/solr/{}/update?optimize=true&wt=json" - CREATE_COLLECTION = "{}/solr/admin/collections?action=CREATE&name={}&collection.configName={}&numShards={}&replicationFactor={}&maxShardsPerNode={}&wt=json" + CREATE_COLLECTION = "{}/solr/admin/collections?action=CREATE&name={}" \ + "&collection.configName={}&numShards={}" \ + "&replicationFactor={}&maxShardsPerNode={}&wt=json" DELETE_COLLECTION = "{}/solr/admin/collections?action=DELETE&name={}&wt=json" DELETE_DATA = "{}/solr/{}/update?commitWithin=1000&overwrite=true&wt=json" QUERY_DATA = "{}/solr/{}/select?q=*:*" CONFIGSET_NAME = "sapl_configset" + CONFIGSET_PATH = "./solr/sapl_configset/conf" + def __init__(self, url): self.url = url @@ -32,7 +43,7 @@ class SolrClient: dic = res.json() return dic["response"]["numFound"] except Exception as e: - print(F"Erro no get_num_docs: {e}") + print(F"Erro no get_num_docs. Erro: {e}") print(res.content) return 0 @@ -40,23 +51,52 @@ class SolrClient: def list_collections(self): req_url = self.LIST_COLLECTIONS.format(self.url) res = requests.get(req_url) - dic = res.json() - return dic['collections'] + try: + dic = res.json() + return dic['collections'] + except Exception as e: + print(F"Erro no list_collections. Erro: {e}") + print(res.content) + return 0 def exists_collection(self, collection_name): collections = self.list_collections() return True if collection_name in collections else False + def zip_configset(self): + try: + base_path = Path(self.CONFIGSET_PATH).expanduser().resolve(strict=True) + + # zip files in memory + _zipfile = BytesIO() + with zipfile.ZipFile(_zipfile, 'w', zipfile.ZIP_DEFLATED) as zipf: + for file in base_path.rglob('*'): + zipf.write(file, file.relative_to(base_path)) + return _zipfile + except Exception as e: + print(e) + raise e + def maybe_upload_configset(self, force=False): req_url = self.LIST_CONFIGSETS.format(self.url) res = requests.get(req_url) - dic = res.json() - configsets = dic['configSets'] + try: + dic = res.json() + configsets = dic['configSets'] + except Exception as e: + print(F"Erro ao configurar configsets. Erro: {e}") + print(res.content) + # UPLOAD configset if not self.CONFIGSET_NAME in configsets or force: + + # GENERATE in memory configset + configset_zip = self.zip_configset() + data = configset_zip.getvalue() + configset_zip.close() + files = {'file': ('saplconfigset.zip', - open('./solr/sapl_configset/conf/saplconfigset.zip', - 'rb'), + data, 'application/octet-stream', {'Expires': '0'})} @@ -64,6 +104,7 @@ class SolrClient: resp = requests.post(req_url, files=files) print(resp.content) + else: print('O %s já presente no servidor, NÃO enviando.' % self.CONFIGSET_NAME) @@ -80,8 +121,12 @@ class SolrClient: print("Collection '%s' created succesfully" % collection_name) else: print("Error creating collection '%s'" % collection_name) - as_json = res.json() - print("Error %s: %s" % (res.status_code, as_json['error']['msg'])) + try: + as_json = res.json() + print("Error %s: %s" % (res.status_code, as_json['error']['msg'])) + except Exception as e: + print(F"Erro ao verificar erro na resposta. Erro: {e}") + print(res.content) return False return True diff --git a/solr/bin/create-collection.sh b/solr/bin/create-collection.sh new file mode 100644 index 000000000..3288ae43c --- /dev/null +++ b/solr/bin/create-collection.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +SOLR_URL=${SOLR_URL-'http://localhost:8983/solr'} +SOLR_COLLECTION=${SOLR_COLLECTION-'sapl'} + +curl -X POST "$SOLR_URL/admin/collections?action=CREATE&name=$SOLR_COLLECTION&numShards=1&replicationFactor=1&collection.configName=sapl_configset" \ No newline at end of file diff --git a/solr/bin/delete-all-documents.sh b/solr/bin/delete-all-documents.sh new file mode 100644 index 000000000..c9a7b09cf --- /dev/null +++ b/solr/bin/delete-all-documents.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +SOLR_URL=${SOLR_URL-'http://localhost:8983/solr'} +SOLR_COLLECTION=${SOLR_COLLECTION-'sapl'} + +curl -X POST "$SOLR_URL/$SOLR_COLLECTION/update?commit=true" -H "Content-Type: text/xml" --data-binary '*:*' diff --git a/solr/bin/delete-collection.sh b/solr/bin/delete-collection.sh new file mode 100644 index 000000000..9e4fb466e --- /dev/null +++ b/solr/bin/delete-collection.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +SOLR_URL=${SOLR_URL-'http://localhost:8983/solr'} +SOLR_COLLECTION=${SOLR_COLLECTION-'sapl'} + +curl -X POST "$SOLR_URL/admin/collections?action=DELETE&name=$SOLR_COLLECTION" \ No newline at end of file diff --git a/solr/bin/delete-config.sh b/solr/bin/delete-config.sh new file mode 100644 index 000000000..b8ca38823 --- /dev/null +++ b/solr/bin/delete-config.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +SOLR_URL=${SOLR_URL-'http://localhost:8983/solr'} + +curl -X POST "$SOLR_URL/admin/configs?action=DELETE&name=sapl_configset&omitHeader=true" \ No newline at end of file diff --git a/solr/bin/list-collections-configs.sh b/solr/bin/list-collections-configs.sh new file mode 100644 index 000000000..32bcb9e43 --- /dev/null +++ b/solr/bin/list-collections-configs.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +SOLR_URL=${SOLR_URL-'http://localhost:8983/solr'} + +curl -X GET "$SOLR_URL/admin/collections?action=LIST" + +curl -X GET "$SOLR_URL/admin/configs?action=LIST" \ No newline at end of file diff --git a/solr/bin/update-configset.sh b/solr/bin/update-configset.sh new file mode 100644 index 000000000..52861868b --- /dev/null +++ b/solr/bin/update-configset.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +if [[ -z $SOLR_HOME ]]; then + echo 'Cannot run script! You need to setup $SOLR_HOME' + exit +fi + +ZK_HOST=${ZK_HOST-'localhost:9983'} + +$SOLR_HOME/bin/solr zk upconfig -n sapl_configset -d solr/sapl_configset/ -z $ZK_HOST diff --git a/solr/bin/upload-config.sh b/solr/bin/upload-config.sh new file mode 100644 index 000000000..8d1d7ba36 --- /dev/null +++ b/solr/bin/upload-config.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +SOLR_URL=${SOLR_URL-'http://localhost:8983/solr'} + +# zip configset sapl_configset +cd ../sapl_configset/conf && zip -r sapl_configset.zip . + +curl -X POST --header "Content-Type:application/octet-stream" --data-binary @sapl_configset.zip "$SOLR_URL/admin/configs?action=UPLOAD&name=sapl_configset" + +cd - +rm ../sapl_configset/conf/sapl_configset.zip \ No newline at end of file diff --git a/solr/sapl_configset/conf/saplconfigset.zip b/solr/sapl_configset/conf/saplconfigset.zip deleted file mode 100644 index 2bd9c45f3..000000000 Binary files a/solr/sapl_configset/conf/saplconfigset.zip and /dev/null differ