Refatora Solr API (#3322)

* Refatora solr_api

* Conserta bug ao zipar subdir

* Remove zip de controle de versão

* Adiciona try/catch nos res.json()

* Faz o zip de subdir

* Mais mudanças

* WIP

* Conserta bug em zip file

* Remoção de linhas de debug

* Adiciona create-collection.sh

* Adiciona upload-config.sh

* Adiciona delete-collection.sh

* Adiciona delete-config.sh

* Adiciona delete-records.sh

* Adiciona list-collections-configs.sh

* Adiciona update-configset.sh

* Update and rename delete-records.sh to delete-all-documents.sh

* Update update-configset.sh

Co-authored-by: eribeiro <edwardr@senado.leg.br>

docker/solr_api.py

@@ -1,26 +1,37 @@
-
+from io import BytesIO
+import argparse
+import os
 import requests
 import subprocess
 import sys
-import argparse
+import zipfile
+from pathlib import Path
 
+##
+## Este módulo deve ser executado na raiz do projeto
+##
 
 class SolrClient:
 
     LIST_CONFIGSETS = "{}/solr/admin/configs?action=LIST&omitHeader=true&wt=json"
     UPLOAD_CONFIGSET = "{}/solr/admin/configs?action=UPLOAD&name={}&wt=json"
     LIST_COLLECTIONS = "{}/solr/admin/collections?action=LIST&wt=json"
-    STATUS_COLLECTION = "{}/solr/admin/collections?action=CLUSTERSTATUS&collection={}&wt=json"
+    STATUS_COLLECTION = "{}/solr/admin/collections?action=CLUSTERSTATUS" \
+                        "&collection={}&wt=json"
     STATUS_CORE = "{}/admin/cores?action=STATUS&name={}"
     EXISTS_COLLECTION = "{}/solr/{}/admin/ping?wt=json"
     OPTIMIZE_COLLECTION = "{}/solr/{}/update?optimize=true&wt=json"
-    CREATE_COLLECTION = "{}/solr/admin/collections?action=CREATE&name={}&collection.configName={}&numShards={}&replicationFactor={}&maxShardsPerNode={}&wt=json"
+    CREATE_COLLECTION = "{}/solr/admin/collections?action=CREATE&name={}" \
+                        "&collection.configName={}&numShards={}" \
+                        "&replicationFactor={}&maxShardsPerNode={}&wt=json"
     DELETE_COLLECTION = "{}/solr/admin/collections?action=DELETE&name={}&wt=json"
     DELETE_DATA = "{}/solr/{}/update?commitWithin=1000&overwrite=true&wt=json"
     QUERY_DATA = "{}/solr/{}/select?q=*:*"
 
     CONFIGSET_NAME = "sapl_configset"
 
+    CONFIGSET_PATH = "./solr/sapl_configset/conf"
+
     def __init__(self, url):
         self.url = url
 
@@ -32,7 +43,7 @@ class SolrClient:
                 dic = res.json()
                 return dic["response"]["numFound"]
             except Exception as e:
-                print(F"Erro no get_num_docs: {e}")
+                print(F"Erro no get_num_docs. Erro: {e}")
                 print(res.content)
 
         return 0
@@ -40,23 +51,52 @@ class SolrClient:
     def list_collections(self):
         req_url = self.LIST_COLLECTIONS.format(self.url)
         res = requests.get(req_url)
-        dic = res.json()
+        try:
-        return dic['collections']
+            dic = res.json()
+            return dic['collections']
+        except Exception as e:
+            print(F"Erro no list_collections. Erro: {e}")
+            print(res.content)
+            return 0
 
     def exists_collection(self, collection_name):
         collections = self.list_collections()
         return True if collection_name in collections else False
 
+    def zip_configset(self):
+        try:
+            base_path = Path(self.CONFIGSET_PATH).expanduser().resolve(strict=True)
+
+            # zip files in memory
+            _zipfile = BytesIO()
+            with zipfile.ZipFile(_zipfile, 'w', zipfile.ZIP_DEFLATED) as zipf:
+                for file in base_path.rglob('*'):
+                    zipf.write(file, file.relative_to(base_path))
+            return _zipfile
+        except Exception as e:
+            print(e)
+            raise e
+
     def maybe_upload_configset(self, force=False):
         req_url = self.LIST_CONFIGSETS.format(self.url)
         res = requests.get(req_url)
-        dic = res.json()
+        try:
-        configsets = dic['configSets']
+            dic = res.json()
+            configsets = dic['configSets']
+        except Exception as e:
+            print(F"Erro ao configurar configsets. Erro: {e}")
+            print(res.content)
+
         # UPLOAD configset
         if not self.CONFIGSET_NAME in configsets or force:
+
+            # GENERATE in memory configset
+            configset_zip = self.zip_configset()
+            data = configset_zip.getvalue()
+            configset_zip.close()
+
             files = {'file': ('saplconfigset.zip',
-                              open('./solr/sapl_configset/conf/saplconfigset.zip',
+                              data,
-                                   'rb'),
                               'application/octet-stream',
                               {'Expires': '0'})}
 
@@ -64,6 +104,7 @@ class SolrClient:
 
             resp = requests.post(req_url, files=files)
             print(resp.content)
+
         else:
             print('O %s já presente no servidor, NÃO enviando.' % self.CONFIGSET_NAME)
 
@@ -80,8 +121,12 @@ class SolrClient:
             print("Collection '%s' created succesfully" % collection_name)
         else:
             print("Error creating collection '%s'" % collection_name)
-            as_json = res.json()
+            try:
-            print("Error %s: %s" % (res.status_code, as_json['error']['msg']))
+                as_json = res.json()
+                print("Error %s: %s" % (res.status_code, as_json['error']['msg']))
+            except Exception as e:
+                print(F"Erro ao verificar erro na resposta. Erro: {e}")
+                print(res.content)
             return False
         return True
     

solr/bin/create-collection.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+SOLR_URL=${SOLR_URL-'http://localhost:8983/solr'}
+SOLR_COLLECTION=${SOLR_COLLECTION-'sapl'}
+
+curl -X POST "$SOLR_URL/admin/collections?action=CREATE&name=$SOLR_COLLECTION&numShards=1&replicationFactor=1&collection.configName=sapl_configset"

solr/bin/delete-all-documents.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+SOLR_URL=${SOLR_URL-'http://localhost:8983/solr'}
+SOLR_COLLECTION=${SOLR_COLLECTION-'sapl'}
+
+curl -X POST "$SOLR_URL/$SOLR_COLLECTION/update?commit=true" -H "Content-Type: text/xml" --data-binary '<delete><query>*:*</query></delete>'

solr/bin/delete-collection.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+SOLR_URL=${SOLR_URL-'http://localhost:8983/solr'}
+SOLR_COLLECTION=${SOLR_COLLECTION-'sapl'}
+
+curl -X POST "$SOLR_URL/admin/collections?action=DELETE&name=$SOLR_COLLECTION"

solr/bin/delete-config.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+SOLR_URL=${SOLR_URL-'http://localhost:8983/solr'}
+
+curl -X POST "$SOLR_URL/admin/configs?action=DELETE&name=sapl_configset&omitHeader=true"

solr/bin/list-collections-configs.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+SOLR_URL=${SOLR_URL-'http://localhost:8983/solr'}
+
+curl -X GET "$SOLR_URL/admin/collections?action=LIST"
+
+curl -X GET "$SOLR_URL/admin/configs?action=LIST"

solr/bin/update-configset.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+if [[ -z $SOLR_HOME ]]; then
+  echo 'Cannot run script! You need to setup $SOLR_HOME'
+  exit
+fi
+
+ZK_HOST=${ZK_HOST-'localhost:9983'}
+
+$SOLR_HOME/bin/solr zk upconfig -n sapl_configset -d solr/sapl_configset/ -z $ZK_HOST

solr/bin/upload-config.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+SOLR_URL=${SOLR_URL-'http://localhost:8983/solr'}
+
+# zip configset sapl_configset
+cd ../sapl_configset/conf && zip -r sapl_configset.zip .
+
+curl -X POST --header "Content-Type:application/octet-stream" --data-binary @sapl_configset.zip "$SOLR_URL/admin/configs?action=UPLOAD&name=sapl_configset"
+
+cd -
+rm ../sapl_configset/conf/sapl_configset.zip