Add s3 files processing

2024-02-12 20:08:58 +03:00
parent 90f7965242
commit 2898bfd7a2
5 changed files with 238 additions and 38 deletions
--- a/4
+++ b/4
@@ -4,7 +4,9 @@ COPY . ./
 RUN cd deps/pygost-5.12/ && \
    python3 setup.py install && \
    cd ../.. && \
-    pip3 install -r requirements.txt
+    pip3 install -r requirements.txt && \
+    mkdir -p /opt/tnt/bin && \
+    ln -s /usr/bin/python3 /opt/tnt/bin/python3
 ENV LD_LIBRARY_PATH "/app"
 ENV PYTHONPATH "${PYTHONPATH}:/app"
 EXPOSE 9000
--- a/config.py
+++ b/config.py
@@ -1,6 +1,6 @@

 class Config:
-    ret_path: str = 'http://10.10.8.60:9000/'
+    ret_path: str = 'http://10.10.8.81:9000/'
    self_bnd: str = 'bnd127'
    enserver: str = 'http://127.0.0.1:7000/xmlrpc'
    remote_bnd: str = 'bnd128'
--- a/config/response2.json
+++ b/config/response2.json
@@ -0,0 +1,12 @@
+{
+  "filestorage": {
+    "type": "s3",
+    "endpoint": "http://10.10.8.83:31006",
+    "key_id": "s57",
+    "access_key": "d9MMinLF3U8TLSj",
+    "download_path": "/tmp"
+  },
+  "file_code": "c1000",
+  "name_code": "c122",
+  "use_version": true
+}
--- a/config/workspaces.json
+++ b/config/workspaces.json
@@ -0,0 +1,167 @@
+{
+  "databases": {
+    "oodb_git": {
+      "host": "10.10.8.83",
+      "port": 32100,
+      "database": "db",
+      "user": "postgres",
+      "password": "Root12345678"
+    }
+  },
+  "workspaces": {
+    "documents_src": {
+      "type": "documents",
+      "group": "src",
+      "database": "oodb_git",
+      "schema": "documents_src",
+      "alias": "Документы исходная"
+    },
+    "documents_standard": {
+      "type": "documents",
+      "group": "order",
+      "database": "oodb_git",
+      "schema": "documents_standard",
+      "alias": "Документы эталон"
+    },
+    "documents_standard_pub": {
+      "type": "documents",
+      "group": "order",
+      "database": "oodb_git",
+      "schema": "documents_standard",
+      "alias": "Документы публичная"
+    },
+    "ood": {
+      "type": "npd",
+      "group": "src",
+      "database": "oodb_git",
+      "schema": "ood",
+      "alias": "ООБД исходные НПД",
+      "map_service": "VUE_APP_GISAIS_URL:/styles/ood/style.json"
+    },
+    "oodb": {
+      "type": "oodb",
+      "group": "order",
+      "database": "oodb_git",
+      "schema": "kartap",
+      "alias": "ООДБ эталон",
+      "map_service": "VUE_APP_GISAIS_URL:/styles/oodb/style.json"
+    },
+    "oodb_standard": {
+      "type": "oodb",
+      "group": "forming_standard",
+      "database": "oodb_git",
+      "schema": "kartap",
+      "alias": "ООДБ эталон",
+      "map_service": "VUE_APP_GISAIS_URL:/styles/oodb/style.json"
+    },
+    "oodb_working": {
+      "type": "oodb",
+      "group": "forming_work",
+      "database": "oodb_git",
+      "schema": "kartap",
+      "alias": "ООДБ рабочая",
+      "map_service": "VUE_APP_GISAIS_URL:/styles/oodb_tech/style.json"
+    },
+    "oodb_pub": {
+      "type": "oodb",
+      "group": "order",
+      "database": "oodb_git",
+      "schema": "kartap",
+      "alias": "ООБД публичная",
+      "map_service": "VUE_APP_GISAIS_URL:/styles/oodb/style.json"
+    },
+    "regions": {
+      "type": "regions",
+      "database": "oodb_git",
+      "schema": "regions_hard",
+      "alias": "Регионы",
+      "map_service": "VUE_APP_GISAIS_URL_GK:/styles/regions/style.json"
+    },
+    "regions_contour": {
+      "type": "regions",
+      "database": "oodb_git",
+      "schema": "regions_hard",
+      "alias": "Регионы",
+      "map_service": "VUE_APP_GISAIS_URL_GK:/styles/regions_contour/style.json"
+    },
+    "npd_9": {
+      "type": "npd",
+      "database": "oodb_git",
+      "schema": "npd_9",
+      "alias": "НПД 9.0"
+    },
+    "npd": {
+      "type": "npd",
+      "database": "oodb_git",
+      "schema": "initial",
+      "alias": "НПД 9.0"
+    },
+    "npd_831": {
+      "type": "npd",
+      "group": "order",
+      "database": "oodb_git",
+      "schema": "npd_831",
+      "alias": "НПД 8.31"
+    },
+    "updater_test": {
+      "type": "npd",
+      "group": "order",
+      "database": "oodb_git",
+      "schema": "npd_831_test",
+      "alias": "НПД 8.31 публичная"
+    },
+    "lukoil": {
+      "type": "oodb",
+      "database": "oodb_git",
+      "schema": "lukoil",
+      "alias": "ЛУКОЙЛ",
+      "map_service": "VUE_APP_GISAIS_URL_GK:/styles/lukoil/style.json"
+    },
+    "geocover": {
+      "type": "ecpz",
+      "group": "order",
+      "database": "oodb_git",
+      "schema": "coverage",
+      "alias": "ЕЦПЗ"
+    },
+    "geocover_test": {
+      "type": "ecpz",
+      "database": "oodb_git",
+      "schema": "coverage",
+      "alias": "ЕЦПЗ тест"
+    },
+    "gcmr": {
+      "type": "gcmr",
+      "group": "order",
+      "database": "oodb_git",
+      "schema": "gcmr",
+      "alias": "ГЦМР"
+    },
+    "orders": {
+      "type": "system",
+      "database": "oodb_git",
+      "schema": "orders",
+      "alias": "Заказы"
+    },
+    "ilo": {
+      "type": "system",
+      "database": "oodb_git",
+      "schema": "ilo",
+      "alias": "ИЛО"
+    },
+    "raz_sgok": {
+      "type": "raz_sgok",
+      "database": "razsgok",
+      "schema": "razsgok",
+      "alias": "СГОК",
+      "map_service": "VUE_APP_GISAIS_URL_GK:/styles/raz_sgok/style.json"
+    },
+    "raz_vtu": {
+      "type": "raz_vtu",
+      "database": "razvtu",
+      "schema": "razvtu",
+      "alias": "ВТУ",
+      "map_service": "VUE_APP_GISAIS_URL_GK:/styles/raz_vtu/style.json"
+    }
+  }
+} 
--- a/main.py
+++ b/main.py
@@ -29,6 +29,7 @@ import uvicorn
 from typing_extensions import Annotated
 import pathlib
 from infra import *
+from shutil import make_archive


 NEW_REPLICATION_REQUEST = 99
@@ -672,37 +673,39 @@ def query_commits(params, files, url):


 def get_data(params, files, url):
+    date = datetime.datetime.now()
+    rxmls = RequestXmlService()
    req = ET.fromstring(params['query_data'])
-    commit_el = req.find('commit')
-    commit_id = commit_el.get('id')
-    schema = commit_el.get('schema')
-    bnd = params['from'].replace('tcp://', '')
+    req_id = rxmls.get_request_uuid(req)
+    res_id = uuid4().hex
+    res = rxmls.get_request_document(res_id, req_id)
+    rxmls.set_result(res, 0, '')
+    request_string = req.find('data').text

-    con = OOConnectionParams(schema, Config.oodb_host, Config.oodb_port, Config.oodb_dbname,
-                             Config.oodb_username, Config.oodb_passwd, schema)
-    ws = OODBWorkspace.ws(schema)
-    if not ws.isInit():
-        res = ws.init(con)
-        logger.warning(res)
-    schema_commits = ws.commits(ws.branch())
-    logger.warning(schema_commits)
-    if commit_id not in schema_commits:
-        logger.warning(f'Error in commits in schema {schema}: no commit {commit_id}')
-        return
-    logger.warning(schema_commits[schema_commits.index(commit_id) + 1:])
+    OODBWorkspaceFactory.init('config/workspaces.json')
+    conf = ResponseWorkerConfig('config/response2.json', 'config/workspaces.json')
+    worker = ResponseWorker(conf)
+    fn = uuid4().hex
+    dir = os.path.join('tmp', fn)
+    os.makedirs(dir)
+    worker.makeResponse(request_string, dir)
+    make_archive(dir, 'zip', dir, '.')

-    conn = db.connect_db()
-    with Session(conn) as session:
-        for commit in schema_commits[schema_commits.index(commit_id) + 1:]:
-            for user in session.query(db.User).filter(db.User.bndname == bnd, db.User.active == True).all():
-                if user.bndname == Config.self_bnd:
-                    continue
-                profiles = {x.scheme: x.to_dict() for x in user.profiles}
-                if len(profiles) == 0 or schema in profiles:
-                    item = db.Queue(user_id=user.id, commit_id=commit, schema=schema)
-                    logging.warning(item)
-                    session.add(item)
-        session.commit()
+    rxmls.set_result(res, 0, '')
+    response_params = {
+        'from': params['to'],
+        'to': params['from'],
+        'ts_added': date.timestamp(),
+        'user_id': '1',
+        'user_id_to': params['user_id'],
+        'query_type': NEW_DATA_RESPONSE,
+        'query_data': ET.tostring(res, encoding='unicode', xml_declaration=True)
+    }
+    filename = fn + '.zip'
+    filepath = os.path.join(os.getcwd(), 'tmp', filename)
+    response_files = [{'name': filename, 'url': filepath, 'size': os.path.getsize(filepath)}]
+    proxy = ServerProxy(url)
+    proxy.send(response_params, response_files, Config.ret_path)


 def receive_data(params, files, url):
@@ -926,12 +929,12 @@ def test():
    #get_metadata(params, files, url)
    #get_catalog()
    # auth_response('123', 'bnd127', False)
-    con = OOConnectionParams(Config.oodb_schema, Config.oodb_host, Config.oodb_port, Config.oodb_dbname,
-                             Config.oodb_username, Config.oodb_passwd, Config.oodb_schema)
-    ws = OODBWorkspace.ws(Config.oodb_schema)
-    ws.init(con)
-    print(ws.currentCommit())
-    print(ws.commits(ws.branch()))
+    # con = OOConnectionParams(Config.oodb_schema, Config.oodb_host, Config.oodb_port, Config.oodb_dbname,
+    #                          Config.oodb_username, Config.oodb_passwd, Config.oodb_schema)
+    # ws = OODBWorkspace.ws(Config.oodb_schema)
+    # ws.init(con)
+    # print(ws.currentCommit())
+    # print(ws.commits(ws.branch()))
    # created, updated, deleted = ws.changes('2dad8c8a-d2db-4074-ab7a-c01c36ada2be')
    # qu = GroupQuery(Envelope())
    # qu.setUids(updated)
@@ -952,9 +955,25 @@ def test():
    # ws.clearData(True)
    # ws.close()
    #replication_old('bnd128', '23c9a275-ec0f-481a-8437-f3e41e4fe4f5', 'documents_src')
+    request_string = """
+    {
+  "type": "group_query",
+  "data_source": "npd",
+  "uids": [ "2e20130c-541a-4b9f-9efb-f2a0e8b10c33" ]
+}
+    """
+    OODBWorkspaceFactory.init('config/workspaces.json')
+    conf = ResponseWorkerConfig('config/response2.json', 'config/workspaces.json')
+    worker = ResponseWorker(conf)
+    fn = uuid4().hex
+    dir = os.path.join('tmp', fn)
+    os.makedirs(dir)
+    worker.makeResponse(request_string, dir)
+    make_archive(dir, 'zip', dir, '.')
+
    pass


 if __name__ == '__main__':
-    main()
-    # test()
+    # main()
+    test()