Skip to content

Commit 0cbe9a4

Browse files
RICHARD-QuentinRICHARD-Quentin
authored
fix populate builds and script (#25)
1 parent c62da7b commit 0cbe9a4

File tree

5 files changed

+97
-78
lines changed

5 files changed

+97
-78
lines changed

‎Makefile‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -62,11 +62,14 @@ stop-dbs: ## Stop databases
6262
@docker stop $(APPNAME)-mongodb || true
6363
@docker stop $(APPNAME)-minio || true
6464

65-
buildseeds: ## Build populate image
66-
docker build -t seeding -f ./populate/Dockerfile ./populate
65+
build_from_data: ## Build populate image
66+
docker build -t seeding_from_data -f ./populate/populate_data/Dockerfile ./populate
67+
68+
build_from_gaiasys: ## Build populate image
69+
docker build -t seeding_from_gaiasys -f ./populate/populate_from_gaiasys/Dockerfile ./populate
6770

6871
populate-db: ## Populate database
69-
docker run --rm --network dev -v $(CURDIR)/populate:/app -v $(CURDIR)/service/.dev-token.json:/app/.dev-token.json seeding
72+
docker run --rm --network dev -v $(CURDIR)/populate:/app -v $(CURDIR)/service/.dev-token.json:/app/.dev-token.json seeding_from_data
7073

7174
populate-from-gaiasys: ## Populate database
72-
docker run --rm --network dev -v $(CURDIR)/populate:/app -v $(CURDIR)/service/.dev-token.json:/app/.dev-token.json seeding python populate_from_gaiasys.py
75+
docker run --rm --network dev -v $(CURDIR)/populate:/app -v $(CURDIR)/service/.dev-token.json:/app/.dev-token.json seeding_from_gaiasys

populate/Dockerfile renamed to populate/populate_data/Dockerfile

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,9 @@ FROM python:3.8
55
WORKDIR /app
66

77
# Copy the current directory contents into the container at /app
8-
COPY ./ /app
8+
COPY ../. /app
99

1010
# Install any needed packages specified in requirements.txt
11-
RUN pip install --trusted-host pypi.python.org requests python-dotenv motor
11+
RUN pip install --no-cache-dir -r requirements.txt
1212

13-
# Run populate_data.py when the container launches
14-
CMD ["python", "populate_data.py"]
13+
CMD [ "python", "populate_data.py" ]

‎populate/populate_from_gaiasys.py‎

Lines changed: 69 additions & 70 deletions
Original file line numberDiff line numberDiff line change
@@ -2,18 +2,17 @@
22
import json
33
import asyncio
44

5-
import bson
6-
import paramiko
5+
from paramiko import SSHClient, AutoAddPolicy
76
from logging import Logger
87
from datetime import datetime
9-
import re
8+
from re import match
109

11-
import motor.motor_asyncio
10+
from motor.motor_asyncio import AsyncIOMotorClient
1211
from dotenv import load_dotenv
1312
from pymongo import MongoClient
1413
import logging
15-
import base64
16-
import requests
14+
from base64 import b64decode
15+
from requests import get, post, exceptions
1716
from bson import ObjectId
1817

1918
# Configure logging
@@ -22,8 +21,8 @@
2221
# Load environment variables from .env file
2322
load_dotenv()
2423

25-
ssh = paramiko.SSHClient()
26-
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
24+
ssh = SSHClient()
25+
ssh.set_missing_host_key_policy(AutoAddPolicy())
2726

2827
ssh.connect(
2928
os.getenv("IMPORT_SSH_HOSTNAME"),
@@ -36,7 +35,7 @@
3635
mongo_url = os.getenv("MONGODB_URL")
3736
token_path = os.path.join(os.path.dirname(__file__), '.dev-token.json')
3837
import_mongo_url = os.getenv("IMPORT_MONGO_DB")
39-
client = motor.motor_asyncio.AsyncIOMotorClient(import_mongo_url)
38+
client = AsyncIOMotorClient(import_mongo_url)
4039

4140
# sequence = LT000001
4241
# sequence = IMP00001
@@ -74,10 +73,10 @@ async def populate_collection_sla(collection, headers):
7473
}
7574
}
7675
try:
77-
response = requests.post(f"{api_endpoint}/{collection.get('endpoint')}", headers=headers, json=data)
76+
response = post(f"{api_endpoint}/{collection.get('endpoint')}", headers=headers, json=data)
7877
response.raise_for_status()
7978
logger.info(f"{collection.get('name')} inserted")
80-
except requests.exceptions.HTTPError as e:
79+
except exceptions.HTTPError as e:
8180
logger.warning(f"Failed to insert {collection.get('name')}: {e.response.json().get('message')}")
8281

8382

@@ -108,10 +107,10 @@ async def populate_collection_entities(collection, headers):
108107
}
109108
}
110109
try:
111-
response = requests.post(f"{api_endpoint}/{collection.get('endpoint')}", headers=headers, json=data)
110+
response = post(f"{api_endpoint}/{collection.get('endpoint')}", headers=headers, json=data)
112111
response.raise_for_status()
113112
logger.info(f"{collection.get('name')} inserted")
114-
except requests.exceptions.HTTPError as e:
113+
except exceptions.HTTPError as e:
115114
logger.warning(f"Failed to insert {collection.get('name')}: {e.response.json().get('message')}")
116115

117116

@@ -135,10 +134,10 @@ async def populate_collection_categories(collection, headers):
135134
}
136135
}
137136
try:
138-
response = requests.post(f"{api_endpoint}/{collection.get('endpoint')}", headers=headers, json=data)
137+
response = post(f"{api_endpoint}/{collection.get('endpoint')}", headers=headers, json=data)
139138
response.raise_for_status()
140139
logger.info(f"{collection.get('name')} inserted")
141-
except requests.exceptions.HTTPError as e:
140+
except exceptions.HTTPError as e:
142141
logger.warning(f"Failed to insert {collection.get('name')}: {e.response.json().get('message')}")
143142
for subcat in document.get('subCategory'):
144143
dataSubCat = {
@@ -157,10 +156,10 @@ async def populate_collection_categories(collection, headers):
157156
}
158157
}
159158
try:
160-
response = requests.post(f"{api_endpoint}/{collection.get('endpoint')}", headers=headers, json=dataSubCat)
159+
response = post(f"{api_endpoint}/{collection.get('endpoint')}", headers=headers, json=dataSubCat)
161160
response.raise_for_status()
162161
logger.info(f"{collection.get('name')} inserted")
163-
except requests.exceptions.HTTPError as e:
162+
except exceptions.HTTPError as e:
164163
logger.warning(f"Failed to insert {collection.get('name')}: {e.response.json()}")
165164

166165

@@ -183,10 +182,10 @@ async def populate_collection_source_requests(collection, headers):
183182
}
184183
}
185184
try:
186-
response = requests.post(f"{api_endpoint}/{collection.get('endpoint')}", headers=headers, json=data)
185+
response = post(f"{api_endpoint}/{collection.get('endpoint')}", headers=headers, json=data)
187186
response.raise_for_status()
188187
logger.info(f"{collection.get('name')} inserted")
189-
except requests.exceptions.HTTPError as e:
188+
except exceptions.HTTPError as e:
190189
logger.warning(f"Failed to insert {collection.get('name')}: {e.response.json().get('message')}")
191190

192191

@@ -279,10 +278,10 @@ async def populate_collection_source_ticket(collection, headers):
279278
}
280279
}
281280
try:
282-
response = requests.post(f"{api_endpoint}/{collection.get('endpoint')}", headers=headers, json=data)
281+
response = post(f"{api_endpoint}/{collection.get('endpoint')}", headers=headers, json=data)
283282
response.raise_for_status()
284283
logger.info(f"{collection.get('name')} inserted")
285-
except requests.exceptions.HTTPError as e:
284+
except exceptions.HTTPError as e:
286285
logger.warning(f"Failed to insert {collection.get('name')}: {e.response.json().get('message')}")
287286

288287
pattern = r'\((.*?)\) <(.*?)>'
@@ -326,7 +325,7 @@ async def populate_collection_source_ticket(collection, headers):
326325
if thread.get('from') is not None:
327326
if isinstance(thread['from'], list) and len(thread['from']) == 0:
328327
thread['from'] = ''
329-
matchFrom = re.match(pattern, thread['from'][0] if isinstance(thread['from'], list) and len(thread['from']) > 0 else thread['from'])
328+
matchFrom = match(pattern, thread['from'][0] if isinstance(thread['from'], list) and len(thread['from']) > 0 else thread['from'])
330329
if matchFrom is not None:
331330
fromMailInfo = {
332331
"name": matchFrom.group(1),
@@ -336,7 +335,7 @@ async def populate_collection_source_ticket(collection, headers):
336335
if thread.get('to') is not None:
337336
if isinstance(thread['to'], list) and len(thread['to']) == 0:
338337
thread['to'] = ''
339-
matchTo = re.match(pattern, thread['to'][0] if isinstance(thread['to'], list) and len(thread['to']) > 0 else thread['to'])
338+
matchTo = match(pattern, thread['to'][0] if isinstance(thread['to'], list) and len(thread['to']) > 0 else thread['to'])
340339
if matchTo is not None:
341340
toMailInfo = {
342341
"name": matchTo.group(1),
@@ -346,7 +345,7 @@ async def populate_collection_source_ticket(collection, headers):
346345
if thread.get('cc') is not None:
347346
if isinstance(thread['cc'], list) and len(thread['cc']) == 0:
348347
thread['cc'] = ''
349-
matchCc = re.match(pattern, thread['cc'][0] if isinstance(thread['cc'], list) and len(thread['cc']) > 0 else thread['cc'])
348+
matchCc = match(pattern, thread['cc'][0] if isinstance(thread['cc'], list) and len(thread['cc']) > 0 else thread['cc'])
350349
if matchCc is not None:
351350
ccMailInfo = {
352351
"name": matchCc.group(1),
@@ -372,10 +371,10 @@ async def populate_collection_source_ticket(collection, headers):
372371
}
373372
else:
374373
try:
375-
response = requests.get(f"{api_endpoint}/{collection.get('endpointThread')}?filters%5BticketId%5D={str(document['_id'])}&filters%5B%3DcustomFields.importData.date%5D={thread['date']}&filters%5B%3DcustomFields.importData.text%5D={thread['text']}", headers=headers)
374+
response = get(f"{api_endpoint}/{collection.get('endpointThread')}?filters%5BticketId%5D={str(document['_id'])}&filters%5B%3DcustomFields.importData.date%5D={thread['date']}&filters%5B%3DcustomFields.importData.text%5D={thread['text']}", headers=headers)
376375
response.raise_for_status()
377376
verifyThread = False if len(response.json()['data']) > 0 else True
378-
except requests.exceptions.HTTPError as e:
377+
except exceptions.HTTPError as e:
379378
logger.warning(f"Failed to search thread duplicate with text : <{thread['text']}>: {e.response.json()}")
380379
verifyThread = False
381380
if not verifyThread:
@@ -404,11 +403,11 @@ async def populate_collection_source_ticket(collection, headers):
404403
}
405404
}
406405
try:
407-
response = requests.post(f"{api_endpoint}/{collection.get('endpointThread')}",
406+
response = post(f"{api_endpoint}/{collection.get('endpointThread')}",
408407
headers=headers, json=dataThread)
409408
response.raise_for_status()
410409
logger.info(f"{collection.get('name')} inserted")
411-
except requests.exceptions.HTTPError as e:
410+
except exceptions.HTTPError as e:
412411
logger.warning(f"Failed to insert {collection.get('nameThread')}: {e.response.json()}")
413412

414413

@@ -439,7 +438,7 @@ async def populate_collection_filestorage(collection, headers):
439438
print(f"Ignore mail : {document['filename']}")
440439
continue
441440
remote_file = sftp.open(os.getenv("IMPORT_SSH_BASEPATH") + document['path'])
442-
decoded_file = base64.b64decode(remote_file.read())
441+
decoded_file = b64decode(remote_file.read())
443442
files = {'file': decoded_file}
444443
# noinspection PyUnresolvedReferences
445444
data = {
@@ -457,52 +456,52 @@ async def populate_collection_filestorage(collection, headers):
457456
# }
458457
}
459458
try:
460-
response = requests.post(f"{api_endpoint}/{collection.get('endpoint')}", data=data, files=files)
459+
response = post(f"{api_endpoint}/{collection.get('endpoint')}", data=data, files=files)
461460
response.raise_for_status()
462461
logger.info(f"{collection.get('name')} inserted")
463-
except requests.exceptions.HTTPError as e:
462+
except exceptions.HTTPError as e:
464463
logger.warning(f"Failed to insert {collection.get('name')}: {e} \n {e.response.text}")
465464

466465
collections = [
467-
# {
468-
# "name": "filestorage",
469-
# "import_db": "libertech-sys",
470-
# "import_db_ticket": "libertech-data",
471-
# "import_db_mail": "mails",
472-
# "import_collection": "userstorage",
473-
# "import_collection_ticket": "tickets",
474-
# "import_collection_mail": "mailsStore",
475-
# "endpoint": "core/filestorage",
476-
# "method": populate_collection_filestorage,
477-
# },
478-
# {
479-
# "name": "sla",
480-
# "import_db": "libertech-data",
481-
# "import_collection": "tickets_sla",
482-
# "endpoint": "tickets/sla",
483-
# "method": populate_collection_sla
484-
# },
485-
# {
486-
# "name": "entities",
487-
# "import_db": "libertech-data",
488-
# "import_collection": "peoples",
489-
# "endpoint": "core/entities",
490-
# "method": populate_collection_entities
491-
# },
492-
# {
493-
# "name": "categories",
494-
# "import_db": "libertech-data",
495-
# "import_collection": "tickets_category",
496-
# "endpoint": "core/categories",
497-
# "method": populate_collection_categories
498-
# },
499-
# {
500-
# "name": "source-requests",
501-
# "import_db": "libertech-data",
502-
# "import_collection": "tickets_sourcetype",
503-
# "endpoint": "tickets/source-request",
504-
# "method": populate_collection_source_requests
505-
# },
466+
{
467+
"name": "filestorage",
468+
"import_db": "libertech-sys",
469+
"import_db_ticket": "libertech-data",
470+
"import_db_mail": "mails",
471+
"import_collection": "userstorage",
472+
"import_collection_ticket": "tickets",
473+
"import_collection_mail": "mailsStore",
474+
"endpoint": "core/filestorage",
475+
"method": populate_collection_filestorage,
476+
},
477+
{
478+
"name": "sla",
479+
"import_db": "libertech-data",
480+
"import_collection": "tickets_sla",
481+
"endpoint": "tickets/sla",
482+
"method": populate_collection_sla
483+
},
484+
{
485+
"name": "entities",
486+
"import_db": "libertech-data",
487+
"import_collection": "peoples",
488+
"endpoint": "core/entities",
489+
"method": populate_collection_entities
490+
},
491+
{
492+
"name": "categories",
493+
"import_db": "libertech-data",
494+
"import_collection": "tickets_category",
495+
"endpoint": "core/categories",
496+
"method": populate_collection_categories
497+
},
498+
{
499+
"name": "source-requests",
500+
"import_db": "libertech-data",
501+
"import_collection": "tickets_sourcetype",
502+
"endpoint": "tickets/source-request",
503+
"method": populate_collection_source_requests
504+
},
506505
{
507506
"name": "ticket",
508507
"nameThread": "thread",

‎populate/populate_from_gaiasys/Dockerfile‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Use an official Python runtime as a parent image
2+
FROM python:3.8
3+
4+
# Set the working directory to /app
5+
WORKDIR /app
6+
7+
# Copy the current directory contents into the container at /app
8+
COPY ../. /app
9+
10+
# Install any needed packages specified in requirements.txt
11+
RUN pip install --no-cache-dir -r requirements.txt
12+
13+
CMD [ "python", "populate_from_gaiasys.py" ]

‎populate/requirements.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
motor==3.3.1
2+
paramiko==2.11.0
3+
pymongo==4.5.0
4+
python-dotenv==1.0.0
5+
requests==2.26.0

0 commit comments

Comments
 (0)