Skip to content

Commit 664dafc

Browse files
sabinemsabinem
authored
Merge pull request #7 from odtp-org/optional-logs-to-mongodb
Make it optional to add logs to the mongo db by adding parameters for both logs as files and logs in mongo db: - When `ODTP_LOGS_AS_FILE=TRUE` the log files will be copied to the output folder - When `ODTP_LOGS_IN_DB=TRUE` the logs will be stored in the mongodb Also improve shell scripts with shellcheck and use `[[..]]` rather than `[..]` in conditionals.
2 parents ff0d9c3 + 8484c34 commit 664dafc

5 files changed

Lines changed: 23 additions & 176 deletions

File tree

‎.env.dist‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,5 @@ ODTP_RESULT=
1818
TRANSFER_INPUT_TO_OUTPUT=FALSE
1919
ODTP_SAVE_SNAPSHOT=FALSE
2020
ODTP_SAVE_IN_RESULT=TRUE
21+
ODTP_LOGS_AS_FILE=TRUE
22+
ODTP_LOGS_IN_DB=FALSE

‎logger.py‎

Lines changed: 0 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -8,81 +8,17 @@
88
import json
99

1010

11-
#################################################################################
12-
# Testing Class MongoManager with v.0.2.0 Schema version of ID.
13-
# Copied from odtp. Once the repo is public we can hide it
14-
1511
class MongoManager:
1612
def __init__(self, mongodbUrl, db_name):
1713
self.client = MongoClient(mongodbUrl)
1814
self.db = self.client[db_name]
19-
20-
def add_user(self, user_data):
21-
users_collection = self.db["users"]
22-
23-
return users_collection.insert_one(user_data).inserted_id
24-
25-
def add_component(self, component_data):
26-
components_collection = self.db["components"]
27-
28-
return components_collection.insert_one(component_data).inserted_id
29-
30-
def add_version(self, componentId, version_data):
31-
versions_collection = self.db["versions"]
32-
version_data["componentId"] = componentId
33-
34-
return versions_collection.insert_one(version_data).inserted_id
35-
36-
def add_digital_twin(self, userRef, digital_twin_data):
37-
digital_twins_collection = self.db["digitalTwins"]
38-
digital_twin_data["userRef"] = userRef
39-
40-
digital_twin_id = digital_twins_collection.insert_one(digital_twin_data).inserted_id
41-
42-
# Add digital twin reference to user
43-
self.db.users.update_one(
44-
{"_id": userRef},
45-
{"$push": {"digitalTwins": digital_twin_id}}
46-
)
47-
48-
return digital_twin_id
4915

5016
def add_logs(self, log_data_list):
5117
logs_collection = self.db["logs"]
5218

5319
log_ids = logs_collection.insert_many(log_data_list).inserted_ids
5420

5521
return log_ids
56-
57-
58-
def append_execution(self, digital_twin_id, execution_data):
59-
executions_collection = self.db["executions"]
60-
execution_data["digitalTwinRef"] = digital_twin_id
61-
62-
execution_id = executions_collection.insert_one(execution_data).inserted_id
63-
64-
# Update digital twin with execution reference
65-
self.db.digitalTwins.update_one(
66-
{"_id": digital_twin_id},
67-
{"$push": {"executions": execution_id}}
68-
)
69-
70-
return execution_id
71-
72-
73-
def append_step(self, execution_id, step_data):
74-
steps_collection = self.db["steps"]
75-
step_data["executionRef"] = execution_id
76-
77-
step_id = steps_collection.insert_one(step_data).inserted_id
78-
79-
# Update execution with step reference
80-
self.db.executions.update_one(
81-
{"_id": execution_id},
82-
{"$push": {"steps": step_id}}
83-
)
84-
85-
return step_id
8622

8723
def add_output(self, step_id, output_data):
8824
output_collection = self.db["outputs"]

‎odtp-app.sh‎

Lines changed: 15 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,11 @@ echo "STARTING ODTP COMPONENT"
88
sleep 2
99

1010
## ODTP LOGGER in the background
11-
if [ -v ODTP_MONGO_SERVER ]; then
12-
echo "STARTING LOGGING IN MONGO SERVER"
11+
if [[ -v ODTP_MONGO_SERVER && "${ODTP_LOGS_IN_DB:-}" == "TRUE" ]]; then
12+
echo "STARTING LOGGING IN 'ODTP_MONGO_SERVER'"
1313
python3 /odtp/odtp-component-client/logger.py >> /odtp/odtp-logs/odtpLoggerDebugging.txt 2>&1 &
1414
else
15-
echo "ODTP_MONGO_SERVER does not exist"
15+
echo "'ODTP_MONGO_SERVER' does not exist or 'ODTP_LOGS_IN_DB' not set to 'TRUE'"
1616
fi
1717

1818

@@ -30,8 +30,8 @@ bash /odtp/odtp-app/app.sh
3030
# TRANSFERRING INPUT TO OUTPUT
3131
#########################################################
3232

33-
if [ "$TRANSFER_INPUT_TO_OUTPUT" == "TRUE" ]; then
34-
echo "COPYING INPUT FILES TO OUTPUT"
33+
if [[ "${TRANSFER_INPUT_TO_OUTPUT:-}" == "TRUE" ]]; then
34+
echo "COPYING INPUT FILES TO OUTPUT because 'TRANSFER_INPUT_TO_OUTPUT' is 'TRUE'"
3535
cp -r /odtp/odtp-input/* /odtp/odtp-output
3636
fi
3737

@@ -41,41 +41,32 @@ fi
4141

4242
# Take output and export it
4343
cd /odtp/odtp-output
44-
zip -rq ../odtp-output.zip *
44+
zip -rq ../odtp-output.zip ./*
4545
mv ../odtp-output.zip odtp-output.zip
4646

4747
#########################################################
4848
# ODTP SNAPSHOT CREATION
4949
#########################################################
5050

5151
# Take snapshot of workdir
52-
if [ "ODTP_SAVE_SNAPSHOT" == "TRUE" ]; then
52+
if [[ "${ODTP_SAVE_SNAPSHOT:-}" = "TRUE" ]]; then
5353
cd /odtp/odtp-workdir
54-
zip -rq ../odtp-snapshot.zip *
54+
zip -rq ../odtp-snapshot.zip ./*
5555
mv ../odtp-snapshot.zip odtp-snapshot.zip
5656
fi
5757

58-
5958
## Saving Snapshot in s3
6059

6160
if [[ -v ODTP_S3_SERVER && -v ODTP_MONGO_SERVER ]]; then
62-
echo "Uploading to ODTP_S3_SERVER"
61+
echo "Uploading to ODTP_S3_SERVER because 'ODTP_S3_SERVER' and 'ODTP_MONGO_SERVER' are both set"
6362
python3 /odtp/odtp-component-client/s3uploader.py 2>&1 | tee /odtp/odtp-logs/odtpS3UploadedDebugging.txt
6463
else
65-
echo "ODTP_S3_SERVER does not exist"
64+
echo "Not uploading to S3 since either 'ODTP_S3_SERVER' or 'ODTP_MONGO_SERVER' is not specified"
6665
fi
6766

68-
# ## Copying logs into output
69-
# cp /odtp/odtp-logs/log.txt /odtp/odtp-output/log.txt
70-
71-
# if [ -v ODTP_S3_SERVER ]; then
72-
# cp /odtp/odtp-logs/odtpLoggerDebugging.txt /odtp/odtp-output/odtpLoggerDebugging.txt
73-
# else
74-
# echo "ODTP_S3_SERVER doesn't exist. Not copying log files."
75-
# fi
67+
## Copying logs into output
7668

77-
# if [[ -v ODTP_S3_SERVER && -v ODTP_MONGO_SERVER ]]; then
78-
# cp /odtp/odtp-logs/odtpS3UploadedDebugging.txt /odtp/odtp-output/odtpS3UploadedDebugging.txt
79-
# else
80-
# echo "ODTP_S3_SERVER doesn't exist. Not copying log files."
81-
# fi
69+
if [[ "${ODTP_LOGS_AS_FILE:-}" == "TRUE" ]]; then
70+
echo "Writing log files to output since 'ODTP_LOGS_AS_FILE' is set to 'TRUE'"
71+
cp /odtp/odtp-logs/log.txt /odtp/odtp-output/log.txt
72+
fi

‎s3uploader.py‎

Lines changed: 5 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -18,39 +18,11 @@ def __init__(self, s3Server, bucketName, accessKey, secretKey):
1818
self.accessKey = accessKey
1919
self.secretKey = secretKey
2020

21-
self.connect()
22-
23-
# Add logging Info
24-
25-
# Method to connect to s3 server
26-
def connect(self):
27-
s3 = boto3.client('s3', endpoint_url=self.s3Server,
28-
aws_access_key_id=self.accessKey,
29-
aws_secret_access_key=self.secretKey)
30-
31-
self.s3 = s3
32-
33-
# Add logging info
34-
35-
# Method to close the client connection
36-
def closeConnection(self):
37-
"""
38-
Closes the connection to the S3 client.
39-
"""
40-
self.s3.meta.client.close()
41-
logging.info("Connection closed")
42-
43-
# Create folder structure
44-
def createFolderStructure(self, structure=["odtp"]):
45-
"""
46-
structure: is a list of paths to create. By default a single "odtp" folder
47-
"""
48-
49-
for path in structure:
50-
# Add a trailing slash to make S3 recognize it as a folder
51-
self.s3.put_object(Bucket=self.bucketName, Key=path + '/')
52-
53-
print("Folder Structure Created")
21+
self.s3 = boto3.client(
22+
's3', endpoint_url=self.s3Server,
23+
aws_access_key_id=self.accessKey,
24+
aws_secret_access_key=self.secretKey
25+
)
5426

5527
# Method to create a specific folder
5628
# The idea is to create paths such as Digital Twin > Execution > Step > Output
@@ -82,60 +54,6 @@ def uploadFile(self, local_path, s3_path):
8254
self.s3.upload_file(local_path, self.bucketName, s3_path)
8355
logging.info(f"File '{local_path}' uploaded to '{s3_path}'")
8456

85-
# Method to download one file from s3 and place it in a folder
86-
# This is needed to make the input/output logic out of barfi
87-
def downloadFile(self, s3_path, local_path):
88-
"""
89-
Downloads a file from a specific path in the S3 bucket and saves it locally.
90-
91-
Args:
92-
s3_path (str): The S3 path of the file to download.
93-
local_path (str): The local path where the file should be saved.
94-
95-
Returns:
96-
None
97-
"""
98-
self.s3.download_file(self.bucketName, s3_path, local_path)
99-
logging.info(f"File '{s3_path}' downloaded to '{local_path}'")
100-
101-
# Method to list folders in s3
102-
def checkObjects(self):
103-
104-
response = self.s3.list_objects_v2(Bucket=self.bucketName,
105-
Delimiter='/')
106-
107-
folders = []
108-
if 'CommonPrefixes' in response:
109-
for prefix in response['CommonPrefixes']:
110-
folders.append(prefix['Prefix'])
111-
112-
return folders
113-
114-
# Method to delete all objects in s3
115-
def deleteAll(self):
116-
117-
bucket = self.s3.Bucket(self.bucketName)
118-
119-
# This will delete all objects in the bucket.
120-
bucket.objects.all().delete()
121-
122-
print("Folder Structure Deleted")
123-
124-
# Method to delete one file in s3
125-
def deleteFile(self, s3_path):
126-
127-
"""
128-
Deletes a file from a specific path in the S3 bucket.
129-
130-
Args:
131-
s3_path (str): The S3 path of the file to delete.
132-
133-
Returns:
134-
None
135-
"""
136-
self.s3.delete_object(Bucket=self.bucketName, Key=s3_path)
137-
logging.info(f"File '{s3_path}' deleted from S3 bucket")
138-
13957
def close(self):
14058
del self.s3
14159
logging.info("S3 Session deleted")

‎startup.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,4 @@ echo "--- ODTP COMPONENT ENDING ---" >> /odtp/odtp-logs/log.txt
1212
sleep 10
1313

1414
# Zip logs
15-
zip -r /odtp/odtp-output/odtp-logs.zip odtp-logs
15+
zip -r /odtp/odtp-output/odtp-logs.zip odtp-logs

0 commit comments

Comments
 (0)