Merge pull request #71 from kartoza/fixes

[skip-release] improve bucket creation

Author: NyakudyaA

.github/dependabot.yml

@@ -3,4 +3,4 @@ updates:
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:
-      interval: "monthly"
+      interval: "weekly"

docker-compose-s3.yml

@@ -21,7 +21,7 @@ services:
     entrypoint: /bin/bash
     command: -c 'minio server /data --console-address ":9001"'
     volumes:
-      - minio_data:/mapproxy
+      - minio_data:/data
     ports:
       - "9000:9000"
       - "9001:9001"

mapproxy_configuration/mapproxy-s3.yaml

@@ -39,13 +39,13 @@ services:
       contact:
         person: Kartoza
         position: GIS Manager
-        organization:
-        address:
-        city:
-        postcode:
+        organization: Kartoza
+        address: Kartoza
+        city: Cape Town
+        postcode: 7700
         country: south Africa
-        phone:
-        email:
+        phone: '276426345'
+        email: info@kartoza.com
       access_constraints: Insert license and copyright information for this service.
       fees: "None"
   tms:

mapproxy_configuration/mapproxy.yaml

@@ -39,13 +39,13 @@ services:
       contact:
         person: Kartoza
         position: GIS Manager
-        organization:
-        address:
-        city:
-        postcode:
+        organization: Kartoza
+        address: Kartoza
+        city: Cape Town
+        postcode: 7700
         country: south Africa
-        phone:
-        email:
+        phone: '276426345'
+        email: info@kartoza.com
       access_constraints:
         Insert license and copyright information for this service.
       fees: 'None'

scenario_tests/s3/docker-compose.yml

@@ -19,7 +19,7 @@ services:
     entrypoint: /bin/bash
     command: -c 'minio server /data --console-address ":9001"'
     volumes:
-      - minio_data:/mapproxy
+      - minio_data:/data
     ports:
       - "9000:9000"
       - "9001:9001"

scenario_tests/s3/mapproxy_configuration/mapproxy.yaml

@@ -39,13 +39,13 @@ services:
       contact:
         person: Kartoza
         position: GIS Manager
-        organization:
-        address:
-        city:
-        postcode:
+        organization: Kartoza
+        address: Kartoza
+        city: Cape Town
+        postcode: 7700
         country: south Africa
-        phone:
-        email:
+        phone: '276426345'
+        email: info@kartoza.com
       access_constraints: Insert license and copyright information for this service.
       fees: "None"
   tms:
@@ -70,7 +70,6 @@ caches:
     grids: [osm_grid]
     meta_size: [5, 5]
     meta_buffer: 20
-    concurrent_tile_creators: 2
     cache:
       type: s3
       directory: /hillshade/

scripts/create_default_buckets.py

@@ -1,42 +1,100 @@
-from os import environ as env
+import os
 import re
-import boto3, botocore
+import logging
+from typing import List
+import boto3
+from botocore.exceptions import ClientError
 
-def check_bucket(s3, bucket_name):
+# Configure logging
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def get_environment_variable(name: str, required: bool = True) -> str:
+    """
+    Retrieves an environment variable or raises an error if it's missing.
+    """
+    value = os.getenv(name)
+    if required and not value:
+        raise ValueError(f"Missing required environment variable: {name}")
+    return value
+
+
+def parse_bucket_list(bucket_list: str) -> List[str]:
+    """
+    Parses a string of bucket names separated by commas, spaces, or semicolons into a list.
+    """
+    return re.split(r'[ ,;]+', bucket_list.strip())
+
+
+def check_bucket(s3_client, bucket_name: str) -> bool:
+    """
+    Checks if a bucket exists and is accessible.
+
+    Returns True if the bucket exists or is private, False otherwise.
+    """
     try:
-        s3.head_bucket(Bucket=bucket_name)
-        print(f"{bucket_name} available")
+        s3_client.head_bucket(Bucket=bucket_name)
+        logger.info(f"Bucket '{bucket_name}' is available.")
         return True
-    except botocore.exceptions.ClientError as e:
+    except ClientError as e:
         error_code = int(e.response['Error']['Code'])
         if error_code == 403:
-            print(f"{bucket_name} is Private. Access denied.")
+            logger.warning(f"Bucket '{bucket_name}' is private. Access denied.")
             return True
         elif error_code == 404:
-            print(f"{bucket_name} does not exist")
+            logger.info(f"Bucket '{bucket_name}' does not exist.")
             return False
+        else:
+            logger.error(f"Error checking bucket '{bucket_name}': {e}")
+            raise
+
+
+def create_bucket(s3_client, bucket_name: str):
+    """
+    Creates a bucket if it does not exist.
+    """
+    try:
+        logger.info(f"Creating bucket '{bucket_name}'.")
+        s3_client.create_bucket(Bucket=bucket_name)
+        logger.info(f"Bucket '{bucket_name}' created successfully.")
+    except ClientError as e:
+        logger.error(f"Failed to create bucket '{bucket_name}': {e}")
+        raise
+
 
 def main():
-    buckets = env['S3_BUCKET_LIST']
-    buckets = re.split(r',| |;', buckets)
-    end_point = env['S3_BUCKET_ENDPOINT']
+    """
+    Main function to check and create S3 buckets as needed.
+    """
+    bucket_list_str = get_environment_variable('S3_BUCKET_LIST')
+    buckets = parse_bucket_list(bucket_list_str)
+    endpoint = get_environment_variable('S3_BUCKET_ENDPOINT')
 
-    session = boto3.session.Session()
+    aws_access_key_id = get_environment_variable('AWS_ACCESS_KEY_ID')
+    aws_secret_access_key = get_environment_variable('AWS_SECRET_ACCESS_KEY')
 
+    session = boto3.session.Session()
     s3 = session.client(
         service_name='s3',
-        aws_access_key_id=env['AWS_ACCESS_KEY_ID'],
-        aws_secret_access_key=env['AWS_SECRET_ACCESS_KEY'],
-        endpoint_url=end_point,
+        aws_access_key_id=aws_access_key_id,
+        aws_secret_access_key=aws_secret_access_key,
+        endpoint_url=endpoint,
     )
 
-    for i in buckets:
-        if not check_bucket(s3, i):
-            print(f"Creating {i}")
-            s3.create_bucket(Bucket=i)
+    for bucket_name in buckets:
+        if not check_bucket(s3, bucket_name):
+            create_bucket(s3, bucket_name)
+
 
-if __name__=="__main__":
-    create_buckets = env['CREATE_DEFAULT_S3_BUCKETS']
-    if create_buckets.lower() == 'true':
-        print("Creating default buckets")
-        main()
+if __name__ == "__main__":
+    try:
+        create_buckets = get_environment_variable('CREATE_DEFAULT_S3_BUCKETS', required=False)
+        if create_buckets and create_buckets.lower() == 'true':
+            logger.info("Starting bucket creation process.")
+            main()
+        else:
+            logger.info("CREATE_DEFAULT_S3_BUCKETS is not set to 'true'. Skipping bucket creation.")
+    except Exception as e:
+        logger.error(f"An error occurred: {e}")
+        exit(1)

scripts/start.sh

@@ -33,6 +33,7 @@ function entry_point_script {
     for f in /docker-entrypoint-mapproxy.d/*; do
       case "$f" in
             *.sh)     echo "$0: running $f"; . "$f" || true;;
+            *.py)     echo "$0: running Python script $f"; python3 "$f" || true;;
             *)        echo "$0: ignoring $f" ;;
         esac
         echo
@@ -50,6 +51,7 @@ function cleanup_files(){
     if [[ $proxy_count != 0 ]];then
       for X in ${PARAM}_*.${EXT}; do
         if [ "$X" != "${PARAM}_${HOSTNAME}.${EXT}" ]; then
+            echo "Removing $X"
             rm -rf "$X"
         fi
       done