Merge pull request syslog-ng#5149 from HofiOne/s3-memleak-fix

s3: workaround on-reload memleak when no role is set

Author: HofiOne

modules/python-modules/syslogng/modules/s3/s3_destination.py

@@ -249,13 +249,16 @@ def open(self) -> bool:
         if self.is_opened():
             return True
 
-        self.session = Session(
-            aws_access_key_id=self.access_key if self.access_key != "" else None,
-            aws_secret_access_key=self.secret_key if self.secret_key != "" else None,
-            region_name=self.region,
-        )
-
+        # NOTE: Creating a client via a Session object does some unusual caching, which increases memory usage
+        # NOTE: each reload.  Because of this, we only create Session object if the role is set, and in that case
+        # NOTE: the memory usage is expected to behave unusually.  Sometime we should investigate this further.
         if self.role != "":
+            self.session = Session(
+                aws_access_key_id=self.access_key if self.access_key != "" else None,
+                aws_secret_access_key=self.secret_key if self.secret_key != "" else None,
+                region_name=self.region,
+            )
+
             # NOTE: The Session.set_credentials always creates a new Credentials object from the given keys.
             # NOTE: The DeferredRefreshableCredentials class is a child of RefreshableCredentials which is a
             # NOTE: child of the Credentials class.
@@ -271,10 +274,18 @@ def open(self) -> bool:
             whoami = sts.get_caller_identity().get("Arn")
             self.logger.info(f"Using {whoami} to access the bucket")
 
-        self.client = self.session.client(
-            service_name="s3",
-            endpoint_url=self.url if self.url != "" else None,
-        )
+            self.client = self.session.client(
+                service_name="s3",
+                endpoint_url=self.url if self.url != "" else None,
+            )
+        else:
+            self.client = client(
+                service_name="s3",
+                endpoint_url=self.url if self.url != "" else None,
+                aws_access_key_id=self.access_key if self.access_key != "" else None,
+                aws_secret_access_key=self.secret_key if self.secret_key != "" else None,
+                region_name=self.region,
+            )
 
         is_opened = False
         try:

news/bugfix-5149.md

@@ -0,0 +1,6 @@
+`s3()`: Eliminated indefinite memory usage increase for each reload.
+
+The increased memory usage is caused by the `botocore` library, which
+caches the session information. We only need the Session object, if
+`role()` is set. The increased memory usage still happens with that set,
+currently we only fixed the unset case.