Avoid collecting 80k lines of logs in checkbox collector (bugfix) (#2417)

* Move code to utility function

* Test new function

lowmem is less than 2Gb

* Also collect less logs for lowmem devices

Dropping journal lines collected helps first avoiding getting killed by
the OOM killer on some platforms with REALLY low ram. Secondly, given
that the submission is less compressed, lowers a lot the egress cost of
the upload, mitigating the 5xx errors we are getting

* Python3.5 is fun

* Better docstring and type hint

Author: Hook25

checkbox-ng/plainbox/impl/__init__.py

@@ -210,3 +210,17 @@ def get_plainbox_dir():
     Return the root directory of the plainbox package.
     """
     return os.path.dirname(getabsfile(plainbox))
+
+
+def low_memory_device() -> bool:
+    """
+    Returns `True` if the system we are running on has very low RAM installed
+
+    This is used to disable or tune down some subsystems to avoid filling
+    the RAM with Checkbox itself and crashing the system.
+    """
+    mem_bytes = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
+    mem_mib = mem_bytes / (1024.0**2)
+    if mem_mib < 1024 * 2:
+        return True
+    return False

checkbox-ng/plainbox/impl/exporter/tar.py

@@ -30,6 +30,7 @@
 import time
 from tempfile import SpooledTemporaryFile
 
+from plainbox.impl import low_memory_device
 from plainbox.impl.exporter import SessionStateExporterBase
 from plainbox.impl.exporter.jinja2 import Jinja2SessionStateExporter
 from plainbox.impl.providers import get_providers
@@ -53,14 +54,12 @@ def dump_from_session_manager(self, manager, stream):
 
         """
         preset = None
-        mem_bytes = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
-        mem_mib = mem_bytes / (1024.0**2)
         # On systems with less than 1GiB of RAM, create the submission tarball
         # without any compression level (i.e preset=0).
         # See https://docs.python.org/3/library/lzma.html
         # With preset 9 for example, the overhead for an LZMACompressor object
         # can be as high as 800 MiB.
-        if mem_mib < 1200:
+        if low_memory_device():
             preset = 0
 
         job_state_map = manager.default_device_context.state.job_state_map

checkbox-ng/plainbox/impl/session/system_information.py

@@ -2,8 +2,10 @@
 
 import abc
 import json
+
 from subprocess import run, PIPE, check_output, STDOUT, CalledProcessError
 
+from plainbox.impl import low_memory_device
 from plainbox import vendor
 from plainbox.impl.session.storage import WellKnownDirsHelper
 from checkbox_ng import __version__ as checkbox_version
@@ -296,6 +298,11 @@ def collector_parser(self, collector_output):
         return list(map(json.loads, collector_output.splitlines()))
 
     def __init__(self):
+        to_collect = 80000  # limit the lines to 80k, ~80Mb of memory
+        if low_memory_device():
+            # when memory is low, we can't afford to load all that json into
+            # memory, try to collect a few logs either way
+            to_collect = 8000
         super().__init__(
             collection_cmd=[
                 "journalctl",
@@ -304,7 +311,7 @@ def __init__(self):
                 "--since",
                 "-3 days",
                 "-n",
-                "80000",  # limit the lines to 80k, ~80Mb of memory
+                str(to_collect),
             ],
             version_cmd=["journalctl", "--version"],
         )

checkbox-ng/plainbox/impl/test_init.py

@@ -23,11 +23,11 @@
 Test definitions for plainbox.impl module
 """
 
-from unittest import TestCase
 import warnings
+from unittest import TestCase
+from unittest.mock import patch
 
-from plainbox.impl import _get_doc_margin
-from plainbox.impl import deprecated
+from plainbox.impl import _get_doc_margin, deprecated, low_memory_device
 
 
 class MiscTests(TestCase):
@@ -113,3 +113,16 @@ def func():
             str(boom.exception),
             "@deprecated() must be called with a parameter",
         )
+
+
+class LowMemoryDeviceTests(TestCase):
+
+    @patch("os.sysconf")
+    def test_not_lowmem(self, sysconf_mock):
+        sysconf_mock.side_effect = [4096, 7627054]
+        self.assertFalse(low_memory_device())
+
+    @patch("os.sysconf")
+    def test_lowmem(self, sysconf_mock):
+        sysconf_mock.side_effect = [4096, 4]
+        self.assertTrue(low_memory_device())