Add large file logging support (#116)

Author: alfinkel

CHANGES.md

@@ -1,3 +1,8 @@
+# [1.17.0](https://github.com/ComplianceAsCode/auditree-framework/releases/tag/v1.17.0)
+
+- [ADDED] Locker get_large_files method added to return large files in the locker.
+- [ADDED] Logging of large files added to remote push operation.
+
 # [1.16.0](https://github.com/ComplianceAsCode/auditree-framework/releases/tag/v1.16.0)
 
 - [ADDED] Locker get_empty_evidences method added to return all empty evidence paths.

compliance/__init__.py

@@ -14,4 +14,4 @@
 # limitations under the License.
 """Compliance automation package."""
 
-__version__ = '1.16.0'
+__version__ = '1.17.0'

compliance/locker.py

@@ -45,6 +45,10 @@
 AE_DEFAULT = 30 * DAY
 READMES = ['README.md', 'readme.md', 'Readme.md']
 AE_EXEMPT = [INDEX_FILE] + READMES
+NOT_EVIDENCE = AE_EXEMPT
+KB = 1000
+MB = KB * 1000
+LF_DEFAULT = 50 * MB
 
 
 class Locker(object):
@@ -635,6 +639,7 @@ def push(self):
             )
             remote.fetch()
             remote.pull(rebase=True)
+            self._log_large_files()
             self.logger.info(
                 f'Pushing local locker to remote repo {self.repo_url}...'
             )
@@ -704,38 +709,43 @@ def get_abandoned_evidences(self, threshold=None):
 
         :returns: a set of abandoned evidence file relative paths.
         """
-        abandoned_evidence = []
-        tree = self.repo.head.commit.tree
-        for f in tree.traverse():
-            if (f.type != 'blob' or f.path.startswith('notifications/')
-                    or f.path == 'check_results.json'
-                    or f.path.split('/').pop() in AE_EXEMPT):
-                continue
+        abandoned_evidence = set()
+        for f in self._get_git_files('evidence'):
             metadata = self.get_evidence_metadata(f.path, dt.utcnow())
             if self._evidence_abandoned(metadata, threshold):
-                abandoned_evidence.append(f.path)
-        return set(abandoned_evidence)
+                abandoned_evidence.add(f.path)
+        return abandoned_evidence
 
     def get_empty_evidences(self):
         """
         Provide a list of evidence paths to empty evidence files.
 
-        Evidence content is deemed empty based on an evidence object's
+        Evidence content is considered empty based on an evidence object's
         is_empty property.  This information is stored in evidence metadata.
 
         :returns: a list of empty evidence file relative paths.
         """
         empty_evidence = []
-        tree = self.repo.head.commit.tree
-        for idx_file in [f for f in tree.traverse() if is_index_file(f.path)]:
-            metadata = json.loads(idx_file.data_stream.read())
-            for ev_name, ev_meta in metadata.items():
+        for f in self._get_git_files('index'):
+            for ev_name, ev_meta in json.loads(f.data_stream.read()).items():
                 if ev_meta.get('empty', False):
                     empty_evidence.append(
-                        str(PurePath(PurePath(idx_file.path).parent, ev_name))
+                        str(PurePath(f.path).with_name(ev_name))
                     )
         return empty_evidence
 
+    def get_large_files(self, size=LF_DEFAULT):
+        """
+        Provide a dictionary of "large" evidence locker files.
+
+        A "large" file is one whose size is > the ``size`` argument provided.
+
+        :param int size: file size threshold.
+
+        :returns: a dictionary of file paths and sizes of "large" files.
+        """
+        return {f.path: f.size for f in self._get_git_files() if f.size > size}
+
     def delete_repo_locally(self):
         """Remove the local git repository."""
         try:
@@ -935,7 +945,52 @@ def _validate_evidence(self, evidence, ignore_ttl):
         if not ignore_ttl and ttl_expired:
             raise StaleEvidenceError(f'Evidence {evidence.path} is stale')
 
+    def _get_git_files(self, file_type='all'):
+        iz = {
+            'all': lambda g: g.type == 'blob',
+            'evidence': is_evidence_file,
+            'index': is_index_file
+        }
+        return filter(iz[file_type], self.repo.head.commit.tree.traverse())
+
+    def _log_large_files(self):
+        large_files = self.get_large_files(
+            get_config().get('locker.large_file_threshold', LF_DEFAULT)
+        )
+        if large_files:
+            msg = ['LARGE FILES (Hosting service may reject due to size):\n']
+            for fpath, size in large_files.items():
+                formatted_size = f'{size/MB:.1f} MB'
+                if formatted_size == '0.0 MB':
+                    formatted_size = f'{str(size)} Bytes'
+                msg.append(f'      {fpath} is {formatted_size}')
+            self.logger.info('\n'.join(msg) + '\n')
+
 
-def is_index_file(path):
-    """Confirm whether the supplied path is to an index file."""
-    return os.path.basename(path) == INDEX_FILE
+def is_evidence_file(git_obj):
+    """
+    Confirm whether the supplied git object is an evidence file.
+
+    :param git_obj: A GitPython object
+
+    :returns: True or False (Object is or isn't an evidence file)
+    """
+    return (
+        git_obj.type == 'blob'
+        and not git_obj.path.startswith('notifications/')
+        and git_obj.path != 'check_results.json'
+        and PurePath(git_obj.path).name not in NOT_EVIDENCE
+    )
+
+
+def is_index_file(obj):
+    """
+    Confirm whether the supplied object is, or points to, an index file.
+
+    :param obj: Either a GitPython object or a relative file path as a string
+
+    :returns: True or False (Object is or isn't an index file)
+    """
+    if isinstance(obj, str):
+        return PurePath(obj).name == INDEX_FILE
+    return obj.type == 'blob' and PurePath(obj.path).name == INDEX_FILE

doc-source/design-principles.rst

@@ -47,11 +47,16 @@ evidence (see :py:mod:`compliance.evidence`):
 See :ref:`fetchers` section for conventions and expectations with
 respect to modifying RawEvidence.
 
-All evidence has a ``ttl`` field (Time To Live) which defines for how
-long an evidence should be considered valid. For instance, healthcheck
-data is only valid during 1 day since new input is generated everyday. For
-this reason, any check trying to use an evidence with an expired ``ttl``
-must error.
+All evidence has a settable ``ttl`` (Time To Live) property that defines how
+long an evidence should be considered valid. For instance, if new data is
+generated on a daily basis then evidence gathered for that data should only be
+valid for 1 day.  For this reason, any check trying to use evidence with an
+expired ``ttl`` will error.
+
+All evidence has an ``is_empty`` property that defines an evidence's empty
+state.  This provides value when monitoring evidence content for completness.
+The property can be overridden to define "empty" for any given evidence.  By default evidence is considered empty if it has no content, is all whitespace,
+or if it is JSON and is an empty dictionary or list (``{}``, ``[]``).
 
 
 Evidence Locker
@@ -89,22 +94,22 @@ for:
 
 * Validating the ``ttl`` for a given evidence.  An optional evidence
   ``ttl`` tolerance value can be configured to be applied during
-  fetcher execution.  Check execution is not affected by this optional
-  tolerance value because checks should only interact with evidence that
-  is fresh (not stale).  This value (in seconds) tells fetchers to
+  fetcher execution.  This value (in seconds) tells fetchers to
   retrieve evidence that is nearly but not yet stale.  If no value is
   supplied then fetchers will only retrieve new evidence after ``ttl``
   has expired.  You can set the optional ``ttl_tolerance`` value in
-  your configuration JSON file like so:
+  your configuration JSON file like so::
 
-.. code-block:: json
+    {
+      "locker": {
+        "repo_url": "https://github.com/my-org/my-evidence-repo",
+        "ttl_tolerance": 3600
+      }
+    }
 
-   {
-     "locker": {
-       "repo_url": "https://github.com/my-org/my-evidence-repo",
-       "ttl_tolerance": 3600
-     }
-   }
+  Check execution is not affected by this optional
+  tolerance value because checks should only interact with evidence that
+  is fresh (not stale).
 
 * It's generally a good idea to regularly "archive" an evidence locker in
   favor of a fresh one.  A yearly locker archive/refresh is a good guideline
@@ -114,38 +119,58 @@ for:
   locker is possible by using the ``prev_repo_url`` option.  With that
   option set, a check that is unable to find historical evidence in the
   current evidence locker will be able to download the previous locker and
-  look for the historical evidence there.  This will continue to do this until
-  the new locker is primed with enough historical evidence to support all
-  checks.  Setting the option in your configuration JSON file would look
-  similar to:
+  look for the historical evidence there.  Setting the option in your configuration JSON file would look
+  similar to::
 
-.. code-block:: json
+    {
+      "locker": {
+        "repo_url": "https://github.com/my-org/my-evidence-repo",
+        "prev_repo_url": "https://github.com/my-org/my-evidence-repo-old"
+      }
+    }
 
-   {
-     "locker": {
-       "repo_url": "https://github.com/my-org/my-evidence-repo",
-       "prev_repo_url": "https://github.com/my-org/my-evidence-repo-old"
-     }
-   }
+  The previous locker will no longer be downloaded once the new locker is
+  primed with enough historical evidence to support all checks.
 
 * A locker can grow large, causing CI/CD jobs to run longer than desired
   due to locker download time.  So in addition to a sound locker archiving
   strategy, it is also possible to configure your locker to only download
-  recent commits by using the ``shallow_days`` option.  When ``shallow_days``
-  is supplied, only commits since the current date minus the number of days set
-  as ``shallow_days`` are included in the locker download.  The option applies
-  to both the locker and the previous locker (if applicable).  Setting the
+  recent commits by using the ``shallow_days`` option.  Setting the
   option in your configuration JSON file would look similar to::
 
-.. code-block:: json
+    {
+      "locker": {
+        "repo_url": "https://github.com/my-org/my-evidence-repo",
+        "prev_repo_url": "https://github.com/my-org/my-evidence-repo-old",
+        "shallow_days": 10
+      }
+    }
+
+  When ``shallow_days`` is supplied, only commits since the current date minus
+  the number of days set as ``shallow_days`` are included in the locker
+  download.  The option applies to both the locker and the previous locker (if
+  applicable).
+
+* Remote hosting services (Github, Gitlab, BitBucket) typically have file size
+  limitations that can vary from service instance to service instance.
+  Exceeding a maximum file size will in turn cause the service managing your
+  evidence locker to reject a remote locker Git push request.  Unfortunately
+  rejection notices from a service aren't always the most descriptive so it
+  often isn't clear why your push request was rejected.  To that end, prior to
+  a remote push, the framework will log a list of "largely sized" files.  The
+  large file size threshold is configurable and can be set by using the
+  ``large_file_threshold`` option.  The value is in bytes and defaults to
+  50 MB.  Setting the option in your configuration JSON file would look similar
+  to::
+
+    {
+      "locker": {
+        "repo_url": "https://github.com/my-org/my-evidence-repo",
+        "large_file_threshold": 50000000
+      }
+    }
 
-   {
-     "locker": {
-       "repo_url": "https://github.com/my-org/my-evidence-repo",
-       "prev_repo_url": "https://github.com/my-org/my-evidence-repo-old",
-       "shallow_days": 10
-     }
-   }
+  This should hopefully add some detail to a remote Git push rejection.
 
 
 .. _fetchers:

test/t_compliance/t_locker/test_locker.py

@@ -281,6 +281,25 @@ def test_empty_evidence(self):
                 ]
             )
 
+    def test_large_files(self):
+        """Test paths to large files are returned."""
+        with Locker(name=REPO_DIR) as locker:
+            large = RawEvidence(
+                'large.txt', 'test_category', DAY, 'Large evidence'
+            )
+            large.set_content('X' * 10000)
+            locker.add_evidence(large)
+            small = RawEvidence(
+                'small.txt', 'test_category', DAY, 'Small evidence'
+            )
+            small.set_content('X' * 10)
+            locker.add_evidence(small)
+            locker.checkin()
+            self.assertEqual(
+                locker.get_large_files(9999),
+                {'raw/test_category/large.txt': 10000}
+            )
+
     def test_add_partitioned_evidence(self):
         """Test that partitioned evidence is added to locker as expected."""
         with Locker(name=REPO_DIR) as locker: