Add filtered content flag and clone logging (#112)

- Add filtered content flag
- Set content flags (including binary content) on metadata
- Add Clone duration logging

Author: alfinkel

CHANGES.md

@@ -1,3 +1,10 @@
+# [1.14.0](https://github.com/ComplianceAsCode/auditree-framework/releases/tag/v1.14.0)
+
+- [ADDED] The `filtered_content` attribute has been added to `RawEvidence`.
+- [ADDED] Locker clone duration logging has been added.
+- [FIXED] The `binary_content` attribute on raw evidence is retained as metadata now.
+- [FIXED] All partitioned evidence defined via constructor correctly retains attributes now.
+
 # [1.13.0](https://github.com/ComplianceAsCode/auditree-framework/releases/tag/v1.13.0)
 
 - [ADDED] Configurable shallow cloning of locker is now supported.

compliance/__init__.py

@@ -14,4 +14,4 @@
 # limitations under the License.
 """Compliance automation package."""
 
-__version__ = '1.13.0'
+__version__ = '1.14.0'

compliance/evidence.py

@@ -37,6 +37,8 @@
 DAY = HOUR * 24
 YEAR = DAY * 365
 
+CONTENT_FLAGS = ['binary_content', 'filtered_content']
+
 LazyLoader = namedtuple('LazyLoader', 'path ev_class')
 
 
@@ -66,11 +68,21 @@ def __init__(self, name, category, ttl=DAY, description='', **kwargs):
 
     @classmethod
     def from_evidence(cls, evidence):
+        kwargs = {}
+        for content_flag in CONTENT_FLAGS:
+            if hasattr(evidence, content_flag):
+                kwargs[content_flag] = getattr(evidence, content_flag)
+        if hasattr(evidence, 'part_fields') or hasattr(evidence, 'part_root'):
+            kwargs['partition'] = {
+                'fields': getattr(evidence, 'part_fields', None),
+                'root': getattr(evidence, 'part_root', None)
+            }
         new_evidence = cls(
             evidence.name,
             evidence.category,
             evidence.ttl,
-            evidence.description
+            evidence.description,
+            **kwargs
         )
         if evidence.content:
             new_evidence.set_content(evidence.content)
@@ -144,6 +156,7 @@ def __init__(self, *args, **kwargs):
         self.part_fields = partition.get('fields')
         self.part_root = partition.get('root')
         self.binary_content = kwargs.get('binary_content', False)
+        self.filtered_content = kwargs.get('filtered_content', False)
 
     @property
     def rootdir(self):

compliance/locker.py

@@ -26,7 +26,7 @@
 from urllib.parse import urlparse
 
 from compliance.config import get_config
-from compliance.evidence import TmpEvidence, get_evidence_class
+from compliance.evidence import CONTENT_FLAGS, TmpEvidence, get_evidence_class
 from compliance.utils.data_parse import (
     format_json, get_sha256_hash, parse_dot_key
 )
@@ -302,6 +302,9 @@ def index(self, evidence, checks=None, evidence_used=None):
                 repo_files.append(self.get_file(evidence.path))
             if tombstones:
                 metadata[evidence.name]['tombstones'] = tombstones
+            for content_flag in CONTENT_FLAGS:
+                if getattr(evidence, content_flag, False):
+                    metadata[evidence.name][content_flag] = True
             if checks is not None:
                 metadata[evidence.name]['checks'] = checks
             if evidence_used is not None:
@@ -575,16 +578,20 @@ def get_locker_repo(self, locker='evidence locker'):
             )
             kwargs = {'branch': self.branch}
             shallow_days = get_config().get('locker.shallow_days', -1)
+            addl_msg = None
             if shallow_days >= 0:
                 since_dt = dt.utcnow() - timedelta(days=shallow_days + 1)
                 since = since_dt.strftime('%Y/%m/%d')
                 kwargs['shallow_since'] = since
-                self.logger.info(
-                    f'{locker.title()} contains commits since {since}...'
-                )
+                addl_msg = f'{locker.title()} contains commits since {since}'
+            start = time.perf_counter()
             self.repo = git.Repo.clone_from(
                 self.repo_url_with_creds, self.local_path, **kwargs
             )
+            duration = time.perf_counter() - start
+            self.logger.info(f'{locker.title()} cloned in {duration:.3f}s')
+            if addl_msg:
+                self.logger.info(addl_msg)
 
     def init_config(self):
         """Apply the git configuration."""
@@ -815,7 +822,9 @@ def _get_evidence(self, evidence_path, ignore_ttl=False, evidence_dt=None):
                 partition={
                     'fields': metadata.get('partition_fields'),
                     'root': metadata.get('partition_root')
-                }
+                },
+                binary_content=metadata.get('binary_content', False),
+                filtered_content=metadata.get('filtered_content', False)
             )
         except TypeError:
             ev_dt_str = (evidence_dt or dt.utcnow()).strftime('%Y-%m-%d')