Entropy validation refactoring (#700)

* Entropy refactoring

* fix

* reduce github_pat_ pattern

* docs

* lxml==5.3.2

* import optimization

* BM ref actual

Author: babenek

.ci/benchmark.txt

@@ -31,7 +31,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23
         with:
           repository: Samsung/CredData
-          ref: 4e9ebe54a749df6f1737737313b3b834a8a06429
+          ref: f3f609275a4147b8b1b7075eef5af318ba8c8a2f
 
       - name: Markup hashing
         run: |
@@ -87,7 +87,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23
         with:
           repository: Samsung/CredData
-          ref: 4e9ebe54a749df6f1737737313b3b834a8a06429
+          ref: f3f609275a4147b8b1b7075eef5af318ba8c8a2f
 
       - name: Markup hashing
         run: |
@@ -190,7 +190,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23
         with:
           repository: Samsung/CredData
-          ref: 4e9ebe54a749df6f1737737313b3b834a8a06429
+          ref: f3f609275a4147b8b1b7075eef5af318ba8c8a2f
 
       - name: Markup hashing
         run: |
@@ -378,7 +378,7 @@ jobs:
         uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - 2024.10.23
         with:
           repository: Samsung/CredData
-          ref: 4e9ebe54a749df6f1737737313b3b834a8a06429
+          ref: f3f609275a4147b8b1b7075eef5af318ba8c8a2f
 
       - name: Markup hashing
         run: |

.github/workflows/benchmark.yml

@@ -885,7 +885,7 @@ mbler
 mean
 measur
 medi
-medusa
+medus
 meet
 mem_
 memb
@@ -925,7 +925,7 @@ month
 morp
 mory
 mote
-motorola
+motor
 mount
 move
 mpeg
@@ -1005,6 +1005,7 @@ origin
 orithm
 ormat
 orph
+otorola
 ottle
 ously
 out
@@ -1485,6 +1486,7 @@ up_
 updat
 upgrade
 url
+usa
 usb
 use
 usin

credsweeper/common/morpheme_checklist.txt

@@ -393,7 +393,7 @@ def to_json(self, hashed: bool, subtext: bool) -> Dict:
         cut_pos = StartEnd(self.variable_start if 0 <= self.variable_start else self.value_start,
                            self.value_end) if subtext else None
         if isinstance(self.value, str):
-            entropy = round(Util.get_shannon_entropy(self.value, string.printable), 5)
+            entropy = round(Util.get_shannon_entropy(self.value), 5)
         else:
             entropy = None
         full_output = {

credsweeper/credentials/line_data.py

@@ -1,7 +1,6 @@
 import contextlib
 import json
 
-from credsweeper.common.constants import Chars
 from credsweeper.config import Config
 from credsweeper.credentials import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
@@ -45,7 +44,7 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
                 # must be all parts in payload
                 return True
             min_entropy = ValueEntropyBase64Check.get_min_data_entropy(len(parts[2]))
-            entropy = Util.get_shannon_entropy(parts[2], Chars.BASE64URL_CHARS.value)
+            entropy = Util.get_shannon_entropy(parts[2])
             # good signature has to be like random bytes
             return entropy < min_entropy
 

credsweeper/filters/value_azure_token_check.py

@@ -1,6 +1,7 @@
 import contextlib
 import re
 import statistics
+from itertools import takewhile
 
 from credsweeper.common.constants import Chars
 from credsweeper.config import Config
@@ -16,8 +17,8 @@ class ValueBase64PartCheck(Filter):
     Check that candidate is NOT a part of base64 long line
     """
 
-    base64_pattern = re.compile(r"^(\\{1,8}[0abfnrtv]|[0-9A-Za-z+/=]){1,4000}")
-    base64_set = set(Chars.BASE64STDPAD_CHARS.value)
+    base64_pattern = re.compile(r"^(\\{1,8}[0abfnrtv]|[0-9A-Za-z+/=]){1,4000}$")
+    base64_char_set = set(Chars.BASE64STDPAD_CHARS.value + '\\')
 
     def __init__(self, config: Config = None) -> None:
         pass
@@ -64,38 +65,46 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
                 elif right_end - left_start >= 2 * len_value:
                     # simple analysis for data too large to yield sensible insights
                     part_set = set(line[left_start:right_end])
-                    if not part_set.difference(self.base64_set):
+                    if not part_set.difference(ValueBase64PartCheck.base64_char_set):
                         # obvious case: all characters are base64 standard
                         return True
 
-                left_part = line[left_start:line_data.value_start]
-                len_left = len(left_part)
-                right_part = line[line_data.value_end:right_end]
-                len_right = len(right_part)
+                left_part = ''.join(
+                    takewhile(lambda x: x in ValueBase64PartCheck.base64_char_set,
+                              reversed(line[left_start:line_data.value_start])))
+
+                right_part = ''.join(
+                    takewhile(lambda x: x in ValueBase64PartCheck.base64_char_set, line[line_data.value_end:right_end]))
 
                 min_entropy_value = ValueEntropyBase64Check.get_min_data_entropy(len_value)
-                value_entropy = Util.get_shannon_entropy(value, Chars.BASE64STD_CHARS.value)
 
-                if ValueEntropyBase64Check.min_length < len_left:
-                    left_entropy = Util.get_shannon_entropy(left_part, Chars.BASE64STD_CHARS.value)
-                    if len_left < len_value:
-                        left_entropy *= len_value / len_left
-                else:
-                    left_entropy = min_entropy_value
+                left_entropy = Util.get_shannon_entropy(left_part)
+                value_entropy = Util.get_shannon_entropy(value)
+                right_entropy = Util.get_shannon_entropy(right_part)
+                common = left_part + value + right_part
+                common_entropy = Util.get_shannon_entropy(common)
+                min_entropy_common = ValueEntropyBase64Check.get_min_data_entropy(len(common))
+                if min_entropy_common < common_entropy:
+                    return True
 
-                if ValueEntropyBase64Check.min_length < len_right:
-                    right_entropy = Util.get_shannon_entropy(right_part, Chars.BASE64STD_CHARS.value)
-                    if len_right < len_value:
-                        left_entropy *= len_right / len_left
+                if left_entropy and right_entropy:
+                    data = [left_entropy, value_entropy, right_entropy, min_entropy_value, common_entropy]
+                elif left_entropy and not right_entropy:
+                    data = [left_entropy, value_entropy, min_entropy_value, min_entropy_value, common_entropy]
+                elif not left_entropy and right_entropy:
+                    data = [value_entropy, right_entropy, min_entropy_value, min_entropy_value, common_entropy]
                 else:
-                    right_entropy = min_entropy_value
+                    return False
 
-                data = [left_entropy, value_entropy, right_entropy, min_entropy_value]
                 avg = statistics.mean(data)
                 stdev = statistics.stdev(data, avg)
                 avg_min = avg - 1.1 * stdev
-                if avg_min <= left_entropy and avg_min <= right_entropy:
+                if (0. == left_entropy or avg_min < left_entropy or left_entropy < value_entropy < right_entropy) \
+                        and (
+                        0. == right_entropy or avg_min < right_entropy or right_entropy < value_entropy < left_entropy):
                     # high entropy of bound parts looks like a part of base64 long line
                     return True
+                else:
+                    return False
 
         return False

credsweeper/filters/value_base64_part_check.py

@@ -1,6 +1,5 @@
 import contextlib
 
-from credsweeper.common.constants import Chars
 from credsweeper.config import Config
 from credsweeper.credentials import LineData
 from credsweeper.file_handler.analysis_target import AnalysisTarget
@@ -32,7 +31,7 @@ def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
             id_part = line_data.value[:dot_separator_index]
             discord_id = int(Util.decode_base64(id_part, padding_safe=True, urlsafe_detect=True))
             entropy_part = line_data.value[dot_separator_index:]
-            entropy = Util.get_shannon_entropy(entropy_part, Chars.BASE64URL_CHARS.value)
+            entropy = Util.get_shannon_entropy(entropy_part)
             min_entropy = ValueEntropyBase64Check.get_min_data_entropy(len(entropy_part))
             if 1000 <= discord_id and min_entropy <= entropy:
                 return False

credsweeper/filters/value_discord_bot_check.py

@@ -1,42 +1,26 @@
 import math
+from functools import cache
 
-from credsweeper.common.constants import Chars
 from credsweeper.config import Config
-from credsweeper.credentials import LineData
-from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
+from credsweeper.filters.value_entropy_base_check import ValueEntropyBaseCheck
 
 
-class ValueEntropyBase32Check(Filter):
-    """Check that candidate have Shanon Entropy (for [a-z0-9])"""
+class ValueEntropyBase32Check(ValueEntropyBaseCheck):
+    """Base32 entropy check"""
 
     def __init__(self, config: Config = None) -> None:
-        pass
+        super().__init__(config)
 
     @staticmethod
+    @cache
     def get_min_data_entropy(x: int) -> float:
         """Returns average entropy for size of random data. Precalculated data is applied for speedup"""
-        if 16 == x:
-            y = 3.46
-        elif 10 <= x:
-            # approximation does not exceed stdev
-            y = 0.64 * math.log2(x) + 0.9
+        if 8 <= x < 17:
+            y = 0.80569236 * math.log2(x) + 0.13439734
+        elif 17 <= x < 33:
+            y = 0.66350481 * math.log2(x) + 0.71143862
+        elif 33 <= x:
+            y = 4.04
         else:
             y = 0
         return y
-
-    def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
-        """Run filter checks on received credential candidate data 'line_data'.
-
-        Args:
-            line_data: credential candidate data
-            target: multiline target from which line data was obtained
-
-        Return:
-            True, if need to filter candidate and False if left
-
-        """
-        entropy = Util.get_shannon_entropy(line_data.value, Chars.BASE32_CHARS.value)
-        min_entropy = ValueEntropyBase32Check.get_min_data_entropy(len(line_data.value))
-        return min_entropy > entropy or 0 == min_entropy

credsweeper/filters/value_entropy_base32_check.py

@@ -1,46 +1,27 @@
 import math
+from functools import cache
 
-from credsweeper.common.constants import Chars
 from credsweeper.config import Config
-from credsweeper.credentials import LineData
-from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
+from credsweeper.filters.value_entropy_base_check import ValueEntropyBaseCheck
 
 
-class ValueEntropyBase36Check(Filter):
-    """Check that candidate have Shanon Entropy (for [a-z0-9])"""
+class ValueEntropyBase36Check(ValueEntropyBaseCheck):
+    """Base36 entropy check"""
 
     def __init__(self, config: Config = None) -> None:
-        pass
+        super().__init__(config)
 
     @staticmethod
+    @cache
     def get_min_data_entropy(x: int) -> float:
         """Returns minimal entropy for size of random data. Precalculated data is applied for speedup"""
         if 15 == x:
-            y = 3.43
-        elif 24 == x:
-            y = 3.91
-        elif 25 == x:
-            y = 3.95
-        elif 10 <= x:
-            # approximation does not exceed standard deviation
-            y = 0.7 * math.log2(x) + 0.7
+            # workaround for Dropbox App secret
+            y = 3.374
+        elif 10 <= x < 26:
+            y = 0.731566857 * math.log2(x) + 0.474132
+        elif 26 <= x:
+            y = 3.9
         else:
             y = 0
         return y
-
-    def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
-        """Run filter checks on received credential candidate data 'line_data'.
-
-        Args:
-            line_data: credential candidate data
-            target: multiline target from which line data was obtained
-
-        Return:
-            True, if need to filter candidate and False if left
-
-        """
-        entropy = Util.get_shannon_entropy(line_data.value, Chars.BASE36_CHARS.value)
-        min_entropy = ValueEntropyBase36Check.get_min_data_entropy(len(line_data.value))
-        return min_entropy > entropy or 0 == min_entropy

credsweeper/filters/value_entropy_base36_check.py

@@ -1,59 +1,34 @@
 import math
+from functools import cache
 
-from credsweeper.common.constants import Chars, ENTROPY_LIMIT_BASE64
 from credsweeper.config import Config
-from credsweeper.credentials import LineData
-from credsweeper.file_handler.analysis_target import AnalysisTarget
-from credsweeper.filters import Filter
-from credsweeper.utils import Util
+from credsweeper.filters.value_entropy_base_check import ValueEntropyBaseCheck
 
 
-class ValueEntropyBase64Check(Filter):
-    """Check that candidate have Shanon Entropy > 3 (for HEX_CHARS or BASE36_CHARS) or > 4.5 (for BASE64_CHARS)."""
-
-    # If the value size is less than this value the entropy evaluation gives an imprecise result
-    min_length = 12
+class ValueEntropyBase64Check(ValueEntropyBaseCheck):
+    """Base64 entropy check"""
 
     def __init__(self, config: Config = None) -> None:
-        pass
+        super().__init__(config)
 
     @staticmethod
+    @cache
     def get_min_data_entropy(x: int) -> float:
         """Returns minimal average entropy for size of random data. Precalculated round data is applied for speedup"""
-        if 18 == x:
-            y = 3.8
-        elif 20 == x:
-            y = 3.9
-        elif 24 == x:
-            y = 4.1
-        elif 32 == x:
-            y = 4.4
-        elif ValueEntropyBase64Check.min_length <= x < 35:
-            # logarithm base 2 - slow, but precise. Approximation does not exceed stdev
-            y = 0.77 * math.log2(x) + 0.62
-        elif 35 <= x < 60:
-            y = ENTROPY_LIMIT_BASE64
-        elif 60 <= x:
-            # the entropy grows slowly after 60
-            y = 5.0
+        if 12 <= x < 18:
+            y = 0.915 * math.log2(x) - 0.047
+        elif 18 <= x < 35:
+            y = 0.767 * math.log2(x) + 0.5677
+        elif 35 <= x < 65:
+            y = 0.944 * math.log2(x) - 0.009 * x - 0.04
+        elif 65 <= x < 256:
+            y = 0.621 * math.log2(x) - 0.003 * x + 1.54
+        elif 256 <= x < 512:
+            y = 5.77
+        elif 512 <= x < 1024:
+            y = 5.89
+        elif 1024 <= x:
+            y = 5.94
         else:
             y = 0
         return y
-
-    def run(self, line_data: LineData, target: AnalysisTarget) -> bool:
-        """Run filter checks on received credential candidate data 'line_data'.
-
-        Args:
-            line_data: credential candidate data
-            target: multiline target from which line data was obtained
-
-        Return:
-            True, if need to filter candidate and False if left
-
-        """
-        if '-' in line_data.value or '_' in line_data.value:
-            entropy = Util.get_shannon_entropy(line_data.value, Chars.BASE64URL_CHARS.value)
-        else:
-            entropy = Util.get_shannon_entropy(line_data.value, Chars.BASE64STD_CHARS.value)
-        min_entropy = ValueEntropyBase64Check.get_min_data_entropy(len(line_data.value))
-        return min_entropy > entropy or 0 == min_entropy