Merge pull request #63 from NetherlandsForensicInstitute/add_check_if_contains

Added check if contains option.

Author: zyronix

bin/demeuk.py

@@ -66,6 +66,8 @@
                                         with as comma-seperated list.
         --check-ending-with <string>    Drop lines ending with string, can be multiple strings. Specify multiple
                                         with as comma-seperated list.
+        --check-contains <string>       Drop lines containing string, can be multiple strings. Specify multiple
+                                        with as comma-seperated list.
         --check-empty-line              Drop lines that are empty or only contain whitespace characters
         --check-regex <string>          Drop lines that do not match the regex. Regex is a comma seperated list of
                                         regexes. Example: [a-z]{1,8},[0-9]{1,8}
@@ -169,7 +171,7 @@
 from unidecode import unidecode
 
 
-version = '4.4.0'
+version = '4.5.0'
 
 # Search from start to finish for the string $HEX[], with block of a-f0-9 with even number
 # of hex chars. The first match group is repeated.
@@ -615,6 +617,23 @@ def check_ending_with(line, strings):
     return False
 
 
+def check_contains(line, strings):
+    """Checks if a line does not contain specific strings
+
+    Params:
+        line (unicode)
+        strings[str]
+
+    Returns:
+        true if line does contain any one of the strings
+
+    """
+    for string in strings:
+        if string in line:
+            return True
+    return False
+
+
 def check_empty_line(line):
     """Checks if a line is empty or only contains whitespace chars
 
@@ -1231,6 +1250,12 @@ def clean_up(lines):
                 log.append(f'Check_ending_with; dropped line because {to_check} found; {line_decoded}{linesep}')
                 stop = True
 
+        if config.get('check-contains') and not stop:
+            to_check = config.get("check-contains")
+            if check_contains(line_decoded, to_check):
+                log.append(f'Check-contains; dropped line because {to_check} found; {line_decoded}{linesep}')
+                stop = True
+
         if config.get('check-empty-line') and not stop:
             if check_empty_line(line_decoded):
                 log_line = "Check_empty_line; dropped line because is empty or only contains whitespace;"
@@ -1390,6 +1415,7 @@ def main():
         'check-starting-with': False,
         'check-uuid': False,
         'check-ending-with': False,
+        'check-contains': False,
         'check-empty-line': False,
         'check-regex': False,
         'check-min-digits': 0,
@@ -1550,6 +1576,12 @@ def main():
         else:
             config['check-ending-with'] = [arguments.get('--check-ending-with')]
 
+    if arguments.get('--check-contains'):
+        if ',' in arguments.get('--check-contains'):
+            config['check-contains'] = arguments.get('--check-contains').split(',')
+        else:
+            config['check-contains'] = [arguments.get('--check-contains')]
+
     if arguments.get('--check-empty-line'):
         config['check-empty-line'] = True
 

docs/usage.rst

@@ -292,14 +292,24 @@ characters are transfered to ':'.
 check-ending-with
 ~~~~~~~~~~~~~~~~~
 Checks if a line ends with the argument of check-ending-with. If the line ends
-with this, the line will be dropped. The string to check can be multiple strings. multiple
+with this, the line will be dropped. The string to check can be multiple strings. Multiple
 values are comma-seperated. Example: #,// would skip lines ending with '#' and with 
 '//'.
 
 If you enabled the '--tab' option and you want to drop lines ending with a tab, add
 ':' to the list of strings to check. '--check ending-with :'. When using --tab tab
 characters are transfered to ':'.
 
+check-contains
+~~~~~~~~~~~~~~
+Checks if a line contains the argument of check-contains. If the line contains this,
+the line will be dropped. The string to check can be multiple strings. Multiple values
+are comma-separated. Example: #,// would skip lines containing '#' and '//'.
+
+If you enabled the '--tab' option and you want to drop lines ending with a tab, add
+':' to the list of strings to check. '--check ending-with :'. When using --tab tab
+characters are transfered to ':'.
+
 check-empty-line
 ~~~~~~~~~~~~~~~~
 Checks if a line only contains whitespace characters or is empty. If this is true,
@@ -312,6 +322,13 @@ matches all of the regexes, the line will be dropped.
 Example: --check-regex '[a-z],[0-9]' will drop lines
 that do not atleast contain one lowercase char and one number.
 
+Want to remove a line that does not contain an underscore?
+--check-regex '^[^_]+$'
+
+Want to remove a line that start with a specific strings?
+--check-regex '^[^this]' will remove lines starting with 'this'
+
+
 check-min-digits
 ~~~~~~~~~~~~~~~~
 Checks if a line contains a minimum number of digit characters. If the line does not contain

tests/conftest.py

@@ -387,3 +387,9 @@
     file.write(f'amsterdam {linesep}')
     file.write(f'ROTTERDAM {linesep}')
     file.write(f'Cookie Monster {linesep}')
+
+with open('testdata/input53', 'w') as file:
+    file.write(f'three_down {linesep}')
+    file.write(f'_amsterdam {linesep}')
+    file.write(f'ROTTERDAM_ {linesep}')
+    file.write(f'Cookie Monster {linesep}')

tests/test_app.py

@@ -965,3 +965,20 @@ def test_add_title_case():
     assert 'Amsterdam' in filecontent
     assert 'Rotterdam' in filecontent
     assert 'Cookie Monster' in filecontent
+
+
+def test_check_contains():
+    testargs = [
+        'demeuk', '-i', 'testdata/input53', '-o', 'testdata/output53', '-l', 'testdata/log53',
+        '--verbose', '--check-contains', '_',
+    ]
+    with patch.object(sys, 'argv', testargs):
+        main()
+
+    with open('testdata/output53') as f:
+        filecontent = f.read()
+
+    assert 'three_down' not in filecontent
+    assert '_amsterdam' not in filecontent
+    assert 'ROTTERDAM_' not in filecontent
+    assert 'Cookie Monster' in filecontent