More precise testing of plugins

Instead of relying on a count of the severity and confidence
levels found within an example file, make use of Python's native
unit testing to verify the results of a plugin.

The existing method of confirming counts can be inaccurate. It's
very easy to have a false positive simply because one issue extra
was found and one issue was missed, thus giving the same count. It
tells nothing of the validation of a particular line of problematic
code.

Relates to #352

Signed-off-by: Eric Brown <[email protected]>

Author: ericwb

tests/functional/test_functional.py

@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: Apache-2.0
+import testtools
+
+from bandit.core import config
+from bandit.core import manager
+from bandit.core import meta_ast
+from bandit.core import metrics
+from bandit.core import node_visitor
+from bandit.core import test_set
+
+
+class BaseTestCase(testtools.TestCase):
+    def setUp(self, test_ids):
+        super().setUp()
+        b_config = config.BanditConfig()
+        self.b_manager = manager.BanditManager(b_config, "file")
+        issue_metrics = metrics.Metrics()
+        issue_metrics.begin("test.py")
+        self.visitor = node_visitor.BanditNodeVisitor(
+            "test.py",
+            None,
+            metaast=meta_ast.BanditMetaAst(),
+            testset=test_set.BanditTestSet(
+                b_config,
+                profile={
+                    "include": test_ids,
+                    "exclude": [],
+                },
+            ),
+            debug=False,
+            nosec_lines={},
+            metrics=issue_metrics,
+        )

tests/unit/plugins/__init__.py

@@ -0,0 +1,71 @@
+# SPDX-License-Identifier: Apache-2.0
+import textwrap
+
+import bandit
+from bandit.core import issue as b_issue
+from tests.unit.plugins import base_test_case
+
+
+class FlaskDebugTests(base_test_case.BaseTestCase):
+    def setUp(self):
+        super().setUp(["B201"])
+
+    def test_app_run_debug_true(self):
+        fdata = textwrap.dedent(
+            """
+            from flask import Flask
+            app = Flask(__name__)
+            app.run(debug=True)
+            """
+        )
+        self.visitor.process(fdata)
+        self.assertEqual(1, len(self.visitor.tester.results))
+        issue = self.visitor.tester.results[0]
+        self.assertEqual(bandit.HIGH, issue.severity)
+        self.assertEqual(bandit.MEDIUM, issue.confidence)
+        self.assertEqual(b_issue.Cwe.CODE_INJECTION, issue.cwe.id)
+        self.assertEqual(4, issue.lineno)
+        self.assertEqual([4], issue.linerange)
+        self.assertEqual(0, issue.col_offset)
+
+    def test_app_run_debug_false(self):
+        fdata = textwrap.dedent(
+            """
+            from flask import Flask
+            app = Flask(__name__)
+            app.run(debug=False)
+            """
+        )
+        self.visitor.process(fdata)
+        self.assertEqual(0, len(self.visitor.tester.results))
+
+    def test_app_run(self):
+        fdata = textwrap.dedent(
+            """
+            from flask import Flask
+            app = Flask(__name__)
+            app.run()
+            """
+        )
+        self.visitor.process(fdata)
+        self.assertEqual(0, len(self.visitor.tester.results))
+
+    def test_app_run_no_import(self):
+        fdata = textwrap.dedent(
+            """
+            app = Flask(__name__)
+            app.run(debug=True)
+            """
+        )
+        self.visitor.process(fdata)
+        self.assertEqual(0, len(self.visitor.tester.results))
+
+    def test_unrelated_run(self):
+        fdata = textwrap.dedent(
+            """
+            from flask import Flask
+            run(debug=True)
+            """
+        )
+        self.visitor.process(fdata)
+        self.assertEqual(0, len(self.visitor.tester.results))

tests/unit/plugins/base_test_case.py

@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: Apache-2.0
+import bandit
+from bandit.core import issue as b_issue
+from tests.unit.plugins import base_test_case
+
+
+class AssertsTests(base_test_case.BaseTestCase):
+    def setUp(self):
+        super().setUp(["B101"])
+
+    def test_asserts(self):
+        fdata = "assert True"
+        self.visitor.process(fdata)
+        self.assertEqual(1, len(self.visitor.tester.results))
+        issue = self.visitor.tester.results[0]
+        self.assertEqual(bandit.LOW, issue.severity)
+        self.assertEqual(bandit.HIGH, issue.confidence)
+        self.assertEqual(
+            b_issue.Cwe.IMPROPER_CHECK_OF_EXCEPT_COND, issue.cwe.id
+        )
+        self.assertEqual(1, issue.lineno)
+        self.assertEqual([1], issue.linerange)
+        self.assertEqual(0, issue.col_offset)