Merge pull request #430 from tableau/dev-tdvt

TDVT 2.1.4

Author: lukewrites

.gitignore

@@ -0,0 +1,147 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# VS Code files
+.vscode/
+
+# PyCharm
+.idea/
+
+# Connector Packager logs
+packager_tests_logs.txt
+packaging_logs.txt
+
+# TDVT logs
+tabquery_logs.zip
+tdvt_actuals_combined.zip
+tdvt_log_combined.txt
+tdvt_output_combined_failed.json
+tdvt_output_combined.json
+test_results_combined.csv

README.md

@@ -7,7 +7,7 @@ This project consists of documentation, example files, the Tableau Datasource Ve
 | Tool                                             | Latest Version     |
 |--------------------------------------------------|--------------------|
 | Connector Packager SDK (Beta) for Tableau 2019.3 | 12-11-2019         |
-| TDVT                                             | 2.1.3 (12-30-2019) |
+| TDVT                                             | 2.1.4 (02-02-2020) |
 | Connector Packager                               | 0.0.1 (10-03-2019) |
 
 * [Why Connectors?](#why-connectors)

tdvt/CHANGELOG.md

@@ -6,6 +6,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## Unreleased
 
+## [2.1.4] - 2020-02-02
+- Update smoke tests to count skipped & disabled tests as "pass". Also update the command line results from a TDVT run with more details.
+- Refactor list command. list is used with suites and list-logical-configs is just for logical configs.
+- Fixes test rerun bug.
+
 ## [2.1.3] - 2019-12-30
 - Add tests specific to Snowflake.
 
@@ -18,3 +23,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Compress the log file zip archive. This relies on the zlib module which seems to usually be installed by default.
 - More logging about test exclusions.
 - Update handling of args.run_file to use Path
+- Update TDVT to add `expected_message` to JSON output of failed tests.

tdvt/tdvt/config_gen/datasource_list.py

@@ -24,15 +24,15 @@ def print_ds(ds, ds_reg):
     for x in test_config.get_logical_tests():
         print("\t" * 2 + str(x))
         root_directory = get_root_dir()
-        tests = x.generate_test_file_list_from_config()
+        tests = x.generate_test_file_list()
         for test in tests:
             print("\t" * 3 + test.test_path)
 
     print("\tExpression tests:")
     for x in test_config.get_expression_tests():
         print("\t" * 2 + str(x))
         root_directory = get_root_dir()
-        tests = x.generate_test_file_list_from_config()
+        tests = x.generate_test_file_list()
         for test in tests:
             print("\t" * 3 + test.test_path)
 

tdvt/tdvt/config_gen/test_config.py

@@ -6,6 +6,7 @@
 import glob
 import re
 from ..resources import *
+from ..tabquery_path import TabQueryPath
 
 class TestFile(object):
     """
@@ -26,7 +27,8 @@ class TestSet(object):
         Represents everything needed to run a set of tests. This includes a path to the test files, which tds etc.
     """
     def __init__(self, ds_name, root_dir, config_name, tds_name, exclusions, test_pattern, is_logical, suite_name, password_file,
-                 expected_message, smoke_test=False, test_is_enabled=True, test_is_skipped=False):
+                 expected_message: str = '', smoke_test: bool = False, test_is_enabled: bool = True,
+                 test_is_skipped: bool = False):
         self.ds_name = ds_name
         self.suite_name = suite_name
         self.config_name = config_name
@@ -40,6 +42,8 @@ def __init__(self, ds_name, root_dir, config_name, tds_name, exclusions, test_pa
         self.smoke_test = smoke_test
         self.test_is_enabled = test_is_enabled
         self.test_is_skipped = test_is_skipped
+        self.test_list_cached = None
+        self.test_list_checked = False
 
 
     def is_logical_test(self):
@@ -62,24 +66,20 @@ def generate_test_file_list(self):
            Return the sorted list of tests.
 
         """
-        final_test_list = self.generate_test_file_list_from_config()
+        if self.test_list_checked:
+            return self.test_list_cached
 
-        logging.debug("Found final list of " + str(len(final_test_list)) + " tests to run.")
-        if len(final_test_list) == 0:
-            logging.warning("Did not find any tests to run.")
-            print("Did not find any tests to run.")
-            return final_test_list
+        final_test_list = self.__generate_test_file_list()
 
-        for x in final_test_list:
-            logging.debug("final test path " + x.test_path)
-
-        return final_test_list
+        self.test_list_cached = final_test_list
+        self.test_list_checked = True
+        return self.test_list_cached
 
     def get_test_dirs(self):
         return (self.root_dir, get_local_test_dir())
 
-    def generate_test_file_list_from_config(self):
-        """Read the config file and generate a list of tests."""
+    def __generate_test_file_list(self):
+        """Private function to generate the list of tests."""
         allowed_tests = []
         exclude_tests = self.get_exclusions()
         exclude_tests.append('expected.')
@@ -181,7 +181,7 @@ def get_actual_and_base_file_path(self, test_file, output_dir):
     def append_test_file(self, file_path):
         self.test_paths.append(file_path)
 
-    def generate_test_file_list_from_config(self):
+    def generate_test_file_list(self):
         tests_to_run = []
         for test_file in self.test_paths:
             added_test = False
@@ -249,30 +249,6 @@ def build_config_name(prefix, dsname):
 def build_tds_name(prefix, dsname):
     return prefix + dsname + '.tds'
 
-class TabQueryPath(object):
-    def __init__(self, linux_path, mac_path, windows_path):
-        self.linux_path = linux_path
-        self.mac_path = mac_path
-        self.windows_path = windows_path
-
-    @staticmethod
-    def from_array(paths):
-        if len(paths) != 3:
-            raise IndexError
-        t = TabQueryPath(paths[0], paths[1], paths[2])
-        return t
-
-    def to_array(self):
-        return [self.linux_path, self.mac_path, self.windows_path]
-
-    def get_path(self, os):
-        if os.startswith("darwin"):
-            return self.mac_path
-        elif os.startswith("linux"):
-            return self.linux_path
-        else:
-            return self.windows_path
-
 class RunTimeTestConfig(object):
     """
         Tracks specifics about how a group of tests were run.

tdvt/tdvt/tabquery.py

@@ -2,6 +2,8 @@
 import sys
 
 from .resources import *
+from .tabquery_path import TabQueryPath
+
 tab_cli_exe = ''
 
 def configure_tabquery_path():
@@ -88,8 +90,14 @@ def build_tabquery_command_line(self, work):
         work.test_config.command_line = cmdline
         return cmdline
 
-def tabquerycli_exists():
+def tabquerycli_exists(tabquery_cli_path: TabQueryPath = None):
     global tab_cli_exe
+    if tabquery_cli_path:
+        resolved_path = tabquery_cli_path.get_path(sys.platform)
+        if os.path.isfile(resolved_path):
+            logging.debug("Found tabquery at [{0}]".format(resolved_path))
+            return True
+
     if os.path.isfile(tab_cli_exe):
         logging.debug("Found tabquery at [{0}]".format(tab_cli_exe))
         return True

tdvt/tdvt/tabquery_path.py

@@ -0,0 +1,23 @@
+class TabQueryPath(object):
+    def __init__(self, linux_path, mac_path, windows_path):
+        self.linux_path = linux_path
+        self.mac_path = mac_path
+        self.windows_path = windows_path
+
+    @staticmethod
+    def from_array(paths):
+        if len(paths) != 3:
+            raise IndexError
+        t = TabQueryPath(paths[0], paths[1], paths[2])
+        return t
+
+    def to_array(self):
+        return [self.linux_path, self.mac_path, self.windows_path]
+
+    def get_path(self, os):
+        if os.startswith("darwin"):
+            return self.mac_path
+        elif os.startswith("linux"):
+            return self.linux_path
+        else:
+            return self.windows_path