Fix functional test suite on Python 3 and modern pytest (#883)

Author: xbmc4lyfe

docs/FUNCTIONAL_TESTING.md

@@ -1,5 +1,3 @@
-**Note: The current implementation of functional tests is broken and requires updating.**
-
 ## Functional testing
 
 During functional testing the program is tested as a whole. This is much like if the testing were made by a human. Although the user interface parts are not tested the core of the program is tested and how the different program components work together. When functional tests succeed that means that the tested scenarios will most likely work properly when the program is run by real users. The quality of total testing obviously depends on the amount and quality of individual tests.
@@ -28,11 +26,11 @@ Technically NServ is integrated into NZBGet as a sub-module. NServ is automatica
 
 ## Testing framework
 
-Functional tests for NZBGet are written in python language. We are using testing framework [py.test](https://pytest.org) to organize test execution and collect test results. Therefore in order to run functional tests py.test [must be installed](https://docs.pytest.org/en/latest/getting-started.html) on your system.
+Functional tests for NZBGet are written in python language (Python 3). We are using testing framework [pytest](https://pytest.org) to organize test execution and collect test results. Therefore in order to run functional tests pytest [must be installed](https://docs.pytest.org/en/latest/getting-started.html) on your system.
 
 ## Running tests
 
-Functional tests and supporting modules are stored in directory `tests/functional`. To run tests open command prompt in this directory and execute `py.test` as explained below.
+Functional tests and supporting modules are stored in directory `tests/functional`. To run tests open command prompt in this directory and execute `pytest` as explained below.
 
 ### Configuring tests
 
@@ -55,33 +53,37 @@ All settings in the ini-file are optional.
 When executing the tests for the first time the test scripts prepare test files and put them into directory `tests/testdata/nserv.temp`. These files take several gigabytes and may need several minutes to generate. When starting tests for the first time it’s recommended to add parameter `-s` to see additional progress logging during preparation stage:
 
 ```bash
-  py.test -v -s
+  pytest -v -s
 ```
 
 ### Executing tests
 
 To run all tests use simple command:
 
 ```bash
-  py.test -v
+  pytest -v
 ```
 
 To run tests from one test script only pass the specific file name:
 
 ```bash
-  py.test -v parcheck\parcheck_force_test.py
+  pytest -v parcheck/parcheck_force_test.py
 ```
 
 To execute only one specific test pass the file name and test function name:
 
 ```bash
-  py.test -v parcheck/parcheck_auto_test.py::test_parchecker_repair
+  pytest -v parcheck/parcheck_auto_test.py::test_parchecker_repair
 ```
 
-For more filter possibilities please see [py.test](https://docs.pytest.org/en/latest/usage.html) documentation.
+For more filter possibilities please see [pytest](https://docs.pytest.org/en/latest/usage.html) documentation.
 
 ### Test failures
 
 Directory used by NZBGet during testing (`tests/testdata/nzbget.temp` by default) is automatically deleted if all tests succeed. If a test fails the directory is kept in order to preserve log-files and queue-files for failure analysis. However during testing NZBGet is started and stopped multiple times. If the failed test wasn’t the last test the preserved NZBGet directory may not contain data of the failed test. In such case it’s better to rerun the specific failed test as the only test (using specific command line).
 
 After test completion NZBGet is terminated. To analyze a test failure it can be useful to have NZBGet running, for example to inspect the state in web-interface. Pass extra parameter `--hold` to achieve this:
+
+```bash
+  pytest -v parcheck/parcheck_auto_test.py::test_parchecker_repair --hold
+```

tests/functional/conftest.py

@@ -4,11 +4,7 @@
 import time
 import shutil
 import base64
-import distutils.spawn
-try:
-	from xmlrpclib import ServerProxy # python 2
-except ImportError:
-	from xmlrpc.client import ServerProxy # python 3
+from xmlrpc.client import ServerProxy
 
 nzbget_srcdir = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
 nzbget_maindir = nzbget_srcdir + '/tests/testdata/nzbget.temp'
@@ -20,13 +16,9 @@
 nzbget_bin = nzbget_srcdir + '/nzbget' + exe_ext
 nserv_datadir = nzbget_srcdir + '/tests/testdata/nserv.temp'
 
-sevenzip_bin = distutils.spawn.find_executable('7z')
-if sevenzip_bin is None:
-	sevenzip_bin = nzbget_srcdir + '/7z' + exe_ext
+sevenzip_bin = shutil.which('7z') or nzbget_srcdir + '/7z' + exe_ext
 
-par2_bin = distutils.spawn.find_executable('par2')
-if par2_bin is None:
-	par2_bin = nzbget_srcdir + '/par2' + exe_ext
+par2_bin = shutil.which('par2') or nzbget_srcdir + '/par2' + exe_ext
 
 has_failures = False
 
@@ -38,30 +30,29 @@ def pytest_addoption(parser):
 	parser.addini('par2_bin', 'path to par2 binary', default=par2_bin)
 	parser.addoption("--hold", action="store_true", help="Hold at the end of test (keep NZBGet running)")
 
-def check_config():
+@pytest.fixture(scope='session')
+def check_config(request):
 	global nzbget_bin
-	nzbget_bin = pytest.config.getini('nzbget_bin')
+	nzbget_bin = request.config.getini('nzbget_bin')
 	if not os.path.exists(nzbget_bin):
 		pytest.exit('Could not find nzbget binary at ' + nzbget_bin + '. Alternative path can be set via pytest ini option "nzbget_bin".')
 
 	global sevenzip_bin, par2_bin
-	sevenzip_bin = pytest.config.getini('sevenzip_bin')
-	par2_bin = pytest.config.getini('par2_bin')
+	sevenzip_bin = request.config.getini('sevenzip_bin')
+	par2_bin = request.config.getini('par2_bin')
 	if not os.path.exists(sevenzip_bin):
 		pytest.exit('Could not find 7-zip binary in search path or at ' + sevenzip_bin + '. Alternative path can be set via pytest ini option "sevenzip_bin".')
 	if not os.path.exists(par2_bin):
 		pytest.exit('Could not find par2 binary in search path or at ' + par2_bin + '. Alternative path can be set via pytest ini option "par2_bin".')
 
 	global nserv_datadir
-	nserv_datadir = pytest.config.getini('nserv_datadir')
+	nserv_datadir = request.config.getini('nserv_datadir')
 
 	global nzbget_maindir
-	nzbget_maindir = pytest.config.getini('nzbget_maindir')
+	nzbget_maindir = request.config.getini('nzbget_maindir')
 	global nzbget_configfile
 	nzbget_configfile = nzbget_maindir + '/nzbget.conf'
 
-pytest.check_config = check_config
-
 class NServ:
 
 	def __init__(self):
@@ -72,9 +63,7 @@ def finalize(self):
 
 @pytest.fixture(scope='session')
 
-def nserv(request):
-	check_config()
-
+def nserv(request, check_config):
 	instance = NServ()
 	request.addfinalizer(instance.finalize)
 	return instance
@@ -91,7 +80,7 @@ def __init__(self, options, session):
 		self.wait_until_started()
 
 	def finalize(self):
-		if pytest.config.getoption("--hold"):
+		if self.session.config.getoption("--hold"):
 			print('\nNZBGet is still running, press Ctrl+C to quit')
 			time.sleep(100000)
 		self.process.kill()
@@ -172,7 +161,7 @@ def wait_until_started(self):
 		print('Started')
 
 	def append_nzb(self, nzb_name, nzb_content, unpack = None, dupekey = '', dupescore = 0, dupemode = 'FORCE', params = None):
-		nzbcontent64 = base64.standard_b64encode(nzb_content)
+		nzbcontent64 = base64.standard_b64encode(nzb_content.encode()).decode()
 		if params is None:
 			params = []
 		if unpack == True:
@@ -210,13 +199,11 @@ def wait_nzb(self, nzb_name):
 		return hist
 
 	def clear(self):
-		self.api.editqueue('HistoryFinalDelete', 0, '', range(1, 1000));
+		self.api.editqueue('HistoryFinalDelete', 0, '', list(range(1, 1000)))
 
 @pytest.fixture(scope='module')
 
-def nzbget(request):
-	check_config()
-
+def nzbget(request, check_config):
 	instance = Nzbget(getattr(request.module, 'nzbget_options', []), request.session)
 	request.addfinalizer(instance.finalize)
 	return instance

tests/functional/download/conftest.py

@@ -10,24 +10,23 @@ def pytest_addoption(parser):
 
 
 @pytest.fixture(scope='session', autouse=True)
-def prepare_testdata(request):
+def prepare_testdata(request, check_config):
 	print('Preparing test data for "download"')
-	pytest.check_config()
 
-	nserv_datadir = pytest.config.getini('nserv_datadir')
-	nzbget_bin = pytest.config.getini('nzbget_bin')
-	sevenzip_bin = pytest.config.getini('sevenzip_bin')
+	nserv_datadir = request.config.getini('nserv_datadir')
+	nzbget_bin = request.config.getini('nzbget_bin')
+	sevenzip_bin = request.config.getini('sevenzip_bin')
 
 	if not os.path.exists(nserv_datadir):
 		print('Creating nserv datadir')
 		os.makedirs(nserv_datadir)
 
 	if not os.path.exists(nserv_datadir + '/medium.nzb'):
-		sizemb = int(pytest.config.getini('sample_medium'))
+		sizemb = int(request.config.getini('sample_medium'))
 		create_test_file(nserv_datadir + '/medium', sevenzip_bin, sizemb, 50)
 
 	if not os.path.exists(nserv_datadir + '/large.nzb'):
-		sizemb = int(pytest.config.getini('sample_large'))
+		sizemb = int(request.config.getini('sample_large'))
 		create_test_file(nserv_datadir + '/large', sevenzip_bin, sizemb, 50)
 
 	if not os.path.exists(nserv_datadir + '/medium.nzb') or not os.path.exists(nserv_datadir + '/large.nzb'):
@@ -56,7 +55,7 @@ def create_test_file(bigdir, sevenzip_bin, sizemb, partmb):
 		os.makedirs(bigdir)
 
 	f = open(bigdir + '/' + str(sizemb) + 'mb.dat', 'wb')
-	for n in xrange(sizemb // partmb):
+	for n in range(sizemb // partmb):
 		print('Writing block %i from %i' % (n + 1, sizemb // partmb))
 		f.write(os.urandom(partmb * 1024 * 1024))
 	f.close()

tests/functional/download/dupecheck_test.py

@@ -14,6 +14,6 @@ def test_dupecheck_small_id(nserv, nzbget):
 	hist = nzbget.download_nzb('small.nzb', dupemode = 'SCORE')
 	assert hist['Status'] == 'SUCCESS/HEALTH'
 	nzb_content = nzbget.load_nzb('small.nzb')
-	nzbcontent64 = base64.standard_b64encode(nzb_content)
+	nzbcontent64 = base64.standard_b64encode(nzb_content.encode()).decode()
 	id = nzbget.api.append('small.copy2.nzb', nzbcontent64, 'test', 0, False, False, '', 0, 'SCORE', [])
 	assert id > 0

tests/functional/download/unpack_test.py

@@ -4,13 +4,13 @@
 
 
 @pytest.fixture(scope='session', autouse=True)
-def prepare_testdata(request):
+def prepare_testdata(request, check_config):
 	print('Preparing test data for "unpack"')
 
-	nserv_datadir = pytest.config.getini('nserv_datadir')
-	nzbget_bin = pytest.config.getini('nzbget_bin')
-	sevenzip_bin = pytest.config.getini('sevenzip_bin')
-	par2_bin = pytest.config.getini('par2_bin')
+	nserv_datadir = request.config.getini('nserv_datadir')
+	nzbget_bin = request.config.getini('nzbget_bin')
+	sevenzip_bin = request.config.getini('sevenzip_bin')
+	par2_bin = request.config.getini('par2_bin')
 
 	if not os.path.exists(nserv_datadir):
 		print('Creating nserv datadir')
@@ -60,8 +60,8 @@ def create_test_file(bigdir, sevenzip_bin, sizemb, partmb):
 		os.makedirs(bigdir)
 
 	f = open(bigdir + '/' + str(sizemb) + 'mb.dat', 'wb')
-	for n in xrange(sizemb / partmb):
-		print('Writing block %i from %i' % (n + 1, sizemb / partmb))
+	for n in range(sizemb // partmb):
+		print('Writing block %i from %i' % (n + 1, sizemb // partmb))
 		f.write(os.urandom(partmb * 1024 * 1024))
 	f.close()
 

tests/functional/parcheck/conftest.py

@@ -5,11 +5,11 @@
 
 
 @pytest.fixture(scope='session', autouse=True)
-def prepare_testdata(request):
+def prepare_testdata(request, check_config):
 	print('Preparing test data for "parcheck"')
 
-	nserv_datadir = pytest.config.getini('nserv_datadir')
-	nzbget_bin = pytest.config.getini('nzbget_bin')
+	nserv_datadir = request.config.getini('nserv_datadir')
+	nzbget_bin = request.config.getini('nzbget_bin')
 
 	if not os.path.exists(nserv_datadir):
 		print('Creating nserv datadir')

tests/functional/rename/conftest.py

@@ -4,14 +4,13 @@
 import pytest
 
 @pytest.fixture(scope='session', autouse=True)
-def prepare_testdata(request):
+def prepare_testdata(request, check_config):
 	print('Preparing test data for "rename"')
-	pytest.check_config()
 
-	nserv_datadir = pytest.config.getini('nserv_datadir')
-	nzbget_bin = pytest.config.getini('nzbget_bin')
-	sevenzip_bin = pytest.config.getini('sevenzip_bin')
-	par2_bin = pytest.config.getini('par2_bin')
+	nserv_datadir = request.config.getini('nserv_datadir')
+	nzbget_bin = request.config.getini('nzbget_bin')
+	sevenzip_bin = request.config.getini('sevenzip_bin')
+	par2_bin = request.config.getini('par2_bin')
 
 	if not os.path.exists(nserv_datadir):
 		print('Creating nserv datadir')
@@ -190,8 +189,8 @@ def create_test_file(bigdir, sevenzip_bin, sizemb, partmb):
 		os.makedirs(bigdir)
 
 	f = open(bigdir + '/' + str(sizemb) + 'mb.dat', 'wb')
-	for n in xrange(sizemb / partmb):
-		print('Writing block %i from %i' % (n + 1, sizemb / partmb))
+	for n in range(sizemb // partmb):
+		print('Writing block %i from %i' % (n + 1, sizemb // partmb))
 		f.write(os.urandom(partmb * 1024 * 1024))
 	f.close()