Allow providing a path for generated artifacts

Author: s0undt3ch

.github/workflows/tests.yml

@@ -255,4 +255,13 @@ jobs:
         run: docker-compose build tests
 
       - name: Run Tests
-        run: docker-compose run --rm tests
+        run: |
+          mkdir artifacts
+          chmod 777 artifacts
+          docker-compose run --rm tests
+
+      - name: Upload Artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: testrun-artifacts
+          path: artifacts/

.gitignore

@@ -141,3 +141,4 @@ user_data/hyperopts
 user_data/logs
 user_data/notebooks
 user_data/plot
+artifacts/

docker-compose.yml

@@ -9,7 +9,7 @@ services:
     volumes:
       - "./:/testing"
     command: >
-      python -m pytest -ra -vv -s --log-cli-level=info tests/
+      python -m pytest -ra -vv -s --log-cli-level=info --artifacts-path=artifacts/ tests/
     entrypoint: []
     working_dir: /testing
   backtesting:

tests/backtests/binance/conftest.py

@@ -1,10 +1,8 @@
-import functools
-
 import pytest
 
-from tests.backtests.helpers import exchange_backtest
+from tests.backtests.helpers import Backtest
 
 
 @pytest.fixture
-def backtest(tmp_path):
-    return functools.partial(exchange_backtest, "binance", tmp_path)
+def backtest(request):
+    return Backtest(request, "binance")

tests/backtests/helpers.py

@@ -1,5 +1,6 @@
 import json
 import logging
+import shutil
 import subprocess
 from types import SimpleNamespace
 
@@ -52,53 +53,63 @@ def __str__(self):
         return message + "\n"
 
 
-def exchange_backtest(
-    exchange,
-    tmp_path,
-    start_date,
-    end_date,
-    pairlist=None,
-    max_open_trades=5,
-    stake_amount="unlimited",
-):
-    exchange_config = f"user_data/{exchange}-usdt-static.json"
-    json_results_file = tmp_path / "backtest-results.json"
-    cmdline = [
-        "freqtrade",
-        "backtesting",
-        f"--user-data=user_data",
-        "--strategy-list=NostalgiaForInfinityNext",
-        f"--timerange={start_date}-{end_date}",
-        f"--max-open-trades={max_open_trades}",
-        f"--stake-amount={stake_amount}",
-        "--config=user_data/pairlists.json",
-        f"--export-filename={json_results_file}",
-    ]
-    if pairlist is None:
-        cmdline.append(f"--config={exchange_config}")
-    else:
-        pairlist_config = {"exchange": {"name": exchange, "pair_whitelist": pairlist}}
-        pairlist_config_file = tmp_path / "test-pairlist.json"
-        pairlist_config_file.write(json.dumps(pairlist_config))
-        cmdline.append(f"--config={pairlist_config_file}")
-    log.info("Running cmdline '%s' on '%s'", " ".join(cmdline), REPO_ROOT)
-    proc = subprocess.run(
-        cmdline, check=False, shell=False, cwd=REPO_ROOT, text=True, capture_output=True
-    )
-    ret = ProcessResult(
-        exitcode=proc.returncode,
-        stdout=proc.stdout.strip(),
-        stderr=proc.stderr.strip(),
-        cmdline=cmdline,
-    )
-    log.info("Command Result:\n%s", ret)
-    assert ret.exitcode == 0
-    generated_results_file = list(tmp_path.rglob("backtest-results-*.json"))[0]
-    results_data = json.loads(generated_results_file.read_text())
-    data = {
-        "stdout": ret.stdout.strip(),
-        "stderr": ret.stderr.strip(),
-        "comparison": results_data["strategy_comparison"],
-        "results": results_data["strategy"]["NostalgiaForInfinityNext"],
-    }
-    return json.loads(json.dumps(data), object_hook=lambda d: SimpleNamespace(**d))
+class Backtest:
+    def __init__(self, request, exchange):
+        self.request = request
+        self.exchange = exchange
+
+    def __call__(
+        self, start_date, end_date, pairlist=None, max_open_trades=5, stake_amount="unlimited"
+    ):
+        tmp_path = self.request.getfixturevalue("tmp_path")
+        exchange_config = f"user_data/{self.exchange}-usdt-static.json"
+        json_results_file = tmp_path / "backtest-results.json"
+        cmdline = [
+            "freqtrade",
+            "backtesting",
+            f"--user-data=user_data",
+            "--strategy-list=NostalgiaForInfinityNext",
+            f"--timerange={start_date}-{end_date}",
+            f"--max-open-trades={max_open_trades}",
+            f"--stake-amount={stake_amount}",
+            "--config=user_data/pairlists.json",
+            f"--export-filename={json_results_file}",
+        ]
+        if pairlist is None:
+            cmdline.append(f"--config={exchange_config}")
+        else:
+            pairlist_config = {"exchange": {"name": self.exchange, "pair_whitelist": pairlist}}
+            pairlist_config_file = tmp_path / "test-pairlist.json"
+            pairlist_config_file.write(json.dumps(pairlist_config))
+            cmdline.append(f"--config={pairlist_config_file}")
+        log.info("Running cmdline '%s' on '%s'", " ".join(cmdline), REPO_ROOT)
+        proc = subprocess.run(
+            cmdline, check=False, shell=False, cwd=REPO_ROOT, text=True, capture_output=True
+        )
+        ret = ProcessResult(
+            exitcode=proc.returncode,
+            stdout=proc.stdout.strip(),
+            stderr=proc.stderr.strip(),
+            cmdline=cmdline,
+        )
+        log.info("Command Result:\n%s", ret)
+        assert ret.exitcode == 0
+        generated_results_file = list(tmp_path.rglob("backtest-results-*.json"))[0]
+        if self.request.config.option.artifacts_path:
+            generated_json_results_artifact_path = (
+                self.request.config.option.artifacts_path / generated_results_file.name
+            )
+            shutil.copyfile(generated_results_file, generated_json_results_artifact_path)
+            generated_txt_results_artifact_path = generated_json_results_artifact_path.with_suffix(
+                ".txt"
+            )
+            generated_txt_results_artifact_path.write_text(ret.stdout.strip())
+
+        results_data = json.loads(generated_results_file.read_text())
+        data = {
+            "stdout": ret.stdout.strip(),
+            "stderr": ret.stderr.strip(),
+            "comparison": results_data["strategy_comparison"],
+            "results": results_data["strategy"]["NostalgiaForInfinityNext"],
+        }
+        return json.loads(json.dumps(data), object_hook=lambda d: SimpleNamespace(**d))

tests/conftest.py

@@ -4,3 +4,16 @@
 # Make sure we can import the strategy module directly
 REPO_ROOT = Path(__file__).parent.parent
 sys.path.insert(0, str(REPO_ROOT))
+
+
+def pytest_addoption(parser):
+    parser.addoption(
+        "--artifacts-path", default=None, help="Path to write generated test artifacts"
+    )
+
+
+def pytest_configure(config):
+    if config.option.artifacts_path:
+        config.option.artifacts_path = Path(config.option.artifacts_path).resolve()
+        if not config.option.artifacts_path.is_dir():
+            config.option.artifacts_path.mkdir()