python-lsp-compare/tests/test_cli.py at main · microsoft/python-lsp-compare · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
from __future__ import annotations

import io
import json
import sys
import tempfile
import unittest
from contextlib import redirect_stdout
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))

from python_lsp_compare.cli import main


class CliTests(unittest.TestCase):
    def test_list_scenarios(self) -> None:
        exit_code = main(["list-scenarios"])
        self.assertEqual(exit_code, 0)

    def test_run_writes_report(self) -> None:
        server_script = Path(__file__).parent / "fixtures" / "fake_lsp_server.py"
        with tempfile.TemporaryDirectory() as temp_dir:
            output_path = Path(temp_dir) / "report.json"
            exit_code = main(
                [
                    "run",
                    "--server-command",
                    sys.executable,
                    "--server-arg",
                    str(server_script),
                    "--scenario",
                    "hover",
                    "--output",
                    str(output_path),
                ]
            )
            self.assertEqual(exit_code, 0)
            report = json.loads(output_path.read_text(encoding="utf-8"))
            self.assertEqual(report["requested_scenarios"], ["hover"])
            self.assertEqual(report["scenario_reports"][0]["name"], "hover")
            self.assertTrue(report["scenario_reports"][0]["success"])

    def test_list_benchmarks(self) -> None:
        exit_code = main(["list-benchmarks", "--benchmark-root", str(Path(__file__).parent / "fixtures")])
        self.assertEqual(exit_code, 0)

    def test_list_benchmarks_can_filter_by_protocol(self) -> None:
        stdout = io.StringIO()
        with redirect_stdout(stdout):
            exit_code = main(["list-benchmarks", "--protocol", "tsp"])

        self.assertEqual(exit_code, 0)
        output = stdout.getvalue()
        self.assertIn("tsp_core:", output)
        self.assertNotIn("pandas:", output)

    def test_run_benchmark_can_filter_by_protocol(self) -> None:
        server_script = Path(__file__).parent / "fixtures" / "fake_lsp_server.py"
        with tempfile.TemporaryDirectory() as temp_dir:
            output_path = Path(temp_dir) / "tsp-only.json"
            exit_code = main(
                [
                    "run-benchmark",
                    "--server-command",
                    sys.executable,
                    "--server-arg",
                    str(server_script),
                    "--protocol",
                    "tsp",
                    "--output",
                    str(output_path),
                ]
            )
            self.assertEqual(exit_code, 0)
            report = json.loads(output_path.read_text(encoding="utf-8"))
            self.assertIn("tsp_core", report["requested_benchmarks"])
            self.assertNotIn("pandas", report["requested_benchmarks"])

    def test_list_servers_marks_configured_baseline_inline(self) -> None:
        server_script = Path(__file__).parent / "fixtures" / "fake_lsp_server.py"
        with tempfile.TemporaryDirectory() as temp_dir:
            config_path = Path(temp_dir) / "servers.json"
            config_path.write_text(
                json.dumps(
                    {
                        "baselineServer": "baseline-server",
                        "servers": [
                            {
                                "id": "baseline-server",
                                "displayName": "Baseline Server",
                                "launch": {
                                    "command": sys.executable,
                                    "args": [str(server_script)],
                                },
                            },
                            {
                                "id": "other-server",
                                "displayName": "Other Server",
                                "launch": {
                                    "command": sys.executable,
                                    "args": [str(server_script)],
                                },
                            },
                        ],
                    }
                ),
                encoding="utf-8",
            )

            stdout = io.StringIO()
            with redirect_stdout(stdout):
                exit_code = main(["list-servers", "--config", str(config_path)])

            self.assertEqual(exit_code, 0)
            output = stdout.getvalue()
            self.assertIn("baseline-server: Baseline Server (enabled, baseline)", output)
            self.assertIn("other-server: Other Server (enabled)", output)


if __name__ == "__main__":
    unittest.main()