eliza/packages/benchmarks/realm/__init__.py at develop · elizaOS/eliza · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
"""
REALM-Bench: Real-World Planning Benchmark for LLMs and Multi-Agent Systems.

Faithful implementation of the 11 canonical scenarios (P1..P11) from:

    Geng et al., arXiv:2502.18836
    https://github.com/genglongling/REALM-Bench

Upstream task definitions, instance datasets, and JSSP benchmarks are
vendored under ``packages/benchmarks/realm/upstream/`` (see
``upstream/ATTRIBUTION.md``).

The agent loop targets the eliza TS bridge via
``eliza_adapter.realm.ElizaREALMAgent``. A deterministic
``_MockREALMAgent`` is exposed for smoke tests and CI.
"""

from benchmarks.realm.dataset import REALMDataset
from benchmarks.realm.evaluator import MetricsCalculator, REALMEvaluator
from benchmarks.realm.runner import REALMRunner
from benchmarks.realm.types import (
    LEADERBOARD_NOTE,
    LEADERBOARD_SCORES,
    MULTI_AGENT_PROBLEMS,
    PROBLEM_DESCRIPTIONS,
    PROBLEM_TO_FAMILY,
    PROBLEMS_WITH_DISRUPTIONS,
    ExecutionModel,
    OracleFamily,
    PlanningAction,
    PlanningStep,
    PlanningTrajectory,
    PlanStatus,
    REALMCategory,  # back-compat alias for RealmProblem
    REALMConfig,
    REALMMetrics,
    REALMReport,
    REALMResult,
    REALMResultDetails,
    REALMResultMetrics,
    REALMTask,
    REALMTestCase,
    RealmProblem,
)

__all__ = [
    "RealmProblem",
    "REALMCategory",
    "OracleFamily",
    "PROBLEM_TO_FAMILY",
    "PROBLEMS_WITH_DISRUPTIONS",
    "MULTI_AGENT_PROBLEMS",
    "PROBLEM_DESCRIPTIONS",
    "REALMConfig",
    "REALMMetrics",
    "REALMReport",
    "REALMResult",
    "REALMResultMetrics",
    "REALMResultDetails",
    "REALMTask",
    "REALMTestCase",
    "PlanningAction",
    "PlanningStep",
    "PlanningTrajectory",
    "PlanStatus",
    "ExecutionModel",
    "LEADERBOARD_SCORES",
    "LEADERBOARD_NOTE",
    "REALMDataset",
    "REALMEvaluator",
    "MetricsCalculator",
    "REALMRunner",
]