diff --git a/formatters/ai_scriptless.py b/formatters/ai_scriptless.py index 3054a80..5efe85d 100644 --- a/formatters/ai_scriptless.py +++ b/formatters/ai_scriptless.py @@ -7,11 +7,19 @@ CommandDefinitionSummary, ScriptFlowElement, ScriptParameter, + ScriptStepArgument, + ScriptStepDetail, + ScriptStepParameter, ScriptVariableSummary, SnapshotListResult, SnapshotSummary, TestStructure, ) +from tools.ai_scriptless.definitions import ( + parameter_label, + restriction_allowed_values, + restriction_range, +) from tools.ai_scriptless.elements import normalize_if_statement_aliases PRIMARY_AI_COMMAND_IDS = ( @@ -30,7 +38,7 @@ def command_selection_policy_info() -> List[str]: " • ai_user-action — user interactions (open browser/app, navigate to URL, tap, type, dismiss overlays); " "argument: action (natural language).", " • ai_validation — checkpoints and assertions; argument: validation (natural language).", - " • ai_visual-comparison — visual/baseline comparison; argument: name.", + " • ai_visual-comparison — visual/baseline comparison; argument: baselineId.", "Prefer ai_user-action for navigation (e.g. open browser and go to URL), not browser_goto / browser_open.", "Do not use browser_*, touch_tap, webpage.element_*, checkpoint_text, etc. unless the user explicitly " "requests a non-AI command or agreed that AI commands cannot meet a documented requirement.", @@ -42,8 +50,11 @@ def command_selection_policy_info() -> List[str]: "Keep command arguments nested inside cmd_arguments: the 'action' parameter of ai_user-action " "collides with the tool's own action key if flattened into args.", "Values are constants by default; pass {\"data_source\": \"VARIABLE\", \"value\": \"\"} " - "to bind an argument to a script variable.", + "to bind an argument to a script variable, or {\"data_source\": \"DATATABLE\", \"table_name\": \"\", " + "\"column\": \"\"} to bind it to a DataTable column.", "modify_command merges: only the arguments sent are replaced, the others keep their current value.", + "Values are validated against the declared type, range, allowed values and data sources; " + "view_test_step reports all four for every argument of an existing step.", ] def format_ai_scriptless_tests_filter_values(tests: dict[str, Any], params: Optional[dict] = None) -> dict[str, Any]: @@ -168,17 +179,17 @@ def _parameter_display_label(param: dict[str, Any]) -> str: def _format_argument_display_value(element: dict[str, Any], argument_name: str) -> Optional[str]: - for argument in element.get("arguments", []): - if argument.get("name") != argument_name: - continue - data = argument.get("data", {}) - value = data.get("value") - if value is None: - return None - if data.get("secured"): - return "" - return str(value) - return None + # A multivalued parameter has several arguments under one name; the UI shows the last. + matches = [a for a in element.get("arguments", []) if a.get("name") == argument_name] + if not matches: + return None + data = matches[-1].get("data", {}) + value = data.get("value") + if value is None: + return None + if data.get("secured"): + return "" + return str(value) def _command_step_display_name( @@ -245,19 +256,24 @@ def _step_display_name(element: dict[str, Any], definitions_map: dict[str, dict[ if element_type == "Loop": iterator = element.get("iterator", {}) + variable = iterator.get("variable") + if variable: + return f"Loop ({variable})" count = iterator.get("count") if count is not None: - return f"Loop ({count})" + # The API serializes the count as a float; 2.0 reads as a broken count. + return f"Loop ({_range_bound(count)})" return "Loop" if element_type == "IfStatement": - expression = element.get("expression") or element.get("label") - if expression: - return f"Condition ({expression})" + label = element.get("label") + if label: + return f"Condition ({label})" return "Condition" if element_type == "LogicalStep": - label = element.get("label") + # The UI stores the group title in `name`; `label` is only what older MCP writes used. + label = element.get("name") or element.get("label") if label: return label return "Step" @@ -357,6 +373,196 @@ def format_test_structure(payload: dict[str, Any], params: Optional[dict] = None ) +def _restriction_allowed_values( + param: dict[str, Any], + command_id: Optional[str] = None, +) -> List[str]: + return list(restriction_allowed_values(param, command_id)) + + +def _renamed_declared_label( + command_id: Optional[str], + name: Optional[str], + declared: Optional[str], +) -> Optional[str]: + """The declared label, reported only where the editor shows a different one.""" + shown = parameter_label(command_id, name, declared) + return declared if shown != declared else None + + +def _range_bound(value: Any) -> str: + # The API serializes bounds as floats (0.0, 3600.0); render integral ones as integers. + if isinstance(value, float) and value.is_integer(): + return str(int(value)) + return str(value) + + +def _restriction_range(param: dict[str, Any]) -> Optional[str]: + minimum, maximum = restriction_range(param) + if minimum is None and maximum is None: + return None + return f"{_range_bound(minimum)}..{_range_bound(maximum)}" + + +def _step_parameters_map(definition: Optional[dict[str, Any]]) -> dict[str, dict[str, Any]]: + if not definition: + return {} + mandatory_names = { + param.get("name") or param.get("parameterName") + for param in _definition_data(definition).get("mandatoryParameters") or [] + if isinstance(param, dict) + } + parameters: dict[str, dict[str, Any]] = {} + for param in _iter_definition_parameters(definition): + name = param.get("name") or param.get("parameterName") + parameters[name] = {**param, "_mandatory": name in mandatory_names} + return parameters + + +def _step_argument( + argument: dict[str, Any], + parameters: dict[str, dict[str, Any]], + command_id: Optional[str] = None, +) -> ScriptStepArgument: + name = argument.get("name", "") + data = argument.get("data") or {} + value = data.get("value") + if data.get("secured") and value: + value = "" + param = parameters.get(name) + display = (param or {}).get("display") or {} + return ScriptStepArgument( + name=name, + value=value, + data_source=data.get("dataSource"), + parameter_type=(param or {}).get("dataType"), + mandatory=(param or {}).get("_mandatory") if param else None, + declared=param is not None or not parameters, + allowed_data_sources=list((param or {}).get("dataSources") or []), + allowed_values=_restriction_allowed_values(param or {}, command_id), + value_range=_restriction_range(param or {}), + label=parameter_label(command_id, name, display.get("name")), + declared_label=_renamed_declared_label(command_id, name, display.get("name")), + table_name=data.get("tableName"), + column=data.get("column"), + ) + + +def _step_argument_is_set(argument: ScriptStepArgument) -> bool: + """A DataTable binding is a value even though it carries no value field.""" + if argument.data_source == "DATATABLE": + return bool(argument.table_name or argument.column) + return argument.value is not None and str(argument.value).strip() != "" + + +def _unset_step_parameters( + element: dict[str, Any], + parameters: dict[str, dict[str, Any]], + command_id: Optional[str] = None, +) -> List[ScriptStepParameter]: + set_names = {argument.get("name") for argument in element.get("arguments", [])} + unset: List[ScriptStepParameter] = [] + for name, param in parameters.items(): + if name in set_names: + continue + display = param.get("display") or {} + mandatory = bool(param.get("_mandatory")) + unset.append(ScriptStepParameter( + name=name, + parameter_type=param.get("dataType"), + mandatory=mandatory, + default_value=param.get("defaultValue"), + allowed_data_sources=list(param.get("dataSources") or []), + allowed_values=_restriction_allowed_values(param, command_id), + value_range=_restriction_range(param), + label=parameter_label(command_id, name, display.get("name")), + declared_label=_renamed_declared_label(command_id, name, display.get("name")), + # Some commands declare ~60 optional parameters with long help texts; carrying them + # all would dwarf the step itself. Names, types and accepted values are enough to + # edit, and get_command_definitions has the full help when it is actually needed. + help_text=(param.get("helpText") or display.get("helpText")) if mandatory else None, + )) + unset.sort(key=lambda parameter: (not parameter.mandatory, parameter.name)) + return unset + + +def _step_children_paths(element: dict[str, Any], step_path: str) -> List[str]: + if element.get("@type") == "IfStatement": + return [ + f"{step_path}.b{branch_index}" + for branch_index, _branch in enumerate(element.get("branches", [])) + ] + return [ + f"{step_path}.{child_index}" + for child_index, _child in enumerate(element.get("flowElements", [])) + ] + + +def _step_detail_notes(element: dict[str, Any], detail_arguments: List[ScriptStepArgument]) -> List[str]: + notes: List[str] = [] + undeclared = [argument.name for argument in detail_arguments if not argument.declared] + if undeclared: + notes.append( + f"Argument(s) not declared by the command: {', '.join(undeclared)}. " + "Perfecto ignores them at execution time; they were most likely persisted by mistake." + ) + if detail_arguments: + notes.append( + "To edit, call modify_command with cmd_arguments keyed by these argument names. " + "Only the arguments you send change; the others keep their current value." + ) + if element.get("active") is False: + notes.append("This step is excluded from the run; re-include it with set_command_enabled.") + empty_mandatory = [ + argument.name for argument in detail_arguments + if argument.mandatory and not _step_argument_is_set(argument) + ] + if empty_mandatory: + notes.append( + f"Mandatory argument(s) with no value: {', '.join(empty_mandatory)}. " + "The step will not do anything until they are set with modify_command." + ) + return notes + + +def format_step_detail( + element: dict[str, Any], + item_key: str, + step_path: str, + command_definitions: Optional[list] = None, + statement_step_path: Optional[str] = None, +) -> ScriptStepDetail: + """Full configuration of one step, joined with what its command declares.""" + definitions_map = _definitions_map(command_definitions) + command_id = _command_id(element.get("command"), element.get("subcommand")) + parameters = _step_parameters_map(definitions_map.get(command_id) if command_id else None) + arguments = [ + _step_argument(argument, parameters, command_id) + for argument in element.get("arguments", []) + ] + iterator = element.get("iterator") or {} + return ScriptStepDetail( + item_key=item_key, + step_path=step_path, + type=element.get("@type", ""), + name=_step_display_name(element, definitions_map), + command_id=command_id, + command=element.get("command"), + subcommand=element.get("subcommand"), + active=element.get("active", True), + error_policy=element.get("errorPolicy"), + comment=element.get("comment"), + arguments=arguments, + unset_parameters=_unset_step_parameters(element, parameters, command_id), + label=element.get("name") or element.get("label"), + statement_step_path=statement_step_path, + loop_count=iterator.get("count"), + loop_variable=iterator.get("variable"), + children=_step_children_paths(element, step_path), + notes=_step_detail_notes(element, arguments), + ) + + def _flatten_command_catalog(node: dict[str, Any], category: Optional[str] = None) -> List[CommandCatalogEntry]: entries: List[CommandCatalogEntry] = [] node_name = node.get("name") diff --git a/models/ai_scriptless.py b/models/ai_scriptless.py index 97c4820..52f6ae8 100644 --- a/models/ai_scriptless.py +++ b/models/ai_scriptless.py @@ -21,12 +21,112 @@ class ScriptParameter(BaseModel): type: str = Field(description="Parameter data type") +class ScriptStepArgument(BaseModel): + name: str = Field(description="Argument name; use it as the cmd_arguments key on modify_command") + value: Optional[Any] = Field(description="Current value ('' when secured)", default=None) + data_source: Optional[str] = Field( + description="Where the value comes from: CONSTANT, VARIABLE (a script variable) or DATATABLE", + default=None, + ) + parameter_type: Optional[str] = Field( + description="Declared data type (STRING, INTEGER, BOOLEAN, HANDSET, ...)", default=None + ) + mandatory: Optional[bool] = Field(description="Whether the command declares it mandatory", default=None) + declared: bool = Field( + description="False when the command definition does not declare this name (Perfecto ignores it)", + default=True, + ) + allowed_data_sources: List[str] = Field( + description="Data sources the parameter accepts", default_factory=list + ) + allowed_values: List[str] = Field( + description="Accepted values when the parameter is an enumeration or combo", default_factory=list + ) + value_range: Optional[str] = Field(description="Accepted numeric range as 'min..max'", default=None) + label: Optional[str] = Field(description="Display label used by the UI", default=None) + declared_label: Optional[str] = Field( + description="Label the command definition declares, when the UI renames it", default=None + ) + table_name: Optional[str] = Field( + description="DataTable the argument reads from (data_source DATATABLE)", default=None + ) + column: Optional[str] = Field( + description="DataTable column the argument reads from (data_source DATATABLE)", default=None + ) + + +class ScriptStepParameter(BaseModel): + """A parameter the command declares that the step does not currently set.""" + + name: str = Field(description="Parameter name; use it as the cmd_arguments key on modify_command") + parameter_type: Optional[str] = Field(description="Declared data type", default=None) + mandatory: bool = Field(description="Whether the command declares it mandatory", default=False) + default_value: Optional[Any] = Field(description="Value Perfecto applies when unset", default=None) + allowed_data_sources: List[str] = Field( + description="Data sources the parameter accepts", default_factory=list + ) + allowed_values: List[str] = Field( + description="Accepted values when the parameter is an enumeration or combo", default_factory=list + ) + value_range: Optional[str] = Field(description="Accepted numeric range as 'min..max'", default=None) + label: Optional[str] = Field(description="Display label used by the UI", default=None) + declared_label: Optional[str] = Field( + description="Label the command definition declares, when the UI renames it", default=None + ) + help_text: Optional[str] = Field(description="Parameter help text", default=None) + + +class ScriptStepDetail(BaseModel): + item_key: str = Field(description="Script itemKey the step belongs to") + step_path: str = Field(description="Dot-separated positional path of the step") + type: str = Field(description="Perfecto @type (Action, Validation, LogicalStep, Loop, IfStatement, Branch)") + name: str = Field(description="Display name, same as in view_test_structure") + command_id: Optional[str] = Field(description="Command ID for get_command_definitions", default=None) + command: Optional[str] = Field(description="Command namespace", default=None) + subcommand: Optional[str] = Field(description="Command subcommand", default=None) + active: bool = Field(description="False when the step is excluded from the run", default=True) + error_policy: Optional[str] = Field( + description="ABORT aborts the test on failure, IGNORE only reports it", default=None + ) + comment: Optional[str] = Field(description="Step comment", default=None) + arguments: List[ScriptStepArgument] = Field( + description="Arguments currently persisted on the step", default_factory=list + ) + unset_parameters: List[ScriptStepParameter] = Field( + description="Declared parameters the step does not set yet", default_factory=list + ) + label: Optional[str] = Field(description="Container label (LogicalStep or IfStatement)", default=None) + statement_step_path: Optional[str] = Field( + description=( + "For an IfStatement: the step whose result decides the branch (the UI's 'Statement'). " + "It is the preceding sibling carrying errorPolicy CATCH; None means the condition has " + "nothing to evaluate yet." + ), + default=None, + ) + loop_count: Optional[int] = Field(description="Iterations of a Loop that repeats a fixed number of times", default=None) + loop_variable: Optional[str] = Field( + description="Number variable that decides the iterations of a Loop, when it is variable-driven", + default=None, + ) + children: List[str] = Field( + description="Step paths of direct children, for containers", default_factory=list + ) + notes: List[str] = Field(description="Editing notes for this step", default_factory=list) + + class ScriptVariableSummary(BaseModel): name: str = Field(description="Variable name") type: str = Field(description="Variable type (string, number, boolean, secured_string, etc.)") value: Optional[Any] = Field(description="Variable value when readable", default=None) secured: bool = Field(description="Whether the value is secured", default=False) - set_at_runtime: bool = Field(description="True when value is provided at execution time", default=False) + set_at_runtime: bool = Field( + description=( + "True for a runtime variable: the stored value is a default, supplied when the run starts and " + "free to change during execution. False means the value is constant for the whole run." + ), + default=False, + ) class TestStructure(BaseModel): diff --git a/tests/conftest.py b/tests/conftest.py index 7a3e4f0..286df26 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -28,35 +28,58 @@ def perfecto_token() -> PerfectoToken: @pytest.fixture(autouse=True) def offline_command_definitions(monkeypatch): - """Keep cmd_arguments validation offline. + """Keep command contracts offline. - Validation resolves declared parameters over HTTP and memoizes them, so the - request is stubbed out (no declared parameters = validation fails open) and the - cache is reset on both ends. Use declare_command_parameters to opt into validation. + Contracts are resolved over HTTP and memoized, so the request is stubbed out + (no contract = element type, error policy and validation fall back to the local + spec) and the cache is reset on both ends. Use declare_commands to opt in. """ - definitions.reset_declared_parameters_cache() + definitions.reset_command_contract_cache() async def offline_api_request(*_args, **_kwargs): return BaseResult(error="command definitions are not fetched in tests") monkeypatch.setattr(definitions, "api_request", offline_api_request) yield - definitions.reset_declared_parameters_cache() + definitions.reset_command_contract_cache() @pytest.fixture -def declare_command_parameters(monkeypatch, offline_command_definitions): - """Declare parameters per command_id: {command_id: (mandatory, optional)}.""" +def declare_commands(monkeypatch, offline_command_definitions): + """Declare contracts per command_id. - def declare(declarations: dict[str, tuple[list[str], list[str]]]) -> None: + {command_id: {"mandatory": [...], "optional": [...], "element_type": ..., "error_policy": ...}} + """ + + def declare(declarations: dict[str, dict]) -> None: async def fake_fetch(_token, command_id): declaration = declarations.get(command_id) if declaration is None: return None - mandatory, optional = declaration - return frozenset(mandatory), frozenset(optional) - - definitions.reset_declared_parameters_cache() - monkeypatch.setattr(definitions, "_fetch_declared_parameters", fake_fetch) + mandatory = frozenset(declaration.get("mandatory", ())) + parameters = { + name: definitions.ParameterContract( + name=name, + data_type=spec.get("data_type"), + data_sources=frozenset(spec.get("data_sources", ("CONSTANT", "VARIABLE"))), + mandatory=name in mandatory, + default_value=spec.get("default_value"), + allowed_values=tuple(spec.get("allowed_values", ())), + minimum=spec.get("minimum"), + maximum=spec.get("maximum"), + ) + for name, spec in (declaration.get("parameters") or {}).items() + } + return definitions.CommandContract( + command_id=command_id, + mandatory=mandatory, + optional=frozenset(declaration.get("optional", ())), + element_type=declaration.get("element_type"), + error_policy=declaration.get("error_policy"), + parameters=parameters, + ) + + definitions.reset_command_contract_cache() + monkeypatch.setattr(definitions, "_fetch_command_contract", fake_fetch) return declare diff --git a/tests/test_ai_scriptless_definitions.py b/tests/test_ai_scriptless_definitions.py index 4268bd5..aa6e40e 100644 --- a/tests/test_ai_scriptless_definitions.py +++ b/tests/test_ai_scriptless_definitions.py @@ -17,16 +17,37 @@ import asyncio import httpx +import pytest from models.result import BaseResult from tools.ai_scriptless import definitions +from tools.ai_scriptless.commands import get_command_spec +from tools.ai_scriptless.elements import new_empty_script +from tools.ai_scriptless.variables import add_script_variable from tools.ai_scriptless.definitions import ( - declared_parameters, + ParameterContract, + coerce_argument_value, + restriction_allowed_values, + restriction_range, + validate_argument_values, + validate_variable_bindings, + CommandContract, + command_contract, empty_mandatory_note, validate_argument_names, ) -USER_ACTION = (frozenset({"action"}), frozenset({"handsetId"})) +USER_ACTION = CommandContract( + command_id="ai_user-action", + mandatory=frozenset({"handsetId", "action"}), + optional=frozenset({"reasoning"}), + element_type="Action", + error_policy="ABORT", +) + + +def _definition(command_id: str, data: dict) -> dict: + return {"definitions": [{"commandId": command_id, "data": data}]} class TestValidateArgumentNames: @@ -38,8 +59,8 @@ def test_accepts_declared_names(self): def test_rejects_undeclared_name_with_suggestion(self): error = validate_argument_names("ai_user-action", {"actions": "Tap"}, USER_ACTION) assert "'actions' (did you mean 'action'?)" in error - assert "Declared parameter names: action, handsetId" in error - assert "mandatory: action" in error + assert "Declared parameter names: action, handsetId, reasoning" in error + assert "mandatory: action, handsetId" in error def test_reports_undeclared_name_without_close_match(self): error = validate_argument_names("ai_user-action", {"xyz": "Tap"}, USER_ACTION) @@ -48,12 +69,18 @@ def test_reports_undeclared_name_without_close_match(self): def test_accepts_alias_in_either_direction(self): # The spec canonicalizes waitDuration to duration; either name may be declared. - assert validate_argument_names("wait", {"duration": "3"}, (frozenset({"waitDuration"}), frozenset())) is None - assert validate_argument_names("wait", {"waitDuration": "3"}, (frozenset({"duration"}), frozenset())) is None + declared_alias = CommandContract("wait", mandatory=frozenset({"waitDuration"})) + declared_canonical = CommandContract("wait", mandatory=frozenset({"duration"})) + assert validate_argument_names("wait", {"duration": "3"}, declared_alias) is None + assert validate_argument_names("wait", {"waitDuration": "3"}, declared_canonical) is None - def test_fails_open_without_declared_parameters(self): + def test_fails_open_without_contract(self): assert validate_argument_names("ai_user-action", {"anything": "value"}, None) is None + def test_fails_open_when_contract_declares_no_parameter(self): + contract = CommandContract("comment", element_type="Action") + assert validate_argument_names("comment", {"anything": "value"}, contract) is None + def test_accepts_variable_data_source_form(self): assert validate_argument_names( "ai_user-action", @@ -66,6 +93,18 @@ class TestEmptyMandatoryNote: def test_notes_mandatory_left_empty_by_spec_default(self): note = empty_mandatory_note("ai_user-action", None, USER_ACTION) assert "Mandatory parameter(s) left empty on 'ai_user-action': action" in note + # handsetId is bound to the DUT variable by the spec, so it is not empty. + assert "handsetId" not in note + + def test_notes_mandatory_the_spec_does_not_seed(self): + # checkpoint_text declares content as mandatory; the spec only injects handsetId. + contract = CommandContract( + "checkpoint_text", + mandatory=frozenset({"handsetId", "content"}), + element_type="Validation", + ) + note = empty_mandatory_note("checkpoint_text", None, contract) + assert "content" in note def test_no_note_when_mandatory_is_provided(self): assert empty_mandatory_note("ai_user-action", {"action": "Tap"}, USER_ACTION) is None @@ -81,59 +120,115 @@ def test_variable_binding_counts_as_provided(self): USER_ACTION, ) is None - def test_no_note_without_declared_parameters(self): + def test_no_note_without_contract(self): assert empty_mandatory_note("ai_user-action", None, None) is None -class TestDeclaredParameters: - def test_parses_and_memoizes_definitions(self, perfecto_token, monkeypatch): - calls: list = [] - +class TestCommandContractParsing: + @staticmethod + def _stub_api(monkeypatch, payload: dict, calls: list | None = None): async def fake_api_request(_token, _method, endpoint=None, result_formatter=None, **kwargs): - calls.append(kwargs.get("json")) - return BaseResult(result=result_formatter({ - "definitions": [{ - "commandId": "ai_validation", - "data": { - "display": {"name": "AI Validation"}, - "mandatoryParameters": [{"name": "validation"}], - "optionalParameters": [{"name": "handsetId"}], - }, - }], - }, None)) - - definitions.reset_declared_parameters_cache() + if calls is not None: + calls.append(kwargs.get("json")) + return BaseResult(result=result_formatter(payload, None)) + + definitions.reset_command_contract_cache() monkeypatch.setattr(definitions, "api_request", fake_api_request) - first = asyncio.run(declared_parameters(perfecto_token, "ai_validation")) - second = asyncio.run(declared_parameters(perfecto_token, "ai_validation")) + def test_parses_parameters_type_and_error_policy(self, perfecto_token, monkeypatch): + self._stub_api(monkeypatch, _definition("ai_validation", { + "display": {"name": "AI Validation"}, + "type": "VALIDATION", + "errorPolicy": "IGNORE", + "mandatoryParameters": [{"name": "handsetId"}, {"name": "validation"}], + "optionalParameters": [{"name": "reasoning"}], + })) + + contract = asyncio.run(command_contract(perfecto_token, "ai_validation")) + + assert contract.element_type == "Validation" + assert contract.error_policy == "IGNORE" + assert contract.mandatory == frozenset({"handsetId", "validation"}) + assert contract.optional == frozenset({"reasoning"}) + assert contract.declared_names == frozenset({"handsetId", "validation", "reasoning"}) + + def test_maps_action_type(self, perfecto_token, monkeypatch): + self._stub_api(monkeypatch, _definition("ai_user-action", { + "type": "ACTION", + "errorPolicy": "ABORT", + "mandatoryParameters": [{"name": "action"}], + })) + + contract = asyncio.run(command_contract(perfecto_token, "ai_user-action")) + + assert contract.element_type == "Action" + assert contract.error_policy == "ABORT" + + def test_null_error_policy_is_left_undeclared(self, perfecto_token, monkeypatch): + # wait declares errorPolicy null: the local default has to fill in. + self._stub_api(monkeypatch, _definition("wait", { + "type": "ACTION", + "errorPolicy": None, + "mandatoryParameters": [{"name": "duration"}], + })) + + contract = asyncio.run(command_contract(perfecto_token, "wait")) + + assert contract.element_type == "Action" + assert contract.error_policy is None + + def test_unknown_type_and_policy_are_left_undeclared(self, perfecto_token, monkeypatch): + self._stub_api(monkeypatch, _definition("odd", { + "type": "SOMETHING_NEW", + "errorPolicy": "RETRY", + "mandatoryParameters": [{"name": "x"}], + })) + + contract = asyncio.run(command_contract(perfecto_token, "odd")) + + assert contract.element_type is None + assert contract.error_policy is None + + def test_contract_without_parameters_still_carries_type(self, perfecto_token, monkeypatch): + self._stub_api(monkeypatch, _definition("wait", { + "type": "ACTION", + "mandatoryParameters": [], + "optionalParameters": [], + })) + + contract = asyncio.run(command_contract(perfecto_token, "wait")) + + assert contract.declared_names == frozenset() + assert contract.element_type == "Action" + + def test_memoizes_per_command(self, perfecto_token, monkeypatch): + calls: list = [] + self._stub_api(monkeypatch, _definition("wait", { + "type": "ACTION", + "mandatoryParameters": [{"name": "duration"}], + }), calls) - assert first == (frozenset({"validation"}), frozenset({"handsetId"})) - assert second == first - assert calls == [{"commandIds": ["ai_validation"]}] + first = asyncio.run(command_contract(perfecto_token, "wait")) + second = asyncio.run(command_contract(perfecto_token, "wait")) - def test_definition_without_parameters_is_treated_as_unknown(self, perfecto_token, monkeypatch): - async def fake_api_request(_token, _method, endpoint=None, result_formatter=None, **kwargs): - return BaseResult(result=result_formatter({ - "definitions": [{ - "commandId": "wait", - "data": {"display": {"name": "Wait"}, "mandatoryParameters": [], "optionalParameters": []}, - }], - }, None)) - - definitions.reset_declared_parameters_cache() - monkeypatch.setattr(definitions, "api_request", fake_api_request) + assert first == second + assert calls == [{"commandIds": ["wait"]}] + + def test_ignores_definitions_for_other_commands(self, perfecto_token, monkeypatch): + self._stub_api(monkeypatch, _definition("other", {"type": "ACTION"})) + + assert asyncio.run(command_contract(perfecto_token, "wait")) is None - assert asyncio.run(declared_parameters(perfecto_token, "wait")) is None +class TestCommandContractFailsOpen: def test_fails_open_on_api_error(self, perfecto_token, monkeypatch): async def fake_api_request(*_args, **_kwargs): return BaseResult(error="Invalid credentials") - definitions.reset_declared_parameters_cache() + definitions.reset_command_contract_cache() monkeypatch.setattr(definitions, "api_request", fake_api_request) - assert asyncio.run(declared_parameters(perfecto_token, "ai_validation")) is None + assert asyncio.run(command_contract(perfecto_token, "ai_validation")) is None def test_fails_open_on_http_exception(self, perfecto_token, monkeypatch): async def fake_api_request(*_args, **_kwargs): @@ -142,10 +237,489 @@ async def fake_api_request(*_args, **_kwargs): "not found", request=request, response=httpx.Response(404, request=request) ) - definitions.reset_declared_parameters_cache() + definitions.reset_command_contract_cache() monkeypatch.setattr(definitions, "api_request", fake_api_request) - assert asyncio.run(declared_parameters(perfecto_token, "ai_validation")) is None + assert asyncio.run(command_contract(perfecto_token, "ai_validation")) is None def test_no_token_returns_none(self): - assert asyncio.run(declared_parameters(None, "ai_validation")) is None + assert asyncio.run(command_contract(None, "ai_validation")) is None + + def test_no_command_id_returns_none(self, perfecto_token): + assert asyncio.run(command_contract(perfecto_token, "")) is None + + +class TestDataTableBindingCountsAsValue: + def test_bound_datatable_column_is_not_empty(self): + contract = CommandContract( + "checkpoint_text", + mandatory=frozenset({"content"}), + element_type="Validation", + ) + assert empty_mandatory_note( + "checkpoint_text", + {"content": {"data_source": "DATATABLE", "table_name": "T", "column": "c"}}, + contract, + ) is None + + def test_unbound_datatable_is_empty(self): + contract = CommandContract( + "checkpoint_text", + mandatory=frozenset({"content"}), + element_type="Validation", + ) + note = empty_mandatory_note( + "checkpoint_text", {"content": {"data_source": "DATATABLE"}}, contract + ) + assert "content" in note + + +WAIT_DURATION = ParameterContract( + name="duration", data_type="INTEGER", data_sources=frozenset({"CONSTANT", "VARIABLE", "DATATABLE"}), + mandatory=True, minimum=0.0, maximum=3600.0, +) +CHECKPOINT_CONTEXT = ParameterContract( + name="context", data_type="STRING", data_sources=frozenset({"CONSTANT", "VARIABLE", "DATATABLE"}), + default_value="all", allowed_values=("all", "body", "lowerPanel", "upperPanel"), +) +AI_ACTION = ParameterContract( + name="action", data_type="STRING", data_sources=frozenset({"CONSTANT", "VARIABLE"}), mandatory=True, +) +AI_REASONING = ParameterContract( + name="reasoning", data_type="BOOLEAN", data_sources=frozenset({"CONSTANT", "VARIABLE"}), + default_value=False, +) + +WAIT = CommandContract( + "wait", mandatory=frozenset({"duration"}), element_type="Action", + parameters={"duration": WAIT_DURATION}, +) +USER_ACTION_TYPED = CommandContract( + "ai_user-action", + mandatory=frozenset({"action"}), + optional=frozenset({"reasoning"}), + element_type="Action", + parameters={"action": AI_ACTION, "reasoning": AI_REASONING}, +) +CHECKPOINT = CommandContract( + "checkpoint_text", optional=frozenset({"context"}), element_type="Validation", + parameters={"context": CHECKPOINT_CONTEXT}, +) + + +class TestRestrictionParsing: + def test_parses_enumeration_values(self): + param = {"restriction": {"type": "ENUMERATION", "value": "primary,native,camera"}} + assert restriction_allowed_values(param) == ("primary", "native", "camera") + + def test_prefers_value_over_label(self): + # The label can be truncated ("all,body,,"); value carries the real list. + param = {"restriction": {"type": "COMBO", "value": "all,body,link", "label": "all,body,,"}} + assert restriction_allowed_values(param) == ("all", "body", "link") + + def test_ignores_non_enumeration_restrictions(self): + assert restriction_allowed_values({"restriction": {"type": "RANGE"}}) == () + + +FAIL_CRITERIA_PARAM = { + "name": "failCriteria", + "dataType": "STRING", + "maxOccurrences": 10, + "dataSources": ["CONSTANT"], + "restriction": { + "type": "ENUMERATION", + "source": "LITERAL", + "value": "style,missing,addition,error,device,value,pixel_difference,uncategorized", + "label": "STYLE,MISSING,ADDITION,ERROR,Device,VALUE,Pixel difference,Uncategorized", + }, +} + + +class TestRenamedParameterLabel: + """The editor labels ai_user-action's `action` "Prompt" (D5).""" + + def test_reports_the_label_the_editor_shows(self): + assert definitions.parameter_label("ai_user-action", "action", "Action") == "Prompt" + + def test_other_parameters_keep_the_declared_label(self): + assert definitions.parameter_label("ai_validation", "validation", "Validation") == "Validation" + assert definitions.parameter_label("ai_user-action", "handsetId", "Device ID") == "Device ID" + assert definitions.parameter_label(None, None, None) is None + + def test_the_editor_label_resolves_to_the_parameter(self): + # Typing into "Prompt" stores `action`, so the name read on screen is accepted as a key. + spec = get_command_spec("ai_user-action") + assert spec.normalize_argument_names({"Prompt": "Open Settings"}) == {"action": "Open Settings"} + assert spec.normalize_argument_names({"prompt": "Open Settings"}) == {"action": "Open Settings"} + + def test_the_alias_passes_name_validation(self): + contract = CommandContract("ai_user-action", mandatory=frozenset({"action"})) + assert validate_argument_names("ai_user-action", {"Prompt": "x"}, contract) is None + + +class TestUIPersistedEnumeration: + """failCriteria's stored values drifted from the declaration in two places (D12).""" + + def test_reports_the_spelling_the_ui_stores(self): + assert restriction_allowed_values(FAIL_CRITERIA_PARAM, "ai_visual-comparison") == ( + "device", "style", "value", "missing", "moved", + "addition", "error", "uncategorized", "pixelDifference", + ) + + def test_declared_values_stand_for_other_commands(self): + # The override is keyed by command and parameter: nothing else changes. + assert "pixel_difference" in restriction_allowed_values(FAIL_CRITERIA_PARAM, "checkpoint_text") + assert "pixel_difference" in restriction_allowed_values(FAIL_CRITERIA_PARAM) + + def test_only_two_options_differ_from_the_declaration(self): + # Why this is a constant and not a transformation: pixel_difference is stored + # camelCased, and moved is offered by the editor but declared nowhere. + declared = set(restriction_allowed_values(FAIL_CRITERIA_PARAM)) + stored = set(restriction_allowed_values(FAIL_CRITERIA_PARAM, "ai_visual-comparison")) + assert stored - declared == {"pixelDifference", "moved"} + assert declared - stored == {"pixel_difference"} + + @pytest.mark.parametrize("supplied, persisted", [ + ("pixel_difference", "pixelDifference"), # the declared value + ("Pixel difference", "pixelDifference"), # the declared label + ("pixelDifference", "pixelDifference"), # what the editor itself stores + ("PIXEL_DIFFERENCE", "pixelDifference"), + ("style", "style"), + ("STYLE", "style"), # the declared label's casing + ("Moved", "moved"), # UI-only, absent from the declaration + ]) + def test_every_spelling_coerces_to_what_the_ui_stores(self, supplied, persisted): + parameter = definitions._parse_parameter_contract( + FAIL_CRITERIA_PARAM, mandatory=True, command_id="ai_visual-comparison", + ) + assert coerce_argument_value(supplied, parameter) == persisted + assert validate_argument_values( + "ai_visual-comparison", + {"failCriteria": supplied}, + CommandContract("ai_visual-comparison", parameters={"failCriteria": parameter}), + ) is None + + def test_still_rejects_a_value_in_neither_vocabulary(self): + parameter = definitions._parse_parameter_contract( + FAIL_CRITERIA_PARAM, mandatory=True, command_id="ai_visual-comparison", + ) + error = validate_argument_values( + "ai_visual-comparison", + {"failCriteria": "pixel_diff"}, + CommandContract("ai_visual-comparison", parameters={"failCriteria": parameter}), + ) + assert error is not None and "pixel_diff" in error + + def test_relaxed_match_does_not_merge_declared_values(self): + # report declares all,all-on-error: dropping punctuation must not make them collide. + param = {"name": "report", "restriction": { + "type": "ENUMERATION", "value": "all,all-on-error,screenshot", + }} + parameter = definitions._parse_parameter_contract(param, mandatory=False) + assert coerce_argument_value("all", parameter) == "all" + assert coerce_argument_value("all on error", parameter) == "all-on-error" + assert coerce_argument_value("allonerror", parameter) == "all-on-error" + + def test_parses_range_bounds(self): + param = {"restriction": {"type": "RANGE", "range": {"minValue": 0.0, "maxValue": 3600.0}}} + assert restriction_range(param) == (0.0, 3600.0) + + def test_ignores_range_of_non_range_restriction(self): + param = {"restriction": {"type": "NONE", "range": {"minValue": -2147483648, "maxValue": 2147483647}}} + assert restriction_range(param) == (None, None) + + +class TestCoerceArgumentValue: + def test_stringifies_integer(self): + # UI-authored scripts persist INTEGER constants as strings ("2", not 2). + assert coerce_argument_value(2, WAIT_DURATION) == "2" + assert coerce_argument_value(2.0, WAIT_DURATION) == "2" + + def test_keeps_string_integer_as_is(self): + assert coerce_argument_value("2", WAIT_DURATION) == "2" + + def test_stringifies_boolean(self): + assert coerce_argument_value(True, AI_REASONING) == "true" + assert coerce_argument_value(False, AI_REASONING) == "false" + + def test_snaps_enumeration_to_declared_casing(self): + assert coerce_argument_value("BODY", CHECKPOINT_CONTEXT) == "body" + + def test_leaves_unknown_enumeration_value_untouched(self): + # Validation reports it; coercion does not silently rewrite it. + assert coerce_argument_value("sidebar", CHECKPOINT_CONTEXT) == "sidebar" + + def test_no_op_without_parameter(self): + assert coerce_argument_value(2, None) == 2 + + +class TestValidateArgumentValues: + def test_accepts_valid_values(self): + assert validate_argument_values("wait", {"duration": "30"}, WAIT) is None + assert validate_argument_values("checkpoint_text", {"context": "body"}, CHECKPOINT) is None + + def test_rejects_non_numeric_integer(self): + error = validate_argument_values("wait", {"duration": "soon"}, WAIT) + assert "'duration' expects a number (INTEGER)" in error + + def test_rejects_value_above_range(self): + error = validate_argument_values("wait", {"duration": 5000}, WAIT) + assert "'duration' must be within 0..3600" in error + + def test_rejects_value_below_range(self): + error = validate_argument_values("wait", {"duration": -1}, WAIT) + assert "must be within 0..3600" in error + + def test_accepts_range_bounds(self): + assert validate_argument_values("wait", {"duration": 0}, WAIT) is None + assert validate_argument_values("wait", {"duration": 3600}, WAIT) is None + + def test_rejects_undeclared_enumeration_value(self): + error = validate_argument_values("checkpoint_text", {"context": "sidebar"}, CHECKPOINT) + assert "must be one of all, body, lowerPanel, upperPanel" in error + + def test_accepts_enumeration_value_in_any_casing(self): + assert validate_argument_values("checkpoint_text", {"context": "BODY"}, CHECKPOINT) is None + + def test_rejects_non_boolean(self): + error = validate_argument_values("ai_user-action", {"reasoning": "maybe"}, USER_ACTION_TYPED) + assert "'reasoning' expects a boolean" in error + + def test_accepts_boolean_spellings(self): + for value in (True, False, "true", "FALSE", "1", "no"): + assert validate_argument_values( + "ai_user-action", {"reasoning": value}, USER_ACTION_TYPED + ) is None + + def test_rejects_data_source_the_parameter_does_not_accept(self): + # ai_user-action's action declares CONSTANT and VARIABLE only. + error = validate_argument_values( + "ai_user-action", + {"action": {"data_source": "DATATABLE", "table_name": "T", "column": "c"}}, + USER_ACTION_TYPED, + ) + assert "does not accept data_source DATATABLE" in error + assert "accepted: CONSTANT, VARIABLE" in error + + def test_accepts_declared_data_source(self): + assert validate_argument_values( + "wait", + {"duration": {"data_source": "DATATABLE", "table_name": "T", "column": "c"}}, + WAIT, + ) is None + + def test_does_not_check_values_behind_a_binding(self): + # A variable's value is only known at execution time. + assert validate_argument_values( + "wait", {"duration": {"data_source": "VARIABLE", "value": "secs"}}, WAIT + ) is None + + def test_reports_every_offending_argument(self): + error = validate_argument_values( + "checkpoint_text", {"context": "sidebar"}, CHECKPOINT + ) + assert error.startswith("Invalid cmd_arguments for command 'checkpoint_text':") + assert "get_command_definitions" in error + + def test_fails_open_without_parameter_contracts(self): + contract = CommandContract("wait", mandatory=frozenset({"duration"})) + assert validate_argument_values("wait", {"duration": "nonsense"}, contract) is None + + def test_ignores_alias_names(self): + assert validate_argument_values("wait", {"waitDuration": 5000}, WAIT) is not None + + +def _script_with_variables() -> dict: + script = new_empty_script() + add_script_variable(script, "waitSecs", "string", "3") + add_script_variable(script, "waitSecsNum", "number", 3) + return script + + +class TestValidateVariableBindings: + def test_accepts_variable_of_matching_type(self): + assert validate_variable_bindings( + "wait", + {"duration": {"data_source": "VARIABLE", "value": "waitSecsNum"}}, + WAIT, + _script_with_variables(), + ) is None + + def test_rejects_variable_of_another_type(self): + # The case the UI refuses: a string variable on a Number parameter. + error = validate_variable_bindings( + "wait", + {"duration": {"data_source": "VARIABLE", "value": "waitSecs"}}, + WAIT, + _script_with_variables(), + ) + assert "'duration' is declared INTEGER but variable 'waitSecs' is a string" in error + assert "only binds a variable of the matching type" in error + assert "waitSecsNum (number)" in error + + def test_rejects_undefined_variable(self): + error = validate_variable_bindings( + "wait", + {"duration": {"data_source": "VARIABLE", "value": "nope"}}, + WAIT, + _script_with_variables(), + ) + assert "variable 'nope', which this test does not define" in error + assert "add_test_variable" in error + + def test_lists_defined_values_with_their_types(self): + error = validate_variable_bindings( + "wait", + {"duration": {"data_source": "VARIABLE", "value": "nope"}}, + WAIT, + _script_with_variables(), + ) + assert "DUT (device)" in error + assert "waitSecs (string)" in error + assert "waitSecsNum (number)" in error + + def test_rejects_missing_variable_name(self): + error = validate_variable_bindings( + "wait", {"duration": {"data_source": "VARIABLE"}}, WAIT, _script_with_variables() + ) + assert "no variable name was given" in error + + def test_accepts_dut_for_a_handset_parameter(self): + # DUT lives in script.parameters[], not variables[]. + contract = CommandContract( + "touch_tap", + mandatory=frozenset({"handsetId"}), + parameters={"handsetId": ParameterContract( + name="handsetId", data_type="HANDSET", + data_sources=frozenset({"CONSTANT", "VARIABLE", "DATATABLE"}), mandatory=True, + )}, + ) + assert validate_variable_bindings( + "touch_tap", + {"handsetId": {"data_source": "VARIABLE", "value": "DUT"}}, + contract, + _script_with_variables(), + ) is None + + def test_rejects_string_variable_on_a_handset_parameter(self): + contract = CommandContract( + "touch_tap", + mandatory=frozenset({"handsetId"}), + parameters={"handsetId": ParameterContract( + name="handsetId", data_type="HANDSET", mandatory=True, + )}, + ) + error = validate_variable_bindings( + "touch_tap", + {"handsetId": {"data_source": "VARIABLE", "value": "waitSecs"}}, + contract, + _script_with_variables(), + ) + assert "declared HANDSET but variable 'waitSecs' is a string" in error + + def test_only_checks_existence_for_unmapped_parameter_types(self): + # checkpoint_text declares an 'ocr' parameter of type PROPERTY. + contract = CommandContract( + "checkpoint_text", + optional=frozenset({"ocr"}), + parameters={"ocr": ParameterContract(name="ocr", data_type="PROPERTY")}, + ) + assert validate_variable_bindings( + "checkpoint_text", + {"ocr": {"data_source": "VARIABLE", "value": "waitSecs"}}, + contract, + _script_with_variables(), + ) is None + + def test_ignores_constant_and_datatable_arguments(self): + script = _script_with_variables() + assert validate_variable_bindings("wait", {"duration": "3"}, WAIT, script) is None + assert validate_variable_bindings( + "wait", + {"duration": {"data_source": "DATATABLE", "table_name": "T", "column": "c"}}, + WAIT, + script, + ) is None + + def test_fails_open_without_parameter_contracts(self): + contract = CommandContract("wait", mandatory=frozenset({"duration"})) + assert validate_variable_bindings( + "wait", + {"duration": {"data_source": "VARIABLE", "value": "nope"}}, + contract, + _script_with_variables(), + ) is None + + def test_normalizes_alias_before_checking(self): + error = validate_variable_bindings( + "wait", + {"waitDuration": {"data_source": "VARIABLE", "value": "waitSecs"}}, + WAIT, + _script_with_variables(), + ) + assert "'duration' is declared INTEGER" in error + + +class TestErrorPolicyCoverage: + """The legacy IDE documents five policies; all of them must survive parsing.""" + + @staticmethod + def _contract(monkeypatch, perfecto_token, policy): + async def fake_api_request(_token, _method, endpoint=None, result_formatter=None, **kwargs): + return BaseResult(result=result_formatter(_definition("cmd", { + "type": "ACTION", + "errorPolicy": policy, + "mandatoryParameters": [{"name": "x"}], + }), None)) + + definitions.reset_command_contract_cache() + monkeypatch.setattr(definitions, "api_request", fake_api_request) + return asyncio.run(command_contract(perfecto_token, "cmd")) + + def test_keeps_every_documented_policy(self, perfecto_token, monkeypatch): + for policy in ("ABORT", "IGNORE", "BREAK", "CONTINUE", "CATCH"): + contract = self._contract(monkeypatch, perfecto_token, policy) + assert contract.error_policy == policy + + def test_still_drops_an_unknown_policy(self, perfecto_token, monkeypatch): + assert self._contract(monkeypatch, perfecto_token, "RETRY").error_policy is None + + +CONCAT_VALUE = ParameterContract( + name="value", data_type="STRING", data_sources=frozenset({"CONSTANT", "VARIABLE", "DATATABLE"}), + mandatory=True, min_occurrences=2, max_occurrences=99, +) +CONCAT = CommandContract( + "text_concat", mandatory=frozenset({"value"}), element_type="Action", + parameters={"value": CONCAT_VALUE}, +) + + +class TestOccurrenceValidation: + def test_accepts_a_list_within_bounds(self): + assert validate_argument_values("text_concat", {"value": ["a", "b", "c"]}, CONCAT) is None + + def test_rejects_fewer_values_than_required(self): + error = validate_argument_values("text_concat", {"value": "a"}, CONCAT) + assert "'value' needs at least 2 values" in error + + def test_rejects_more_values_than_allowed(self): + capped = CommandContract( + "cmd", parameters={"x": ParameterContract(name="x", max_occurrences=2)} + ) + error = validate_argument_values("cmd", {"x": ["a", "b", "c"]}, capped) + assert "'x' accepts at most 2 value(s), got 3" in error + + def test_validates_every_occurrence(self): + numbers = CommandContract("cmd", parameters={"n": ParameterContract( + name="n", data_type="INTEGER", minimum=0.0, maximum=10.0, max_occurrences=5, + )}) + error = validate_argument_values("cmd", {"n": [1, 99]}, numbers) + assert "must be within 0..10" in error + + def test_single_value_parameter_is_unaffected(self): + assert validate_argument_values("wait", {"duration": 30}, WAIT) is None + + def test_is_multivalued_reflects_max_occurrences(self): + assert CONCAT_VALUE.is_multivalued is True + assert WAIT_DURATION.is_multivalued is False diff --git a/tests/test_ai_scriptless_flow_element_counts.py b/tests/test_ai_scriptless_flow_element_counts.py index ef78919..c591cd9 100644 --- a/tests/test_ai_scriptless_flow_element_counts.py +++ b/tests/test_ai_scriptless_flow_element_counts.py @@ -65,8 +65,8 @@ def test_two_direct_then_children_gives_if_num_five(self): def test_nested_if_counts_direct_child_only_not_recursive(self): script = new_empty_script() - script["flowElements"] = [build_if_statement("outer", "Outer")] - inner = build_if_statement("inner", "Inner") + script["flowElements"] = [build_if_statement("Outer")] + inner = build_if_statement("Inner") insert_flow_element(script, inner, parent_path="0.b0") insert_flow_element(script, _comment("deep"), parent_path="0.b0.0.b0") outer = script["flowElements"][0] diff --git a/tests/test_ai_scriptless_formatters.py b/tests/test_ai_scriptless_formatters.py index e809f8f..aa57e79 100644 --- a/tests/test_ai_scriptless_formatters.py +++ b/tests/test_ai_scriptless_formatters.py @@ -24,6 +24,7 @@ format_ai_scriptless_tests_filter_values, format_command_catalog, format_command_definitions, + format_step_detail, format_snapshots_list, format_test_structure, format_test_variables, @@ -92,7 +93,7 @@ def _tree_api_payload() -> dict: tap["uuid"] = "ignored-in-formatter" group = build_logical_step("Setup") group["flowElements"] = [build_flow_element("comment", {"text": "inside group"})] - condition = build_if_statement("x == 1", "Check x") + condition = build_if_statement("Check x") condition["branches"][0]["flowElements"] = [ build_flow_element("ai_validation", {"validation": "OK"}), ] @@ -151,7 +152,7 @@ def test_formats_nested_flow_with_step_paths(self): assert structure.flow_elements[0].step_path == "0" assert structure.flow_elements[1].name == "Setup" assert structure.flow_elements[1].children[0].step_path == "1.0" - assert structure.flow_elements[2].name == "Condition (x == 1)" + assert structure.flow_elements[2].name == "Condition (Check x)" then_branch = structure.flow_elements[2].children[0] assert then_branch.type == "Branch" assert then_branch.step_path == "2.b0" @@ -397,3 +398,260 @@ def test_applies_pagination_and_visibility_filter(self): class TestFormatterCommandId: def test_replaces_slashes_in_subcommand(self): assert _command_id("webpage", "element/click") == "webpage_element_click" + + +def _checkpoint_text_definition() -> dict: + """Shape observed in the real command repository payload.""" + return { + "commandId": "checkpoint_text", + "data": { + "display": {"name": "Check text"}, + "type": "VALIDATION", + "errorPolicy": "IGNORE", + "mandatoryParameters": [ + { + "name": "handsetId", + "dataType": "HANDSET", + "dataSources": ["CONSTANT", "VARIABLE", "DATATABLE"], + "display": {"name": "Device ID", "editorLevel": "PUBLIC", "inReport": False}, + }, + { + "name": "content", + "dataType": "STRING", + "dataSources": ["CONSTANT", "VARIABLE", "DATATABLE"], + "display": {"name": "Text", "editorLevel": "PUBLIC", "inReport": True}, + }, + ], + "optionalParameters": [ + { + "name": "context", + "dataType": "STRING", + "dataSources": ["CONSTANT", "VARIABLE", "DATATABLE"], + "defaultValue": "all", + "restriction": {"type": "COMBO", "value": "all,body,link", "label": "all,body,"}, + "display": {"name": "Context", "editorLevel": "PUBLIC", "inReport": False}, + "helpText": "Where to look for the text", + }, + { + "name": "timeout", + "dataType": "INTEGER", + "dataSources": ["CONSTANT", "VARIABLE"], + "defaultValue": "0", + "restriction": {"type": "RANGE", "range": {"minValue": 0, "maxValue": 600}}, + "display": {"name": "Timeout", "editorLevel": "PUBLIC", "inReport": False}, + }, + ], + }, + } + + +class TestFormatStepDetail: + @staticmethod + def _element() -> dict: + element = build_flow_element("checkpoint_text", {"content": "Welcome"}) + element["arguments"].append({ + "@type": "FunctionArgument", + "name": "context", + "data": {"@type": "ConstantArgumentData", "dataSource": "CONSTANT", "value": "body"}, + }) + return element + + def _detail(self, element=None, step_path="2.0"): + return format_step_detail( + element if element is not None else self._element(), + item_key="PRIVATE:Folder/Test.xml", + step_path=step_path, + command_definitions=[_checkpoint_text_definition()], + ) + + def test_reports_step_identity(self): + detail = self._detail() + assert detail.item_key == "PRIVATE:Folder/Test.xml" + assert detail.step_path == "2.0" + assert detail.command_id == "checkpoint_text" + assert detail.type == "Validation" + assert detail.error_policy == "IGNORE" + assert detail.active is True + + def test_reports_argument_values_and_sources(self): + detail = self._detail() + arguments = {argument.name: argument for argument in detail.arguments} + assert arguments["content"].value == "Welcome" + assert arguments["content"].data_source == "CONSTANT" + assert arguments["handsetId"].value == "DUT" + assert arguments["handsetId"].data_source == "VARIABLE" + + def test_joins_declared_parameter_metadata(self): + detail = self._detail() + arguments = {argument.name: argument for argument in detail.arguments} + assert arguments["content"].parameter_type == "STRING" + assert arguments["content"].mandatory is True + assert arguments["content"].label == "Text" + assert arguments["context"].mandatory is False + assert arguments["context"].allowed_values == ["all", "body", "link"] + assert arguments["handsetId"].allowed_data_sources == ["CONSTANT", "VARIABLE", "DATATABLE"] + + def test_lists_unset_declared_parameters(self): + detail = self._detail() + unset = {parameter.name: parameter for parameter in detail.unset_parameters} + assert "timeout" in unset + assert unset["timeout"].parameter_type == "INTEGER" + assert unset["timeout"].value_range == "0..600" + assert unset["timeout"].default_value == "0" + assert unset["timeout"].mandatory is False + # Arguments already set are not repeated as unset. + assert "content" not in unset and "context" not in unset + + def test_flags_undeclared_argument(self): + element = self._element() + element["arguments"].append({ + "@type": "FunctionArgument", + "name": "typo", + "data": {"@type": "ConstantArgumentData", "dataSource": "CONSTANT", "value": "x"}, + }) + detail = self._detail(element) + typo = next(argument for argument in detail.arguments if argument.name == "typo") + assert typo.declared is False + assert any("not declared by the command" in note for note in detail.notes) + + def test_masks_secured_values(self): + element = self._element() + element["arguments"].append({ + "@type": "FunctionArgument", + "name": "password", + "data": {"@type": "ConstantArgumentData", "dataSource": "CONSTANT", + "value": "hunter2", "secured": True}, + }) + detail = self._detail(element) + password = next(argument for argument in detail.arguments if argument.name == "password") + assert password.value == "" + + def test_notes_excluded_step(self): + element = self._element() + element["active"] = False + detail = self._detail(element) + assert detail.active is False + assert any("excluded from the run" in note for note in detail.notes) + + def test_reports_container_children_paths(self): + group = build_logical_step("Setup") + group["flowElements"] = [build_flow_element("wait"), build_flow_element("comment")] + detail = format_step_detail(group, item_key="PRIVATE:F/T.xml", step_path="1") + assert detail.type == "LogicalStep" + assert detail.label == "Setup" + assert detail.children == ["1.0", "1.1"] + + def test_reports_condition_branches_and_label(self): + condition = build_if_statement("Check x") + detail = format_step_detail(condition, item_key="PRIVATE:F/T.xml", step_path="5") + assert detail.type == "IfStatement" + assert detail.label == "Check x" + assert detail.children == ["5.b0", "5.b1"] + + def test_works_without_definitions(self): + detail = format_step_detail( + build_flow_element("wait", {"duration": "3"}), + item_key="PRIVATE:F/T.xml", + step_path="0", + ) + arguments = {argument.name: argument for argument in detail.arguments} + assert arguments["duration"].value == "3" + # Without a definition nothing can be called undeclared. + assert all(argument.declared for argument in detail.arguments) + assert detail.unset_parameters == [] + + def test_renders_integral_float_range_as_integers(self): + # The script payload serializes bounds as floats (0.0 / 600.0). + definition = _checkpoint_text_definition() + timeout = definition["data"]["optionalParameters"][1] + timeout["restriction"]["range"] = {"minValue": 0.0, "maxValue": 600.0} + detail = format_step_detail( + self._element(), + item_key="PRIVATE:Folder/Test.xml", + step_path="0", + command_definitions=[definition], + ) + unset = {parameter.name: parameter for parameter in detail.unset_parameters} + assert unset["timeout"].value_range == "0..600" + + def test_omits_help_text_for_optional_unset_parameters(self): + detail = self._detail() + unset = {parameter.name: parameter for parameter in detail.unset_parameters} + assert unset["timeout"].mandatory is False + assert unset["timeout"].help_text is None + + def test_keeps_help_text_for_mandatory_unset_parameters(self): + definition = _checkpoint_text_definition() + definition["data"]["mandatoryParameters"].append({ + "name": "extra", + "dataType": "STRING", + "helpText": "Needed for the step to run", + "display": {"name": "Extra", "editorLevel": "PUBLIC", "inReport": False}, + }) + detail = format_step_detail( + self._element(), + item_key="PRIVATE:Folder/Test.xml", + step_path="0", + command_definitions=[definition], + ) + unset = {parameter.name: parameter for parameter in detail.unset_parameters} + assert unset["extra"].help_text == "Needed for the step to run" + # Mandatory unset parameters come first. + assert detail.unset_parameters[0].name == "extra" + + def test_notes_mandatory_argument_left_empty(self): + element = build_flow_element("checkpoint_text") + element["arguments"].append({ + "@type": "FunctionArgument", + "name": "content", + "data": {"@type": "ConstantArgumentData", "dataSource": "CONSTANT", "value": ""}, + }) + detail = self._detail(element) + assert any("Mandatory argument(s) with no value: content" in note for note in detail.notes) + + def test_exposes_datatable_binding(self): + element = build_flow_element("checkpoint_text", { + "content": {"data_source": "DATATABLE", "table_name": "ProbeTable", "column": "text"}, + }) + detail = self._detail(element) + content = next(a for a in detail.arguments if a.name == "content") + assert content.data_source == "DATATABLE" + assert content.table_name == "ProbeTable" + assert content.column == "text" + assert content.value is None + + def test_datatable_binding_is_not_reported_as_empty_mandatory(self): + element = build_flow_element("checkpoint_text", { + "content": {"data_source": "DATATABLE", "table_name": "ProbeTable", "column": "text"}, + }) + detail = self._detail(element) + assert not any("Mandatory argument(s) with no value" in note for note in detail.notes) + + def test_unbound_datatable_is_reported_as_empty_mandatory(self): + element = build_flow_element("checkpoint_text", {"content": {"data_source": "DATATABLE"}}) + detail = self._detail(element) + assert any("Mandatory argument(s) with no value: content" in note for note in detail.notes) + + def test_reads_group_name_written_by_the_ui(self): + # Shape captured from a UI-authored group: the title lives in `name`. + group = {"@type": "LogicalStep", "transaction": "", "name": "Setup", "flowElements": []} + detail = format_step_detail(group, item_key="PRIVATE:F/T.xml", step_path="2") + assert detail.name == "Setup" + assert detail.label == "Setup" + + def test_still_reads_the_legacy_label(self): + group = {"@type": "LogicalStep", "label": "Old", "flowElements": []} + assert format_step_detail(group, item_key="PRIVATE:F/T.xml", step_path="2").name == "Old" + + def test_renders_integral_float_loop_count(self): + loop = {"@type": "Loop", "iterator": {"@type": "RepeatIterator", "count": 2.0}, + "flowElements": []} + detail = format_step_detail(loop, item_key="PRIVATE:F/T.xml", step_path="1") + assert detail.name == "Loop (2)" + assert detail.loop_count == 2 + + def test_step_name_shows_the_last_occurrence_of_a_multivalued_parameter(self): + # The UI labels the step with the last value in the list; match it. + element = build_flow_element("text_concat", {"value": ["first", "last"]}) + from formatters.ai_scriptless import _format_argument_display_value + assert _format_argument_display_value(element, "value") == "last" diff --git a/tests/test_ai_scriptless_if_statement_tree.py b/tests/test_ai_scriptless_if_statement_tree.py index 9611466..c145720 100644 --- a/tests/test_ai_scriptless_if_statement_tree.py +++ b/tests/test_ai_scriptless_if_statement_tree.py @@ -91,7 +91,7 @@ def _clause_child_commands(ifs: dict) -> list[str | None]: class TestIfStatementBuilderContract: def test_new_builder_aliases_clauses_to_branches(self): - ifs = build_if_statement("x", "Check") + ifs = build_if_statement("Check") assert ifs["thenClause"] is ifs["branches"][0] assert ifs["elseClause"] is ifs["branches"][1] @@ -183,7 +183,7 @@ def test_prefers_clause_when_it_has_more_children(self): def test_nested_if_inside_then_branch(self): inner = _api_style_if_statement(then_in_clause=[_comment("inner")]) - outer = build_if_statement("outer", "Outer") + outer = build_if_statement("Outer") outer["branches"][0]["flowElements"] = [inner] outer["thenClause"] = build_branch("THEN") script = _script_with_if_statement(outer) @@ -248,7 +248,7 @@ def test_find_element_round_trip(self, script: dict): assert element["command"] == "comment" def test_insert_nested_condition_in_then(self, script: dict): - inner = build_if_statement("inner", "Inner") + inner = build_if_statement("Inner") insert_flow_element(script, inner, parent_path="0.b0") nested = script["flowElements"][0]["branches"][0]["flowElements"][0] assert nested["thenClause"] is nested["branches"][0] diff --git a/tests/test_ai_scriptless_manager.py b/tests/test_ai_scriptless_manager.py index 0deb50d..be6410e 100644 --- a/tests/test_ai_scriptless_manager.py +++ b/tests/test_ai_scriptless_manager.py @@ -26,6 +26,7 @@ from models.result import BaseResult from tools import ai_scriptless_manager from tools.ai_scriptless import definitions +from tools.ai_scriptless.variables import add_script_variable from tools.ai_scriptless.elements import ( build_flow_element, build_if_statement, @@ -713,27 +714,28 @@ def test_add_loop_inside_logical_step(self, perfecto_token, monkeypatch): assert nested[0]["iterator"]["count"] == 2 assert result.result["step_path"] == "0.0" - def test_add_condition_with_expression(self, perfecto_token, monkeypatch): + def test_add_condition_notes_the_statement_mechanism(self, perfecto_token, monkeypatch): captured: dict = {} _mock_load_and_mutate(monkeypatch, captured=captured) manager = AiScriptlessManager(perfecto_token, ctx=None) - result = asyncio.run(manager.add_condition(TEST_ID, expression="x == 1", label="Check")) + result = asyncio.run(manager.add_condition(TEST_ID, label="Check")) _assert_step_path_notes(result) - assert result.result["expression"] == "x == 1" assert captured["script"]["flowElements"][0]["@type"] == "IfStatement" + assert captured["script"]["flowElements"][0]["label"] == "Check" + assert any("no expression" in note for note in result.result["notes"]) def test_add_condition_inside_then_branch(self, perfecto_token, monkeypatch): captured: dict = {} script = new_empty_script() - script["flowElements"] = [build_if_statement("x == 1", "Check")] + script["flowElements"] = [build_if_statement("Check")] _mock_load_and_mutate(monkeypatch, script, captured) manager = AiScriptlessManager(perfecto_token, ctx=None) result = asyncio.run(manager.add_condition( TEST_ID, - expression="y == 2", + label="Nested", parent_path="0.b0", )) @@ -743,23 +745,29 @@ def test_add_condition_inside_then_branch(self, perfecto_token, monkeypatch): assert then_branch["flowElements"][0]["@type"] == "IfStatement" assert result.result["step_path"] == "0.b0.0" - def test_set_condition_expression(self, perfecto_token, monkeypatch): + def test_set_command_error_policy(self, perfecto_token, monkeypatch): captured: dict = {} - script = new_empty_script() - script["flowElements"] = [build_if_statement("old", "If")] - _mock_load_and_mutate(monkeypatch, script, captured) + _mock_load_and_mutate(monkeypatch, _script_with_steps("checkpoint_text"), captured) manager = AiScriptlessManager(perfecto_token, ctx=None) - result = asyncio.run(manager.set_condition_expression(TEST_ID, "0", "new == true")) + result = asyncio.run(manager.set_command_error_policy(TEST_ID, "0", "catch")) _assert_step_path_notes(result) - assert result.result["expression"] == "new == true" - assert captured["script"]["flowElements"][0]["expression"] == "new == true" + assert result.result["error_policy"] == "CATCH" + assert captured["script"]["flowElements"][0]["errorPolicy"] == "CATCH" + assert any("Statement" in note for note in result.result["notes"]) + + def test_set_command_error_policy_rejects_unknown_value(self, perfecto_token): + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.set_command_error_policy(TEST_ID, "0", "RETRY")) + assert "error_policy must be one of" in result.error + assert "CATCH" in result.error - def test_set_condition_expression_requires_expression(self, perfecto_token): + def test_add_condition_rejects_an_expression(self, perfecto_token): manager = AiScriptlessManager(perfecto_token, ctx=None) - result = asyncio.run(manager.set_condition_expression(TEST_ID, "0", "")) - assert result.error == "expression is required" + result = asyncio.run(manager.add_condition(TEST_ID, expression="x == 1")) + assert "A condition has no expression" in result.error + assert "set_command_error_policy" in result.error class TestVariableOperations: @@ -786,8 +794,12 @@ async def fake_fetch(_token, _test_id): result = asyncio.run(manager.list_test_variables(TEST_ID)) assert result.error is None - assert len(result.result) == 1 - assert result.result[0].name == "token" + # Runtime parameters come first, as in the UI dialog: DUT, then the variables. + assert [variable.name for variable in result.result] == ["DUT", "token"] + dut = result.result[0] + assert dut.type == "device" + assert dut.set_at_runtime is True + assert result.result[1].set_at_runtime is False def test_list_test_variables_propagates_fetch_error(self, perfecto_token, monkeypatch): async def fake_fetch(_token, _test_id): @@ -812,7 +824,23 @@ def test_add_test_variable(self, perfecto_token, monkeypatch): assert result.result["name"] == "count" assert result.result["type"] == "number" assert result.result["set_at_runtime"] is True - assert captured["script"]["variables"][0]["@type"] == "Parameter" + # set_at_runtime=True means a Parameter in parameters[], next to DUT. + assert captured["script"]["variables"] == [] + added = captured["script"]["parameters"][-1] + assert added["@type"] == "Parameter" + assert added["data"]["name"] == "count" + + def test_add_test_variable_without_runtime_flag_lands_in_variables( + self, perfecto_token, monkeypatch): + captured: dict = {} + _mock_load_and_mutate(monkeypatch, captured=captured) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_test_variable(TEST_ID, "count", "number", 42)) + + assert result.error is None + assert [p["data"]["name"] for p in captured["script"]["parameters"]] == ["DUT"] + assert captured["script"]["variables"][0]["@type"] == "Variable" def test_modify_test_variable_requires_change(self, perfecto_token): manager = AiScriptlessManager(perfecto_token, ctx=None) @@ -1057,7 +1085,7 @@ def _dispatcher_action_cases() -> list[tuple[str, dict]]: ("add_logical_step", {"test_id": test_id, "label": "Group"}), ("add_loop", {"test_id": test_id, "count": 2}), ("add_condition", {"test_id": test_id, "expression": "true", "label": "If"}), - ("set_condition_expression", {"test_id": test_id, "step_path": "1", "expression": "false"}), + ("set_command_error_policy", {"test_id": test_id, "step_path": "0", "error_policy": "IGNORE"}), ("move_command", {"test_id": test_id, "step_path": "0", "after_path": "0"}), ("delete_test", {"test_id": test_id}), ("move_test", {"test_id": test_id, "folder": "Moved"}), @@ -1074,7 +1102,7 @@ def _setup_dispatcher_mocks(monkeypatch, captured: dict | None = None, persisted script = new_empty_script() script["flowElements"] = [ build_flow_element("wait"), - build_if_statement("x == 1", "If"), + build_if_statement("If"), ] script["numOfFlowElements"] = 2 script["variables"] = [{ @@ -1259,7 +1287,7 @@ async def fake_fetch(_token, _test_id): result = asyncio.run(_call_tool(tool, "list_test_variables", {"test_id": TEST_ID})) assert result.error is None - assert result.result[0].name == "flag" + assert [variable.name for variable in result.result] == ["DUT", "flag"] def test_defaults_none_args_to_empty_dict(self, perfecto_token, monkeypatch): async def fake_api_request(*_args, **_kwargs): @@ -1433,7 +1461,7 @@ def test_routes_add_test_variable(self, perfecto_token, monkeypatch): assert result.error is None assert result.result["name"] == "retry" - names = [v["data"]["name"] for v in captured["script"]["variables"]] + names = [p["data"]["name"] for p in captured["script"]["parameters"]] assert "retry" in names @pytest.mark.parametrize("action,args", _dispatcher_action_cases()) @@ -1468,8 +1496,8 @@ def decorator(fn): class TestCmdArgumentsValidation: def test_add_command_rejects_undeclared_argument_name( - self, perfecto_token, monkeypatch, declare_command_parameters): - declare_command_parameters({"ai_user-action": (["action"], ["handsetId"])}) + self, perfecto_token, monkeypatch, declare_commands): + declare_commands({"ai_user-action": {"mandatory": ["action"], "optional": ["handsetId"]}}) _mock_load_and_mutate(monkeypatch) manager = AiScriptlessManager(perfecto_token, ctx=None) @@ -1484,9 +1512,9 @@ def test_add_command_rejects_undeclared_argument_name( assert "get_command_definitions" in result.error def test_add_command_accepts_declared_argument_name( - self, perfecto_token, monkeypatch, declare_command_parameters): + self, perfecto_token, monkeypatch, declare_commands): captured: dict = {} - declare_command_parameters({"ai_user-action": (["action"], ["handsetId"])}) + declare_commands({"ai_user-action": {"mandatory": ["action"], "optional": ["handsetId"]}}) _mock_load_and_mutate(monkeypatch, captured=captured) manager = AiScriptlessManager(perfecto_token, ctx=None) @@ -1501,9 +1529,9 @@ def test_add_command_accepts_declared_argument_name( assert {argument["name"] for argument in arguments} == {"action", "handsetId"} def test_add_command_accepts_canonical_name_when_alias_is_declared( - self, perfecto_token, monkeypatch, declare_command_parameters): + self, perfecto_token, monkeypatch, declare_commands): # The repository declares waitDuration; the spec canonicalizes it to duration. - declare_command_parameters({"wait": (["waitDuration"], [])}) + declare_commands({"wait": {"mandatory": ["waitDuration"]}}) _mock_load_and_mutate(monkeypatch) manager = AiScriptlessManager(perfecto_token, ctx=None) @@ -1524,8 +1552,8 @@ def test_add_command_fails_open_without_definitions(self, perfecto_token, monkey assert result.error is None def test_add_command_notes_empty_mandatory_parameter( - self, perfecto_token, monkeypatch, declare_command_parameters): - declare_command_parameters({"ai_user-action": (["action"], ["handsetId"])}) + self, perfecto_token, monkeypatch, declare_commands): + declare_commands({"ai_user-action": {"mandatory": ["action"], "optional": ["handsetId"]}}) _mock_load_and_mutate(monkeypatch) manager = AiScriptlessManager(perfecto_token, ctx=None) @@ -1535,8 +1563,8 @@ def test_add_command_notes_empty_mandatory_parameter( assert any("Mandatory parameter(s) left empty" in note for note in result.result["notes"]) def test_modify_command_rejects_undeclared_argument_name( - self, perfecto_token, monkeypatch, declare_command_parameters): - declare_command_parameters({"wait": ([], ["duration"])}) + self, perfecto_token, monkeypatch, declare_commands): + declare_commands({"wait": {"optional": ["duration"]}}) _mock_load_and_mutate(monkeypatch, _script_with_steps("wait")) manager = AiScriptlessManager(perfecto_token, ctx=None) @@ -1545,9 +1573,9 @@ def test_modify_command_rejects_undeclared_argument_name( assert "Unknown cmd_arguments for command 'wait'" in result.error def test_modify_command_accepts_declared_argument_name( - self, perfecto_token, monkeypatch, declare_command_parameters): + self, perfecto_token, monkeypatch, declare_commands): captured: dict = {} - declare_command_parameters({"wait": ([], ["duration"])}) + declare_commands({"wait": {"optional": ["duration"]}}) _mock_load_and_mutate(monkeypatch, _script_with_steps("wait"), captured) manager = AiScriptlessManager(perfecto_token, ctx=None) @@ -1564,10 +1592,14 @@ def test_definitions_are_fetched_once_per_command(self, perfecto_token, monkeypa async def counting_fetch(_token, command_id): calls.append(command_id) - return frozenset({"action"}), frozenset({"handsetId"}) + return definitions.CommandContract( + command_id=command_id, + mandatory=frozenset({"action"}), + optional=frozenset({"handsetId"}), + ) - definitions.reset_declared_parameters_cache() - monkeypatch.setattr(definitions, "_fetch_declared_parameters", counting_fetch) + definitions.reset_command_contract_cache() + monkeypatch.setattr(definitions, "_fetch_command_contract", counting_fetch) _mock_load_and_mutate(monkeypatch) manager = AiScriptlessManager(perfecto_token, ctx=None) @@ -1589,3 +1621,567 @@ def test_unknown_action_hints_cmd_arguments_collision(self, perfecto_token): assert "not found in AI Scriptless manager tool" in result.error assert "must stay nested inside 'cmd_arguments'" in result.error + + +class TestElementTypeFromContract: + def test_add_command_uses_declared_element_type_and_error_policy( + self, perfecto_token, monkeypatch, declare_commands): + captured: dict = {} + # A validation command the local spec cannot recognize by name. + declare_commands({"verify_page": { + "mandatory": ["expected"], + "element_type": "Validation", + "error_policy": "IGNORE", + }}) + _mock_load_and_mutate(monkeypatch, captured=captured) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_command( + TEST_ID, + "verify_page", + cmd_arguments={"expected": "Dashboard"}, + )) + + assert result.error is None + element = captured["script"]["flowElements"][0] + assert element["@type"] == "Validation" + assert element["errorPolicy"] == "IGNORE" + + def test_add_command_keeps_local_default_when_contract_is_silent( + self, perfecto_token, monkeypatch, declare_commands): + captured: dict = {} + # wait declares errorPolicy null upstream. + declare_commands({"wait": {"mandatory": ["duration"], "element_type": "Action"}}) + _mock_load_and_mutate(monkeypatch, captured=captured) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_command(TEST_ID, "wait", cmd_arguments={"duration": "3"})) + + assert result.error is None + element = captured["script"]["flowElements"][0] + assert element["@type"] == "Action" + assert element["errorPolicy"] == "ABORT" + + def test_add_command_falls_back_without_contract(self, perfecto_token, monkeypatch): + captured: dict = {} + _mock_load_and_mutate(monkeypatch, captured=captured) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_command(TEST_ID, "checkpoint_text")) + + assert result.error is None + element = captured["script"]["flowElements"][0] + assert element["@type"] == "Validation" + assert element["errorPolicy"] == "IGNORE" + + +class TestViewTestStep: + @staticmethod + def _payload() -> dict: + script = new_empty_script() + group = build_logical_step("Setup") + group["flowElements"] = [build_flow_element("wait", {"duration": "3"})] + script["flowElements"] = [build_flow_element("ai_user-action", {"action": "Tap Login"}), group] + return { + "script": script, + "commandDefinitions": [{ + "commandId": "ai_user-action", + "data": { + "display": {"name": "AI action"}, + "mandatoryParameters": [ + {"name": "handsetId", "dataType": "HANDSET", + "display": {"name": "Device ID", "editorLevel": "PUBLIC", "inReport": False}}, + {"name": "action", "dataType": "STRING", + "display": {"name": "Action", "editorLevel": "PUBLIC", "inReport": True}}, + ], + "optionalParameters": [ + {"name": "reasoning", "dataType": "BOOLEAN", "defaultValue": False, + "display": {"name": "Reasoning", "editorLevel": "PUBLIC", "inReport": False}}, + ], + }, + }], + } + + def _mock_fetch(self, monkeypatch, captured: dict | None = None): + payload = self._payload() + + async def fake_fetch(_token, test_id): + if captured is not None: + captured["test_id"] = test_id + return BaseResult(result=copy.deepcopy(payload)) + + monkeypatch.setattr(ai_scriptless_manager, "fetch_script_payload", fake_fetch) + + def test_returns_step_configuration(self, perfecto_token, monkeypatch): + captured: dict = {} + self._mock_fetch(monkeypatch, captured) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.view_test_step(TEST_ID, "0")) + + assert result.error is None + assert captured["test_id"] == TEST_ID + detail = result.result + assert detail.step_path == "0" + assert detail.command_id == "ai_user-action" + arguments = {argument.name: argument for argument in detail.arguments} + assert arguments["action"].value == "Tap Login" + assert arguments["action"].parameter_type == "STRING" + assert arguments["action"].mandatory is True + assert [parameter.name for parameter in detail.unset_parameters] == ["reasoning"] + assert result.info is not None + + def test_resolves_nested_step_path(self, perfecto_token, monkeypatch): + self._mock_fetch(monkeypatch) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.view_test_step(TEST_ID, "1.0")) + + assert result.error is None + assert result.result.command_id == "wait" + assert result.result.arguments[0].name == "duration" + + def test_requires_test_id(self, perfecto_token): + manager = AiScriptlessManager(perfecto_token, ctx=None) + assert "test_id is required" in asyncio.run(manager.view_test_step("", "0")).error + + def test_requires_step_path(self, perfecto_token): + manager = AiScriptlessManager(perfecto_token, ctx=None) + assert "step_path is required" in asyncio.run(manager.view_test_step(TEST_ID, "")).error + + def test_unknown_step_path_returns_error(self, perfecto_token, monkeypatch): + self._mock_fetch(monkeypatch) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.view_test_step(TEST_ID, "9")) + + assert "step_path not found: 9" in result.error + assert "view_test_structure" in result.error + + def test_propagates_fetch_error(self, perfecto_token, monkeypatch): + async def failing_fetch(_token, _test_id): + return BaseResult(error="Invalid credentials") + + monkeypatch.setattr(ai_scriptless_manager, "fetch_script_payload", failing_fetch) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + assert asyncio.run(manager.view_test_step(TEST_ID, "0")).error == "Invalid credentials" + + def test_dispatcher_routes_view_test_step(self, perfecto_token, monkeypatch): + self._mock_fetch(monkeypatch) + + tool = _register_tool(perfecto_token) + result = asyncio.run(_call_tool(tool, "view_test_step", {"test_id": TEST_ID, "step_path": "0"})) + + assert result.error is None + assert result.result.command_id == "ai_user-action" + + +class TestArgumentValueValidation: + WAIT = { + "mandatory": ["duration"], + "element_type": "Action", + "parameters": { + "duration": { + "data_type": "INTEGER", + "data_sources": ("CONSTANT", "VARIABLE", "DATATABLE"), + "minimum": 0.0, + "maximum": 3600.0, + }, + }, + } + CHECKPOINT = { + "mandatory": ["content"], + "optional": ["context"], + "element_type": "Validation", + "parameters": { + "content": {"data_type": "STRING"}, + "context": {"data_type": "STRING", "allowed_values": ("all", "body", "lowerPanel")}, + }, + } + + def test_add_command_rejects_value_out_of_range( + self, perfecto_token, monkeypatch, declare_commands): + declare_commands({"wait": self.WAIT}) + _mock_load_and_mutate(monkeypatch) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_command(TEST_ID, "wait", cmd_arguments={"duration": 5000})) + + assert "'duration' must be within 0..3600" in result.error + + def test_add_command_rejects_non_numeric_value( + self, perfecto_token, monkeypatch, declare_commands): + declare_commands({"wait": self.WAIT}) + _mock_load_and_mutate(monkeypatch) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_command(TEST_ID, "wait", cmd_arguments={"duration": "soon"})) + + assert "expects a number" in result.error + + def test_add_command_stringifies_integer_value( + self, perfecto_token, monkeypatch, declare_commands): + captured: dict = {} + declare_commands({"wait": self.WAIT}) + _mock_load_and_mutate(monkeypatch, captured=captured) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_command(TEST_ID, "wait", cmd_arguments={"duration": 30})) + + assert result.error is None + argument = captured["script"]["flowElements"][0]["arguments"][0] + assert argument["data"]["value"] == "30" + + def test_add_command_rejects_undeclared_data_source( + self, perfecto_token, monkeypatch, declare_commands): + declare_commands({"checkpoint_text": self.CHECKPOINT}) + _mock_load_and_mutate(monkeypatch) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_command( + TEST_ID, + "checkpoint_text", + cmd_arguments={"content": {"data_source": "DATATABLE", "table_name": "T", "column": "c"}}, + )) + + assert "does not accept data_source DATATABLE" in result.error + + def test_modify_command_rejects_undeclared_enumeration_value( + self, perfecto_token, monkeypatch, declare_commands): + declare_commands({"checkpoint_text": self.CHECKPOINT}) + _mock_load_and_mutate(monkeypatch, _script_with_steps("checkpoint_text")) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.modify_command(TEST_ID, "0", {"context": "sidebar"})) + + assert "must be one of all, body, lowerPanel" in result.error + + def test_modify_command_snaps_enumeration_casing( + self, perfecto_token, monkeypatch, declare_commands): + captured: dict = {} + declare_commands({"checkpoint_text": self.CHECKPOINT}) + _mock_load_and_mutate(monkeypatch, _script_with_steps("checkpoint_text"), captured) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.modify_command(TEST_ID, "0", {"context": "BODY"})) + + assert result.error is None + arguments = captured["script"]["flowElements"][0]["arguments"] + context = next(a for a in arguments if a["name"] == "context") + assert context["data"]["value"] == "body" + + def test_validation_is_skipped_without_definitions(self, perfecto_token, monkeypatch): + captured: dict = {} + _mock_load_and_mutate(monkeypatch, captured=captured) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_command(TEST_ID, "wait", cmd_arguments={"duration": 5000})) + + assert result.error is None + assert captured["script"]["flowElements"][0]["arguments"][0]["data"]["value"] == 5000 + + +class TestVariableBindingValidation: + WAIT = { + "mandatory": ["duration"], + "element_type": "Action", + "parameters": { + "duration": { + "data_type": "INTEGER", + "data_sources": ("CONSTANT", "VARIABLE", "DATATABLE"), + "minimum": 0.0, + "maximum": 3600.0, + }, + }, + } + + @staticmethod + def _script_with_variables(*steps: str) -> dict: + script = _script_with_steps(*steps) if steps else new_empty_script() + add_script_variable(script, "waitSecs", "string", "3") + add_script_variable(script, "waitSecsNum", "number", 3) + return script + + def test_add_command_rejects_variable_of_another_type( + self, perfecto_token, monkeypatch, declare_commands): + declare_commands({"wait": self.WAIT}) + _mock_load_and_mutate(monkeypatch, self._script_with_variables()) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_command( + TEST_ID, "wait", + cmd_arguments={"duration": {"data_source": "VARIABLE", "value": "waitSecs"}}, + )) + + assert "declared INTEGER but variable 'waitSecs' is a string" in result.error + + def test_add_command_rejects_undefined_variable( + self, perfecto_token, monkeypatch, declare_commands): + declare_commands({"wait": self.WAIT}) + _mock_load_and_mutate(monkeypatch, self._script_with_variables()) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_command( + TEST_ID, "wait", + cmd_arguments={"duration": {"data_source": "VARIABLE", "value": "typo"}}, + )) + + assert "which this test does not define" in result.error + + def test_add_command_accepts_variable_of_matching_type( + self, perfecto_token, monkeypatch, declare_commands): + captured: dict = {} + declare_commands({"wait": self.WAIT}) + _mock_load_and_mutate(monkeypatch, self._script_with_variables(), captured) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_command( + TEST_ID, "wait", + cmd_arguments={"duration": {"data_source": "VARIABLE", "value": "waitSecsNum"}}, + )) + + assert result.error is None + argument = captured["script"]["flowElements"][0]["arguments"][0] + assert argument["data"] == { + "@type": "VariableArgumentData", + "dataSource": "VARIABLE", + "value": "waitSecsNum", + } + + def test_modify_command_rejects_variable_of_another_type( + self, perfecto_token, monkeypatch, declare_commands): + declare_commands({"wait": self.WAIT}) + _mock_load_and_mutate(monkeypatch, self._script_with_variables("wait")) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.modify_command( + TEST_ID, "0", {"duration": {"data_source": "VARIABLE", "value": "waitSecs"}}, + )) + + assert "declared INTEGER but variable 'waitSecs' is a string" in result.error + + def test_modify_command_accepts_variable_of_matching_type( + self, perfecto_token, monkeypatch, declare_commands): + captured: dict = {} + declare_commands({"wait": self.WAIT}) + _mock_load_and_mutate(monkeypatch, self._script_with_variables("wait"), captured) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.modify_command( + TEST_ID, "0", {"duration": {"data_source": "VARIABLE", "value": "waitSecsNum"}}, + )) + + assert result.error is None + argument = captured["script"]["flowElements"][0]["arguments"][0] + assert argument["data"]["dataSource"] == "VARIABLE" + assert argument["data"]["value"] == "waitSecsNum" + + def test_spec_default_dut_binding_is_not_flagged( + self, perfecto_token, monkeypatch, declare_commands): + # The spec injects handsetId -> DUT; that default must not be rejected. + declare_commands({"ai_user-action": { + "mandatory": ["handsetId", "action"], + "element_type": "Action", + "parameters": { + "handsetId": {"data_type": "HANDSET", "data_sources": ("CONSTANT", "VARIABLE", "DATATABLE")}, + "action": {"data_type": "STRING"}, + }, + }}) + _mock_load_and_mutate(monkeypatch, self._script_with_variables()) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_command( + TEST_ID, "ai_user-action", cmd_arguments={"action": "Tap Login"}, + )) + + assert result.error is None + + +class TestSecuredVariableEncryption: + """The UI encrypts a secured value through the server before storing it.""" + + @staticmethod + def _mock_encrypt(monkeypatch, captured: dict, ciphertext="CIPHER=="): + async def fake_api_request(_token, method, endpoint=None, **_kwargs): + captured["endpoint"] = endpoint + captured["method"] = method + return BaseResult(result=ciphertext) + + monkeypatch.setattr(ai_scriptless_manager, "api_request", fake_api_request) + + def test_add_encrypts_before_storing(self, perfecto_token, monkeypatch): + captured: dict = {} + script_captured: dict = {} + self._mock_encrypt(monkeypatch, captured) + _mock_load_and_mutate(monkeypatch, captured=script_captured) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_test_variable( + TEST_ID, "secret", "secured_string", "p4ssw0rd", + )) + + assert result.error is None + assert "/script/variable/encrypt?value=p4ssw0rd" in captured["endpoint"] + stored = script_captured["script"]["variables"][0]["data"] + assert stored["value"] == "CIPHER==" + assert stored["secured"] is True + + def test_add_never_echoes_the_secret(self, perfecto_token, monkeypatch): + self._mock_encrypt(monkeypatch, {}) + _mock_load_and_mutate(monkeypatch) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_test_variable( + TEST_ID, "secret", "secured_string", "p4ssw0rd", + )) + + assert result.result["value"] == "" + assert "p4ssw0rd" not in json.dumps(result.result) + assert "CIPHER" not in json.dumps(result.result) + + def test_add_url_encodes_the_plaintext(self, perfecto_token, monkeypatch): + captured: dict = {} + self._mock_encrypt(monkeypatch, captured) + _mock_load_and_mutate(monkeypatch) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + asyncio.run(manager.add_test_variable(TEST_ID, "secret", "secured_string", "a b&c=d")) + + assert "value=a%20b%26c%3Dd" in captured["endpoint"] + + def test_encryption_failure_aborts_the_write(self, perfecto_token, monkeypatch): + async def failing_api_request(*_args, **_kwargs): + return BaseResult(error="Invalid credentials") + + monkeypatch.setattr(ai_scriptless_manager, "api_request", failing_api_request) + captured: dict = {} + _mock_load_and_mutate(monkeypatch, captured=captured) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_test_variable( + TEST_ID, "secret", "secured_string", "p4ssw0rd", + )) + + assert "Could not encrypt the secured value" in result.error + assert captured == {} + + def test_plain_types_do_not_call_the_endpoint(self, perfecto_token, monkeypatch): + captured: dict = {} + self._mock_encrypt(monkeypatch, captured) + _mock_load_and_mutate(monkeypatch) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + asyncio.run(manager.add_test_variable(TEST_ID, "plain", "string", "visible")) + + assert captured == {} + + def test_modify_encrypts_too(self, perfecto_token, monkeypatch): + captured: dict = {} + script_captured: dict = {} + self._mock_encrypt(monkeypatch, captured) + script = new_empty_script() + add_script_variable(script, "secret", "secured_string", "old") + _mock_load_and_mutate(monkeypatch, script, script_captured) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.modify_test_variable( + TEST_ID, "secret", value="rotated", variable_type="secured_string", + )) + + assert result.error is None + assert "value=rotated" in captured["endpoint"] + assert script_captured["script"]["variables"][0]["data"]["value"] == "CIPHER==" + + +class TestConditionStatementReporting: + @staticmethod + def _payload_with_condition(catch: bool) -> dict: + script = new_empty_script() + checkpoint = build_flow_element("checkpoint_text", {"content": "Settings"}) + if catch: + checkpoint["errorPolicy"] = "CATCH" + script["flowElements"] = [checkpoint, build_if_statement("Gate")] + return {"script": script, "commandDefinitions": []} + + def _mock_fetch(self, monkeypatch, catch: bool): + payload = self._payload_with_condition(catch) + + async def fake_fetch(_token, _test_id): + return BaseResult(result=copy.deepcopy(payload)) + + monkeypatch.setattr(ai_scriptless_manager, "fetch_script_payload", fake_fetch) + + def test_reports_the_statement_step(self, perfecto_token, monkeypatch): + self._mock_fetch(monkeypatch, catch=True) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.view_test_step(TEST_ID, "1")) + + assert result.result.type == "IfStatement" + assert result.result.statement_step_path == "0" + assert not any("no expression" in note for note in result.result.notes) + + def test_warns_when_the_condition_has_no_statement(self, perfecto_token, monkeypatch): + self._mock_fetch(monkeypatch, catch=False) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.view_test_step(TEST_ID, "1")) + + assert result.result.statement_step_path is None + assert any("no expression" in note for note in result.result.notes) + + +class TestVariableDrivenLoop: + @staticmethod + def _mock_fetch(monkeypatch): + script = new_empty_script() + add_script_variable(script, "iterations", "number", 3) + add_script_variable(script, "label", "string", "x") + + async def fake_fetch(_token, _test_id): + return BaseResult(result={"script": copy.deepcopy(script)}) + + monkeypatch.setattr(ai_scriptless_manager, "fetch_script_payload", fake_fetch) + + def test_adds_a_loop_driven_by_a_number_variable(self, perfecto_token, monkeypatch): + captured: dict = {} + self._mock_fetch(monkeypatch) + _mock_load_and_mutate(monkeypatch, captured=captured) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_loop(TEST_ID, variable="iterations")) + + assert result.error is None + assert result.result["variable"] == "iterations" + assert captured["script"]["flowElements"][0]["iterator"] == { + "@type": "VariableIterator", "variable": "iterations", + } + + def test_rejects_a_variable_of_another_type(self, perfecto_token, monkeypatch): + self._mock_fetch(monkeypatch) + _mock_load_and_mutate(monkeypatch) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_loop(TEST_ID, variable="label")) + + assert "a loop counts with a number variable; 'label' is a string" in result.error + + def test_rejects_an_undefined_variable(self, perfecto_token, monkeypatch): + self._mock_fetch(monkeypatch) + _mock_load_and_mutate(monkeypatch) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_loop(TEST_ID, variable="nope")) + + assert "is not defined on this test" in result.error + + def test_count_mode_is_unaffected(self, perfecto_token, monkeypatch): + captured: dict = {} + _mock_load_and_mutate(monkeypatch, captured=captured) + + manager = AiScriptlessManager(perfecto_token, ctx=None) + result = asyncio.run(manager.add_loop(TEST_ID, count=4)) + + assert result.result["count"] == 4 + assert captured["script"]["flowElements"][0]["iterator"]["count"] == 4 diff --git a/tests/test_ai_scriptless_script.py b/tests/test_ai_scriptless_script.py index d19fa55..6fc141c 100644 --- a/tests/test_ai_scriptless_script.py +++ b/tests/test_ai_scriptless_script.py @@ -19,6 +19,7 @@ import pytest from formatters.ai_scriptless import PRIMARY_AI_COMMAND_IDS +from tools.ai_scriptless.definitions import CommandContract from tools.ai_scriptless.elements import build_branch from tools.ai_scriptless_script import ( add_script_variable, @@ -37,11 +38,14 @@ format_test_ui_location, folder_type, insert_flow_element, + list_script_variables, + locate_variable, modify_script_variable, move_element_by_path, new_empty_script, parse_command_id, - set_condition_expression, + find_condition_statement, + set_element_error_policy, set_element_enabled, split_item_key, item_key_file_name, @@ -57,7 +61,7 @@ def _sample_script() -> dict: wait = build_flow_element("wait", {"duration": "2"}) group = build_logical_step("Setup") group["flowElements"] = [build_flow_element("comment", {"text": "inside group"})] - condition = build_if_statement("x == 1", "Check x") + condition = build_if_statement("Check x") then_branch = condition["branches"][0] then_branch["flowElements"] = [build_flow_element("ai_validation", {"validation": "OK"})] script = new_empty_script() @@ -147,6 +151,19 @@ def test_build_flow_element_ai_validation(self): assert element["@type"] == "Validation" assert element["errorPolicy"] == "IGNORE" + def test_build_flow_element_prefers_declared_contract(self): + # A command the local spec cannot recognize as a validation by name. + contract = CommandContract("verify_something", element_type="Validation", error_policy="IGNORE") + element = build_flow_element("verify_something", None, contract) + assert element["@type"] == "Validation" + assert element["errorPolicy"] == "IGNORE" + + def test_build_flow_element_falls_back_when_contract_is_silent(self): + contract = CommandContract("wait", element_type=None, error_policy=None) + element = build_flow_element("wait", {"duration": "3"}, contract) + assert element["@type"] == "Action" + assert element["errorPolicy"] == "ABORT" + def test_build_flow_element_default_handset_argument(self): element = build_flow_element("touch_tap") handset_args = [arg for arg in element["arguments"] if arg["name"] == "handsetId"] @@ -227,9 +244,11 @@ def test_new_empty_script_has_dut_parameter(self): assert dut["@type"] == "HandsetData" def test_build_logical_step(self): + # The UI keeps the group title in `name` (its "Name" parameter), not in `label`. step = build_logical_step("Group A") assert step["@type"] == "LogicalStep" - assert step["label"] == "Group A" + assert step["name"] == "Group A" + assert "label" not in step assert step["flowElements"] == [] def test_build_loop(self): @@ -238,15 +257,15 @@ def test_build_loop(self): assert loop["iterator"]["count"] == 3 def test_build_if_statement_has_branches(self): - condition = build_if_statement("flag", "Flag check") + condition = build_if_statement("Flag check") assert condition["@type"] == "IfStatement" - assert condition["expression"] == "flag" + assert condition["label"] == "Flag check" assert len(condition["branches"]) == 2 assert condition["branches"][0]["clause"] == "THEN" assert condition["branches"][1]["clause"] == "ELSE" def test_build_if_statement_clauses_alias_branches(self): - condition = build_if_statement("flag", "Flag check") + condition = build_if_statement("Flag check") assert condition["thenClause"] is condition["branches"][0] assert condition["elseClause"] is condition["branches"][1] child = build_flow_element("comment", {"text": "in then"}) @@ -321,11 +340,26 @@ def test_modify_script_variable(self): modify_script_variable(script, "flag", value=False) assert script["variables"][0]["data"]["value"] is False - def test_modify_script_variable_set_at_runtime(self): + def test_modify_script_variable_set_at_runtime_moves_it_to_parameters(self): + # "Set at runtime" decides the array: parameters[] for a Parameter, variables[] otherwise. script = new_empty_script() add_script_variable(script, "env", "string", "dev") + assert [v["data"]["name"] for v in script["variables"]] == ["env"] + modify_script_variable(script, "env", set_at_runtime=True) - assert script["variables"][0]["@type"] == "Parameter" + assert script["variables"] == [] + moved = script["parameters"][-1] + assert moved["@type"] == "Parameter" + assert moved["data"]["name"] == "env" + + def test_modify_script_variable_unset_at_runtime_moves_it_back(self): + script = new_empty_script() + add_script_variable(script, "env", "string", "dev", set_at_runtime=True) + assert [p["data"]["name"] for p in script["parameters"]] == ["DUT", "env"] + + modify_script_variable(script, "env", set_at_runtime=False) + assert [p["data"]["name"] for p in script["parameters"]] == ["DUT"] + assert script["variables"][0]["@type"] == "Variable" def test_delete_script_variable(self): script = new_empty_script() @@ -463,16 +497,45 @@ def test_set_element_enabled(self): _, _, element = find_element_by_path(script, "0") assert element["active"] is False - def test_set_condition_expression(self): + def test_set_element_enabled_marks_status_disabled(self): + # The UI writes both fields; active alone leaves a stale status. + script = _sample_script() + set_element_enabled(script, "0", False) + _, _, element = find_element_by_path(script, "0") + assert (element["active"], element["status"]) == (False, "DISABLED") + set_element_enabled(script, "0", True) + _, _, element = find_element_by_path(script, "0") + assert (element["active"], element["status"]) == (True, None) + + def test_set_element_error_policy(self): script = _sample_script() - set_condition_expression(script, "3", "y > 0") - _, _, element = find_element_by_path(script, "3") - assert element["expression"] == "y > 0" + set_element_error_policy(script, "0", "CATCH") + _, _, element = find_element_by_path(script, "0") + assert element["errorPolicy"] == "CATCH" - def test_set_condition_expression_rejects_non_if(self): + def test_set_element_error_policy_rejects_a_container(self): script = _sample_script() - with pytest.raises(ValueError, match="must reference an IfStatement"): - set_condition_expression(script, "0", "x") + with pytest.raises(ValueError, match="not a container"): + set_element_error_policy(script, "1", "CATCH") + + def test_find_condition_statement(self): + # A condition is fed by the preceding sibling marked CATCH. + script = _sample_script() + assert find_condition_statement(script, "3") is None + set_element_error_policy(script, "2", "CATCH") + assert find_condition_statement(script, "3") == "2" + + def test_find_condition_statement_ignores_other_policies(self): + script = _sample_script() + set_element_error_policy(script, "2", "IGNORE") + assert find_condition_statement(script, "3") is None + + def test_find_condition_statement_requires_a_condition(self): + script = _sample_script() + assert find_condition_statement(script, "0") is None + + def test_build_if_statement_carries_no_expression(self): + assert "expression" not in build_if_statement("Gate") def test_move_element_within_root(self): script = _sample_script() @@ -510,3 +573,251 @@ def test_strip_does_not_mutate_unrelated_fields(self): strip_non_api_script_fields(script) script["flowElements"][0].pop("uuid", None) assert script["flowElements"][0] == original["flowElements"][0] + + +class TestWireFormatMatchesUi: + """Shapes captured from UI-authored scripts (GET /native-automation/script).""" + + def test_constant_argument_shape(self): + element = build_flow_element("wait", {"duration": "2"}) + data = element["arguments"][0]["data"] + assert data == { + "@type": "ConstantArgumentData", + "dataSource": "CONSTANT", + "secured": False, + "value": "2", + } + + def test_variable_argument_shape(self): + element = build_flow_element("ai_user-action", {"action": "Tap"}) + handset = next(a for a in element["arguments"] if a["name"] == "handsetId") + assert handset["data"] == { + "@type": "VariableArgumentData", + "dataSource": "VARIABLE", + "value": "DUT", + } + + def test_datatable_argument_shape(self): + element = build_flow_element("wait", { + "duration": {"data_source": "DATATABLE", "table_name": "Data", "column": "secs"}, + }) + data = element["arguments"][0]["data"] + assert data == { + "@type": "DataTableArgumentData", + "dataSource": "DATATABLE", + "tableName": "Data", + "column": "secs", + } + # A DataTable binding carries no value field. + assert "value" not in data + + def test_datatable_binding_without_column(self): + element = build_flow_element("wait", {"duration": {"data_source": "DATATABLE"}}) + assert element["arguments"][0]["data"]["tableName"] is None + assert element["arguments"][0]["data"]["column"] is None + + def test_modify_switches_argument_to_datatable(self): + element = build_flow_element("wait", {"duration": "2"}) + update_element_arguments(element, { + "duration": {"data_source": "DATATABLE", "table_name": "T", "column": "c"}, + }) + data = element["arguments"][0]["data"] + assert data["@type"] == "DataTableArgumentData" + assert (data["tableName"], data["column"]) == ("T", "c") + + def test_action_element_carries_validations_list(self): + element = build_flow_element("wait") + assert element["validations"] == [] + + def test_validation_element_omits_validations_list(self): + # UI-authored Validation steps have no validations[] key. + element = build_flow_element("ai_validation", {"validation": "OK"}) + assert element["@type"] == "Validation" + assert "validations" not in element + + +class TestVariableArrays: + """Runtime parameters live in parameters[], plain variables in variables[].""" + + def test_add_runtime_parameter_lands_in_parameters(self): + script = new_empty_script() + add_script_variable(script, "env", "string", "dev", set_at_runtime=True) + assert script["variables"] == [] + assert script["parameters"][-1] == { + "@type": "Parameter", + "data": { + "@type": "StringData", + "description": None, + "displayName": None, + "name": "env", + "secured": False, + "value": "dev", + }, + } + + def test_duplicate_name_is_rejected_across_both_arrays(self): + script = new_empty_script() + add_script_variable(script, "env", "string", "dev", set_at_runtime=True) + with pytest.raises(ValueError, match="variable already exists: env"): + add_script_variable(script, "env", "string", "other") + + add_script_variable(script, "other", "string", "x") + with pytest.raises(ValueError, match="variable already exists: other"): + add_script_variable(script, "other", "string", "y", set_at_runtime=True) + + def test_list_covers_both_arrays_with_parameters_first(self): + script = new_empty_script() + add_script_variable(script, "plain", "string", "x") + add_script_variable(script, "runtime", "string", "y", set_at_runtime=True) + assert [v["data"]["name"] for v in list_script_variables(script)] == [ + "DUT", "runtime", "plain", + ] + + def test_locate_reports_the_array_it_found(self): + script = new_empty_script() + add_script_variable(script, "plain", "string", "x") + add_script_variable(script, "runtime", "string", "y", set_at_runtime=True) + assert locate_variable(script, "plain")[0] == "variables" + assert locate_variable(script, "runtime")[0] == "parameters" + assert locate_variable(script, "DUT")[0] == "parameters" + assert locate_variable(script, "missing") is None + + def test_modify_finds_a_runtime_parameter(self): + script = new_empty_script() + add_script_variable(script, "runtime", "string", "y", set_at_runtime=True) + modify_script_variable(script, "runtime", value="z") + assert script["parameters"][-1]["data"]["value"] == "z" + + def test_delete_finds_a_runtime_parameter(self): + script = new_empty_script() + add_script_variable(script, "runtime", "string", "y", set_at_runtime=True) + delete_script_variable(script, "runtime") + assert [p["data"]["name"] for p in script["parameters"]] == ["DUT"] + + def test_dut_is_listed_but_protected(self): + script = new_empty_script() + assert [v["data"]["name"] for v in list_script_variables(script)] == ["DUT"] + with pytest.raises(ValueError, match="cannot be modified"): + modify_script_variable(script, "DUT", value="DEVICE-1") + with pytest.raises(ValueError, match="breaks the test"): + delete_script_variable(script, "DUT") + + def test_unsupported_existing_type_is_reported(self): + # A media/datatable variable cannot be edited through the four scalar types. + script = new_empty_script() + script["variables"] = [{ + "@type": "Variable", + "data": {"@type": "MediaData", "name": "clip", "value": None, "secured": False}, + }] + with pytest.raises(ValueError, match="Unsupported variable type: media"): + modify_script_variable(script, "clip", value="x") + + +class TestMultivaluedArguments: + """A multivalued parameter persists as repeated FunctionArguments sharing the name.""" + + CONCAT = CommandContract( + "text_concat", + mandatory=frozenset({"variable", "value"}), + element_type="Action", + error_policy="CONTINUE", + ) + + def test_list_becomes_one_argument_per_occurrence_in_order(self): + # Shape captured from a UI-authored text_concat step. + element = build_flow_element("text_concat", { + "variable": {"data_source": "VARIABLE", "value": "result"}, + "value": ["Hello", {"data_source": "VARIABLE", "value": "constStr"}], + }, self.CONCAT) + arguments = [(a["name"], a["data"]["dataSource"], a["data"].get("value")) + for a in element["arguments"]] + assert arguments == [ + ("variable", "VARIABLE", "result"), + ("value", "CONSTANT", "Hello"), + ("value", "VARIABLE", "constStr"), + ] + + def test_undeclared_spec_default_is_not_injected(self): + # text_concat declares no handsetId; the fallback spec must not add one. + element = build_flow_element("text_concat", {"value": ["a", "b"]}, self.CONCAT) + assert all(a["name"] != "handsetId" for a in element["arguments"]) + + def test_spec_default_survives_without_a_contract(self): + element = build_flow_element("text_concat", {"value": ["a", "b"]}) + assert any(a["name"] == "handsetId" for a in element["arguments"]) + + def test_modify_keeps_untouched_occurrences(self): + # Regression: a name-keyed dict dropped every occurrence but the last. + element = build_flow_element("text_concat", { + "variable": {"data_source": "VARIABLE", "value": "result"}, + "value": ["Hello", "World"], + }) + update_element_arguments(element, {"variable": {"data_source": "VARIABLE", "value": "other"}}) + values = [a["data"]["value"] for a in element["arguments"] if a["name"] == "value"] + assert values == ["Hello", "World"] + result = next(a for a in element["arguments"] if a["name"] == "variable") + assert result["data"]["value"] == "other" + + def test_modify_replaces_the_whole_list(self): + element = build_flow_element("text_concat", {"value": ["Hello", "World"]}) + update_element_arguments(element, {"value": ["Bye"]}) + values = [a["data"]["value"] for a in element["arguments"] if a["name"] == "value"] + assert values == ["Bye"] + + def test_modify_preserves_argument_order(self): + element = build_flow_element("text_concat", { + "variable": {"data_source": "VARIABLE", "value": "result"}, + "value": ["a", "b"], + }) + update_element_arguments(element, {"value": ["c", "d"]}) + assert [a["name"] for a in element["arguments"]] == [ + "handsetId", "variable", "value", "value", + ] + + +class TestLoopIterators: + """Shapes captured from UI-authored loops.""" + + def test_repeat_iterator(self): + assert build_loop(3)["iterator"] == {"@type": "RepeatIterator", "count": 3} + + def test_variable_iterator(self): + loop = build_loop(variable="waitSecsNum") + assert loop["iterator"] == {"@type": "VariableIterator", "variable": "waitSecsNum"} + + def test_variable_wins_over_count(self): + loop = build_loop(5, variable="n") + assert loop["iterator"]["@type"] == "VariableIterator" + assert "count" not in loop["iterator"] + + +class TestBooleanNullState: + """A boolean variable has three states in the UI: Null, True and False.""" + + def test_none_is_stored_as_null(self): + script = new_empty_script() + add_script_variable(script, "flag", "boolean", None) + assert script["variables"][0]["data"]["value"] is None + + def test_the_null_literal_is_accepted(self): + script = new_empty_script() + add_script_variable(script, "flag", "boolean", "Null") + assert script["variables"][0]["data"]["value"] is None + + def test_true_and_false_still_work(self): + script = new_empty_script() + add_script_variable(script, "yes", "boolean", "true") + add_script_variable(script, "no", "boolean", False) + values = [v["data"]["value"] for v in script["variables"]] + assert values == [True, False] + + def test_a_non_boolean_is_still_rejected(self): + script = new_empty_script() + with pytest.raises(ValueError, match="true, false or null"): + add_script_variable(script, "flag", "boolean", "maybe") + + def test_modify_can_clear_a_boolean_to_null(self): + script = new_empty_script() + add_script_variable(script, "flag", "boolean", True) + modify_script_variable(script, "flag", value="null") + assert script["variables"][0]["data"]["value"] is None diff --git a/tests/test_ai_scriptless_script_aggregate.py b/tests/test_ai_scriptless_script_aggregate.py index b4d08df..c61a818 100644 --- a/tests/test_ai_scriptless_script_aggregate.py +++ b/tests/test_ai_scriptless_script_aggregate.py @@ -78,7 +78,8 @@ class TestScriptVariables: def test_add_list_modify_delete_variable(self): script = Script.empty() script.add_variable("token", "string", "abc") - assert len(script.list_variables()) == 1 + # The listing covers both arrays, so DUT (a runtime parameter) is included. + assert [v["data"]["name"] for v in script.list_variables()] == ["DUT", "token"] script.modify_variable("token", value="xyz") assert script.find_variable("token")[1]["data"]["value"] == "xyz" script.delete_variable("token") diff --git a/tools/ai_scriptless/__init__.py b/tools/ai_scriptless/__init__.py index 6c30ffc..205793d 100644 --- a/tools/ai_scriptless/__init__.py +++ b/tools/ai_scriptless/__init__.py @@ -6,10 +6,17 @@ parse_command_id, ) from tools.ai_scriptless.definitions import ( - declared_parameters, + CommandContract, + ParameterContract, + coerce_argument_value, + command_contract, empty_mandatory_note, - reset_declared_parameters_cache, + reset_command_contract_cache, + restriction_allowed_values, + restriction_range, validate_argument_names, + validate_argument_values, + validate_variable_bindings, ) from tools.ai_scriptless.elements import ( build_arguments, @@ -19,6 +26,7 @@ build_logical_step, build_loop, new_empty_script, + normalize_if_statement_aliases, strip_non_api_script_fields, update_element_arguments, ) @@ -54,20 +62,29 @@ find_step_path_for_element, insert_flow_element, move_element_by_path, - set_condition_expression, + find_condition_statement, + set_element_error_policy, set_element_enabled, update_flow_element_counts, validate_step_path, ) from tools.ai_scriptless.variables import ( SUPPORTED_VARIABLE_TYPES, + VARIABLE_DATA_TYPES_BY_PARAMETER_TYPE, + bindable_values, + describe_bindable_values, + variable_type_label, VARIABLE_TYPE_ALIASES, add_script_variable, build_variable_data, build_variable_entry, delete_script_variable, find_variable, + DUT_NAME, + VARIABLE_ARRAYS, list_script_variables, + locate_variable, + variable_array_for, modify_script_variable, validate_variable_name, ) @@ -81,7 +98,11 @@ "Script", "ScriptInput", "SUPPORTED_VARIABLE_TYPES", + "VARIABLE_DATA_TYPES_BY_PARAMETER_TYPE", "VARIABLE_TYPE_ALIASES", + "bindable_values", + "describe_bindable_values", + "variable_type_label", "VISIBILITY_UI_ROOT", "_persist_script", "add_script_variable", @@ -100,7 +121,10 @@ "coerce_script_dict", "coerce_step_path", "command_id_from_element", - "declared_parameters", + "CommandContract", + "ParameterContract", + "coerce_argument_value", + "command_contract", "delete_element_by_path", "delete_script_variable", "empty_mandatory_note", @@ -114,22 +138,32 @@ "get_command_spec", "insert_flow_element", "item_key_file_name", + "DUT_NAME", + "VARIABLE_ARRAYS", "list_script_variables", + "locate_variable", + "variable_array_for", "load_and_mutate", "modify_script_variable", "move_element_by_path", "new_empty_script", + "normalize_if_statement_aliases", "persist_script", "parse_command_id", - "reset_declared_parameters_cache", + "reset_command_contract_cache", + "restriction_allowed_values", + "restriction_range", "script_write_lock", - "set_condition_expression", + "find_condition_statement", + "set_element_error_policy", "set_element_enabled", "split_item_key", "strip_non_api_script_fields", "update_element_arguments", "update_flow_element_counts", "validate_argument_names", + "validate_argument_values", + "validate_variable_bindings", "validate_step_path", "validate_variable_name", ] diff --git a/tools/ai_scriptless/commands.py b/tools/ai_scriptless/commands.py index df5ecfc..81763b1 100644 --- a/tools/ai_scriptless/commands.py +++ b/tools/ai_scriptless/commands.py @@ -72,9 +72,22 @@ def _command_spec( COMMAND_SPECS: dict[str, CommandSpec] = { spec.command_id: spec for spec in ( - _command_spec("ai_user-action", "Action", {**_HANDSET_DUT, "action": ("CONSTANT", "")}), + # The editor labels `action` "Prompt"; the value typed there is stored as `action`. + # Accepted as an alias so the name a user reads in the UI resolves to the parameter. + _command_spec( + "ai_user-action", + "Action", + {**_HANDSET_DUT, "action": ("CONSTANT", "")}, + argument_aliases={"Prompt": "action", "prompt": "action"}, + ), _command_spec("ai_validation", "Validation", {**_HANDSET_DUT, "validation": ("CONSTANT", "")}), - _command_spec("ai_visual-comparison", "Action", {**_HANDSET_DUT, "name": ("CONSTANT", "")}), + # The baseline parameter is baselineId; `name` was an early guess, kept as an alias. + _command_spec( + "ai_visual-comparison", + "Action", + {**_HANDSET_DUT, "baselineId": ("CONSTANT", "")}, + argument_aliases={"name": "baselineId"}, + ), _command_spec("comment", "Action", {"text": ("CONSTANT", "")}), _command_spec( "wait", diff --git a/tools/ai_scriptless/definitions.py b/tools/ai_scriptless/definitions.py index 0fc1f9a..a5e22e7 100644 --- a/tools/ai_scriptless/definitions.py +++ b/tools/ai_scriptless/definitions.py @@ -1,19 +1,25 @@ -"""Cross-check between command parameters (command repository API) and command -arguments (script model). - -The command repository declares ``mandatoryParameters`` / ``optionalParameters``; -the script persists those same names as ``FunctionArgument`` entries. The names -match, the vocabulary does not: a parameter is the declaration, an argument is -the assigned value. Perfecto silently ignores arguments whose name is not -declared, so an unknown name only shows up as a step that does nothing at -execution time. These helpers turn that into an error at authoring time. - -Validation fails open on purpose: when the definitions API is unreachable or -returns no declared parameters, authoring keeps working unvalidated rather than -becoming unavailable. +"""Command contracts from the command repository API. + +The repository declares what a command is (``data.type``, ``data.errorPolicy``) +and what it takes (``mandatoryParameters`` / ``optionalParameters``); the script +persists a flow element with an ``@type`` and a list of ``FunctionArgument`` +entries whose names are those same parameter names. Parameter is the +declaration, argument is the assigned value. + +Reading the contract instead of inferring it keeps two things honest: +- element type and error policy, which decide whether a failing step aborts the + test (Action/ABORT) or is only reported (Validation/IGNORE); +- argument names, which Perfecto silently ignores when undeclared, so a typo + only shows up as a step that does nothing at execution time. + +Everything fails open on purpose: when the definitions API is unreachable or +does not declare something, authoring keeps working with the local defaults +rather than becoming unavailable. """ import asyncio +import re +from dataclasses import dataclass, field from difflib import get_close_matches from typing import Any, Optional @@ -22,22 +28,223 @@ from tools.ai_scriptless.commands import get_command_spec from tools.utils import api_request -# (cloud_name, command_id) -> (mandatory, optional) names, or None when unknown. -DeclaredParameters = tuple[frozenset[str], frozenset[str]] +# data.type -> flow element @type in the script model. +ELEMENT_TYPE_BY_DEFINITION_TYPE = { + "ACTION": "Action", + "VALIDATION": "Validation", +} + +# The five policies the legacy IDE documents (and the UI's "On-fail Result" offers): +# IGNORE reports the failure and goes on, ABORT ends the run, BREAK and CONTINUE act on the +# enclosing loop (abort outside one), CATCH feeds the result to a following condition. +SUPPORTED_ERROR_POLICIES = frozenset({"ABORT", "IGNORE", "BREAK", "CONTINUE", "CATCH"}) + + +INTEGER_DATA_TYPES = frozenset({"INTEGER", "NUMBER"}) +BOOLEAN_TRUE = frozenset({"true", "1", "yes"}) +BOOLEAN_FALSE = frozenset({"false", "0", "no"}) + +# Enumerations whose persisted values are not exactly the declared restriction.value. +# +# For every dropdown-style enumeration the UI persists the declared value: setting +# checkpoint_text's "Match mode" to the label "Start with" stores ``startwith``. +# ai_visual-comparison's "Fail criteria" is the exception. It is edited through a +# multi-select dialog whose option list is hardcoded in the UI, and that list has +# drifted from the declaration in two places: the declared ``pixel_difference`` is +# stored as ``pixelDifference``, and ``moved`` is offered but declared nowhere. A +# value stored in the declared spelling is counted by the editor ("1 selected") but +# shows no ticked box, so it cannot be edited in the UI. +# +# Harvested by ticking every option in the editor, saving, and reading the persisted +# arguments back -- not from the script tree, which renders restriction.label rather +# than the stored value. +UI_PERSISTED_ENUM_VALUES: dict[tuple[str, str], tuple[str, ...]] = { + ("ai_visual-comparison", "failCriteria"): ( + "device", "style", "value", "missing", "moved", + "addition", "error", "uncategorized", "pixelDifference", + ), +} + + +# Parameters the editor labels differently from their declared display.name. +# +# The declaration is not wrong about the parameter, only about what the user reads: typing into +# the field labelled "Prompt" stores `action`, verified by matching the editor's row (whose +# data-aid carries the parameter name) against the definition. Reporting the declared label +# would name a field nobody can find on screen, so the editor's wins and the declared one is +# reported alongside it. +UI_PARAMETER_LABELS: dict[tuple[str, str], str] = { + ("ai_user-action", "action"): "Prompt", +} + + +def parameter_label( + command_id: Optional[str], + name: Optional[str], + declared_label: Optional[str], +) -> Optional[str]: + """The label the editor shows for this parameter, falling back to the declared one.""" + return UI_PARAMETER_LABELS.get((command_id or "", name or ""), declared_label) + + +def _enum_key(value: str) -> str: + """Punctuation- and case-insensitive key, so ``pixel_difference`` matches ``pixelDifference``.""" + return re.sub(r"[^0-9a-z]+", "", str(value).casefold()) + + +def restriction_allowed_values( + param: dict[str, Any], + command_id: Optional[str] = None, +) -> tuple[str, ...]: + """Persisted values of an ENUMERATION/COMBO parameter, in declared order. + + Reports the spelling Perfecto stores, which for the parameters listed in + UI_PERSISTED_ENUM_VALUES is the UI's display string rather than the declared one. + """ + restriction = param.get("restriction") or {} + if str(restriction.get("type") or "").upper() not in ("ENUMERATION", "COMBO"): + return () + name = param.get("name") or param.get("parameterName") or "" + persisted = UI_PERSISTED_ENUM_VALUES.get((command_id or "", name)) + if persisted: + return persisted + raw_values = restriction.get("value") or restriction.get("label") or "" + candidates = raw_values if isinstance(raw_values, list) else str(raw_values).split(",") + return tuple(str(value).strip() for value in candidates if str(value).strip()) + + +def restriction_range(param: dict[str, Any]) -> tuple[Optional[float], Optional[float]]: + """Numeric bounds of a RANGE parameter, or (None, None) when unrestricted.""" + restriction = param.get("restriction") or {} + if str(restriction.get("type") or "").upper() != "RANGE": + return None, None + value_range = restriction.get("range") or {} + return value_range.get("minValue"), value_range.get("maxValue") + + +@dataclass(frozen=True) +class ParameterContract: + """What the repository declares for one parameter of one command.""" + + name: str + data_type: Optional[str] = None + data_sources: frozenset[str] = frozenset() + mandatory: bool = False + default_value: Any = None + allowed_values: tuple[str, ...] = () + minimum: Optional[float] = None + maximum: Optional[float] = None + min_occurrences: int = 1 + max_occurrences: int = 1 + + @property + def is_multivalued(self) -> bool: + """True when the parameter takes a list: the UI edits it as ordered rows.""" + return self.max_occurrences > 1 + + @property + def is_integer(self) -> bool: + return (self.data_type or "").upper() in INTEGER_DATA_TYPES + + @property + def is_boolean(self) -> bool: + return (self.data_type or "").upper() == "BOOLEAN" -_declared_parameters_cache: dict[tuple[str, str], Optional[DeclaredParameters]] = {} + def match_allowed_value(self, value: str) -> Optional[str]: + """The persisted spelling of value, or None when it is not accepted. + + Falls back to a punctuation-insensitive comparison, which is what lets the declared + ``pixel_difference`` (and the label ``Pixel difference``) resolve to the + ``pixelDifference`` the UI stores. No declared enumeration relies on punctuation + alone to tell two of its own values apart. + """ + for allowed in self.allowed_values: + if allowed.casefold() == value.casefold(): + return allowed + key = _enum_key(value) + for allowed in self.allowed_values: + if _enum_key(allowed) == key: + return allowed + return None + + +@dataclass(frozen=True) +class CommandContract: + """What the repository declares for one command. Fields are None when undeclared.""" + + command_id: str + mandatory: frozenset[str] = frozenset() + optional: frozenset[str] = frozenset() + element_type: Optional[str] = None + error_policy: Optional[str] = None + parameters: dict[str, ParameterContract] = field(default_factory=dict) + + @property + def declared_names(self) -> frozenset[str]: + return self.mandatory | self.optional + + def parameter(self, name: str) -> Optional[ParameterContract]: + return self.parameters.get(name) + + +_command_contract_cache: dict[tuple[str, str], Optional[CommandContract]] = {} _cache_guard = asyncio.Lock() -def reset_declared_parameters_cache() -> None: - """Drop memoized definitions (used by tests and after a cloud switch).""" - _declared_parameters_cache.clear() +def reset_command_contract_cache() -> None: + """Drop memoized contracts (used by tests and after a cloud switch).""" + _command_contract_cache.clear() + + +def _parse_parameter_contract( + param: dict[str, Any], + mandatory: bool, + command_id: str = "", +) -> ParameterContract: + minimum, maximum = restriction_range(param) + return ParameterContract( + name=param.get("name") or param.get("parameterName") or "", + data_type=param.get("dataType"), + data_sources=frozenset(str(source).upper() for source in (param.get("dataSources") or [])), + mandatory=mandatory, + default_value=param.get("defaultValue"), + allowed_values=restriction_allowed_values(param, command_id), + minimum=minimum, + maximum=maximum, + min_occurrences=int(param.get("minOccurrences") or 1), + max_occurrences=int(param.get("maxOccurrences") or 1), + ) + +def _parse_command_contract(command_id: str, definition: dict[str, Any]) -> CommandContract: + data = definition.get("data", definition) if isinstance(definition, dict) else {} + element_type = ELEMENT_TYPE_BY_DEFINITION_TYPE.get(str(data.get("type") or "").upper()) + error_policy = str(data.get("errorPolicy") or "").upper() or None + if error_policy not in SUPPORTED_ERROR_POLICIES: + # wait, for one, declares errorPolicy null: leave it to the local default. + error_policy = None -async def _fetch_declared_parameters( + parameters: dict[str, ParameterContract] = {} + for bucket, mandatory in (("mandatoryParameters", True), ("optionalParameters", False)): + for param in data.get(bucket) or []: + if not isinstance(param, dict): + continue + contract = _parse_parameter_contract(param, mandatory, command_id) + if contract.name: + parameters[contract.name] = contract + + return CommandContract( + command_id=command_id, + element_type=element_type, + error_policy=error_policy, + parameters=parameters, + ) + + +async def _fetch_command_contract( token: PerfectoToken, command_id: str, -) -> Optional[DeclaredParameters]: +) -> Optional[CommandContract]: # Local import: formatters.ai_scriptless imports tools.ai_scriptless.elements. from formatters.ai_scriptless import format_command_definitions @@ -58,30 +265,33 @@ async def _fetch_declared_parameters( for definition in result.result: if definition.command_id != command_id: continue - mandatory = frozenset(definition.mandatory_parameters) - optional = frozenset(definition.optional_parameters) - # A definition with no declared parameter carries no usable contract. - if not mandatory and not optional: - return None - return mandatory, optional + contract = _parse_command_contract(command_id, definition.raw or {}) + return CommandContract( + command_id=command_id, + mandatory=frozenset(definition.mandatory_parameters), + optional=frozenset(definition.optional_parameters), + element_type=contract.element_type, + error_policy=contract.error_policy, + parameters=contract.parameters, + ) return None -async def declared_parameters( +async def command_contract( token: Optional[PerfectoToken], command_id: str, -) -> Optional[DeclaredParameters]: - """Declared parameter names for command_id, or None when unavailable.""" +) -> Optional[CommandContract]: + """Contract declared for command_id, or None when unavailable.""" if not token or not command_id: return None cache_key = (token.cloud_name, command_id) async with _cache_guard: - if cache_key in _declared_parameters_cache: - return _declared_parameters_cache[cache_key] - declared = await _fetch_declared_parameters(token, command_id) + if cache_key in _command_contract_cache: + return _command_contract_cache[cache_key] + contract = await _fetch_command_contract(token, command_id) async with _cache_guard: - _declared_parameters_cache[cache_key] = declared - return declared + _command_contract_cache[cache_key] = contract + return contract def _accepted_names(command_id: str, name: str) -> set[str]: @@ -94,6 +304,9 @@ def _accepted_names(command_id: str, name: str) -> set[str]: def _argument_value(value: Any) -> Any: if isinstance(value, dict) and "data_source" in value: + if str(value.get("data_source") or "").upper() == "DATATABLE": + # A DataTable binding has no value field; the table/column pair is the value. + return value.get("table_name") or value.get("tableName") or value.get("column") return value.get("value") return value @@ -101,13 +314,12 @@ def _argument_value(value: Any) -> Any: def validate_argument_names( command_id: str, cmd_arguments: Optional[dict[str, Any]], - declared: Optional[DeclaredParameters], + contract: Optional[CommandContract], ) -> Optional[str]: """Error message when cmd_arguments carries names the command does not declare.""" - if not cmd_arguments or not declared: + if not cmd_arguments or not contract or not contract.declared_names: return None - mandatory, optional = declared - known = mandatory | optional + known = contract.declared_names unknown = [name for name in cmd_arguments if not (_accepted_names(command_id, name) & known)] if not unknown: return None @@ -119,22 +331,214 @@ def validate_argument_names( return ( f"Unknown cmd_arguments for command '{command_id}': {', '.join(reported)}. " f"Declared parameter names: {', '.join(sorted(known))}" - f" (mandatory: {', '.join(sorted(mandatory)) or 'none'}). " + f" (mandatory: {', '.join(sorted(contract.mandatory)) or 'none'}). " "Keys of cmd_arguments are the parameter names from get_command_definitions; " "Perfecto ignores undeclared argument names instead of failing." ) +def _format_bound(bound: Optional[float]) -> str: + if isinstance(bound, float) and bound.is_integer(): + return str(int(bound)) + return str(bound) + + +def coerce_argument_value(value: Any, parameter: Optional[ParameterContract]) -> Any: + """Normalize a constant to the spelling Perfecto persists. + + Every constant observed in a UI-authored script is a string, including INTEGER + parameters ("2", not 2), so numbers are stringified. Enumerations are snapped to + their declared casing. + """ + if parameter is None: + return value + if isinstance(value, bool): + # Not confirmed against a UI-authored boolean argument; consistent with every + # other constant being a string. + return "true" if value else "false" + if isinstance(value, (int, float)) and parameter.is_integer: + return str(int(value)) if float(value).is_integer() else str(value) + if isinstance(value, str) and parameter.allowed_values: + return parameter.match_allowed_value(value) or value + return value + + +def _validate_argument_value( + name: str, + raw: Any, + parameter: ParameterContract, +) -> Optional[str]: + data_source = "CONSTANT" + value: Any = raw + if isinstance(raw, dict) and "data_source" in raw: + data_source = str(raw.get("data_source") or "").upper() + if parameter.data_sources and data_source not in parameter.data_sources: + return ( + f"'{name}' does not accept data_source {data_source}; " + f"accepted: {', '.join(sorted(parameter.data_sources))}." + ) + if data_source != "CONSTANT": + # Variable and DataTable bindings are resolved at execution time. + return None + value = raw.get("value") + + if value is None or isinstance(value, bool): + pass + elif parameter.is_integer: + try: + numeric = float(str(value).strip()) + except (TypeError, ValueError): + return f"'{name}' expects a number ({parameter.data_type}), got {value!r}." + if parameter.minimum is not None and numeric < parameter.minimum: + return ( + f"'{name}' must be within {_format_bound(parameter.minimum)}.." + f"{_format_bound(parameter.maximum)}, got {value!r}." + ) + if parameter.maximum is not None and numeric > parameter.maximum: + return ( + f"'{name}' must be within {_format_bound(parameter.minimum)}.." + f"{_format_bound(parameter.maximum)}, got {value!r}." + ) + + if parameter.is_boolean and not isinstance(value, bool) and value is not None: + if str(value).strip().casefold() not in (BOOLEAN_TRUE | BOOLEAN_FALSE): + return f"'{name}' expects a boolean, got {value!r}." + + if parameter.allowed_values and isinstance(value, str) and value.strip(): + if parameter.match_allowed_value(value) is None: + return ( + f"'{name}' must be one of {', '.join(parameter.allowed_values)}, got {value!r}." + ) + return None + + +def _validate_occurrences(name: str, count: int, parameter: ParameterContract) -> Optional[str]: + if count > parameter.max_occurrences: + return ( + f"'{name}' accepts at most {parameter.max_occurrences} value(s), got {count}." + ) + if count < parameter.min_occurrences: + return ( + f"'{name}' needs at least {parameter.min_occurrences} values; " + f"pass a list of {parameter.min_occurrences} or more." + ) + return None + + +def validate_argument_values( + command_id: str, + cmd_arguments: Optional[dict[str, Any]], + contract: Optional[CommandContract], +) -> Optional[str]: + """Error message when a value breaks what the parameter declares. + + Names are checked by validate_argument_names; this only looks at values, and only + for parameters the contract describes. + """ + if not cmd_arguments or not contract or not contract.parameters: + return None + spec = get_command_spec(command_id) + errors: list[str] = [] + for name, raw in spec.normalize_argument_names(cmd_arguments).items(): + parameter = contract.parameter(name) + if parameter is None: + continue + occurrences = raw if isinstance(raw, list) else [raw] + count_error = _validate_occurrences(name, len(occurrences), parameter) + if count_error: + errors.append(count_error) + continue + for occurrence in occurrences: + error = _validate_argument_value(name, occurrence, parameter) + if error: + errors.append(error) + if not errors: + return None + return ( + f"Invalid cmd_arguments for command '{command_id}': " + " ".join(errors) + + " See view_test_step or get_command_definitions for accepted values." + ) + + +def _variable_binding_error( + name: str, + variable_name: Any, + parameter: ParameterContract, + script: dict[str, Any], +) -> Optional[str]: + from tools.ai_scriptless.variables import ( + VARIABLE_DATA_TYPES_BY_PARAMETER_TYPE, + bindable_values, + describe_bindable_values, + variable_type_label, + ) + + if not variable_name or not isinstance(variable_name, str): + return ( + f"'{name}' is bound to a variable but no variable name was given; " + f"pass {{\"data_source\": \"VARIABLE\", \"value\": \"\"}}. " + f"Defined: {describe_bindable_values(script)}." + ) + + bindable = bindable_values(script) + data = bindable.get(variable_name) + if data is None: + return ( + f"'{name}' is bound to variable '{variable_name}', which this test does not define. " + f"Defined: {describe_bindable_values(script)}. " + "Add it with add_test_variable or bind an existing one." + ) + + expected = VARIABLE_DATA_TYPES_BY_PARAMETER_TYPE.get((parameter.data_type or "").upper()) + if expected and data.get("@type") not in expected: + return ( + f"'{name}' is declared {parameter.data_type} but variable '{variable_name}' is a " + f"{variable_type_label(data)}. Perfecto only binds a variable of the matching type; " + f"create one with add_test_variable or pick another. " + f"Defined: {describe_bindable_values(script)}." + ) + return None + + +def validate_variable_bindings( + command_id: str, + cmd_arguments: Optional[dict[str, Any]], + contract: Optional[CommandContract], + script: dict[str, Any], +) -> Optional[str]: + """Error message when an argument binds to a variable the script cannot provide. + + Needs the script, so this runs where the script is loaded: the variable must exist + and its type must match the parameter, the same rule the UI enforces by only + offering compatible variables in its picker. + """ + if not cmd_arguments or not contract or not contract.parameters: + return None + spec = get_command_spec(command_id) + errors: list[str] = [] + for name, raw in spec.normalize_argument_names(cmd_arguments).items(): + if not isinstance(raw, dict) or "data_source" not in raw: + continue + if str(raw.get("data_source") or "").upper() != "VARIABLE": + continue + parameter = contract.parameter(name) + if parameter is None: + continue + error = _variable_binding_error(name, raw.get("value"), parameter, script) + if error: + errors.append(error) + if not errors: + return None + return f"Invalid cmd_arguments for command '{command_id}': " + " ".join(errors) + + def empty_mandatory_note( command_id: str, cmd_arguments: Optional[dict[str, Any]], - declared: Optional[DeclaredParameters], + contract: Optional[CommandContract], ) -> Optional[str]: """Note when a mandatory parameter is left empty (the step persists but does nothing).""" - if not declared: - return None - mandatory, _optional = declared - if not mandatory: + if not contract or not contract.mandatory: return None spec = get_command_spec(command_id) @@ -145,7 +549,7 @@ def empty_mandatory_note( values[name] = _argument_value(value) empty = sorted( - name for name in mandatory + name for name in contract.mandatory if values.get(name) is None or (isinstance(values[name], str) and not values[name].strip()) ) if not empty: diff --git a/tools/ai_scriptless/elements.py b/tools/ai_scriptless/elements.py index e7e58b0..97476c5 100644 --- a/tools/ai_scriptless/elements.py +++ b/tools/ai_scriptless/elements.py @@ -1,4 +1,4 @@ -from typing import Any, Optional +from typing import TYPE_CHECKING, Any, Optional from tools.ai_scriptless.commands import ( command_id_from_element, @@ -6,6 +6,9 @@ parse_command_id, ) +if TYPE_CHECKING: # type-only: definitions reaches the API, elements must stay pure + from tools.ai_scriptless.definitions import CommandContract + def _make_argument(name: str, value: Any, data_source: str = "CONSTANT") -> dict[str, Any]: if data_source == "VARIABLE": @@ -14,6 +17,15 @@ def _make_argument(name: str, value: Any, data_source: str = "CONSTANT") -> dict "dataSource": "VARIABLE", "value": value, } + elif data_source == "DATATABLE": + # A DataTable binding carries tableName/column instead of a value. + binding = value if isinstance(value, dict) else {} + data = { + "@type": "DataTableArgumentData", + "dataSource": "DATATABLE", + "tableName": binding.get("tableName"), + "column": binding.get("column"), + } else: data = { "@type": "ConstantArgumentData", @@ -24,33 +36,99 @@ def _make_argument(name: str, value: Any, data_source: str = "CONSTANT") -> dict return {"@type": "FunctionArgument", "name": name, "data": data} -def build_arguments(command_id: str, arguments: Optional[dict[str, Any]]) -> list[dict[str, Any]]: +def _coerce(name: str, value: Any, contract: Optional["CommandContract"]) -> Any: + """Normalize a constant to the declared type; a no-op without a contract.""" + if contract is None: + return value + # Local import: definitions reaches the API, elements must stay importable without it. + from tools.ai_scriptless.definitions import coerce_argument_value + + return coerce_argument_value(value, contract.parameter(name)) + + +def _argument_payload(value: dict[str, Any]) -> Any: + """What travels with the data source: a DataTable binding, or a plain value.""" + if str(value.get("data_source") or "").upper() == "DATATABLE": + return { + "tableName": value.get("table_name", value.get("tableName")), + "column": value.get("column"), + } + return value.get("value") + + +def _argument_occurrences( + name: str, + value: Any, + contract: Optional["CommandContract"], +) -> list[tuple[str, Any]]: + """(data source, payload) per occurrence. + + A list means a multivalued parameter: Perfecto persists one FunctionArgument per + occurrence, all sharing the parameter name, in order. + """ + values = value if isinstance(value, list) else [value] + occurrences: list[tuple[str, Any]] = [] + for item in values: + if isinstance(item, dict) and "data_source" in item: + occurrences.append((item["data_source"], _argument_payload(item))) + else: + occurrences.append(("CONSTANT", _coerce(name, item, contract))) + return occurrences + + +def build_arguments( + command_id: str, + arguments: Optional[dict[str, Any]], + contract: Optional["CommandContract"] = None, +) -> list[dict[str, Any]]: spec = get_command_spec(command_id) - merged = spec.default_arguments_merged() + declared = contract.declared_names if contract else frozenset() + merged: dict[str, list[tuple[str, Any]]] = { + name: [(source, value)] + for name, (source, value) in spec.default_arguments_merged().items() + # The fallback spec seeds handsetId for every unregistered command; keep a default + # only when the command actually declares that parameter. + if not declared or name in declared + } if arguments: for name, value in spec.normalize_argument_names(arguments).items(): - if isinstance(value, dict) and "data_source" in value: - merged[name] = (value["data_source"], value.get("value")) - else: - merged[name] = ("CONSTANT", value) + merged[name] = _argument_occurrences(name, value, contract) spec.drop_superseded_aliases(merged) - return [_make_argument(name, value, source) for name, (source, value) in merged.items()] + return [ + _make_argument(name, value, source) + for name, occurrences in merged.items() + for source, value in occurrences + ] + +def build_flow_element( + command_id: str, + arguments: Optional[dict[str, Any]] = None, + contract: Optional["CommandContract"] = None, +) -> dict[str, Any]: + """Build a flow element, preferring the declared contract over the local spec. -def build_flow_element(command_id: str, arguments: Optional[dict[str, Any]] = None) -> dict[str, Any]: + element_type and errorPolicy decide whether a failing step aborts the test or is + only reported, so the repository declaration wins whenever it is available. + """ command, subcommand = parse_command_id(command_id) spec = get_command_spec(command_id) - return { - "@type": spec.element_type, - "validations": [], - "errorPolicy": spec.error_policy, + element_type = (contract.element_type if contract else None) or spec.element_type + error_policy = (contract.error_policy if contract else None) or spec.error_policy + element = { + "@type": element_type, + "errorPolicy": error_policy, "command": command, "subcommand": subcommand, - "arguments": build_arguments(command_id, arguments), + "arguments": build_arguments(command_id, arguments, contract), "comment": None, "status": None, "active": True, } + if element_type != "Validation": + # The UI writes validations[] on Action steps only; Validation steps omit it. + element["validations"] = [] + return element def build_branch(clause: str) -> dict[str, Any]: @@ -67,20 +145,27 @@ def build_branch(clause: str) -> dict[str, Any]: def build_logical_step(label: Optional[str] = None) -> dict[str, Any]: + # The group's title lives in `name` (the UI's "Name" parameter), not in `label`. return { "@type": "LogicalStep", "flowElements": [], "active": True, - "label": label or "", + "name": label or "", + "transaction": "", "comment": None, "status": None, } -def build_loop(count: int = 1) -> dict[str, Any]: +def build_loop(count: int = 1, variable: Optional[str] = None) -> dict[str, Any]: + """A loop repeats a fixed number of times, or as many times as a number variable says.""" + iterator = ( + {"@type": "VariableIterator", "variable": variable} if variable + else {"@type": "RepeatIterator", "count": count} + ) return { "@type": "Loop", - "iterator": {"@type": "RepeatIterator", "count": count}, + "iterator": iterator, "flowElements": [], "active": True, "comment": None, @@ -88,7 +173,13 @@ def build_loop(count: int = 1) -> dict[str, Any]: } -def build_if_statement(expression: Optional[str] = None, label: Optional[str] = None) -> dict[str, Any]: +def build_if_statement(label: Optional[str] = None) -> dict[str, Any]: + """Build a condition. + + A condition carries no expression: the branch taken depends on the result of the + preceding step, which the UI shows as the condition's "Statement" and marks with + errorPolicy CATCH. An `expression` field is dropped by the API. + """ then_branch = build_branch("THEN") else_branch = build_branch("ELSE") statement: dict[str, Any] = { @@ -103,8 +194,6 @@ def build_if_statement(expression: Optional[str] = None, label: Optional[str] = "status": None, "active": True, } - if expression: - statement["expression"] = expression return statement @@ -187,17 +276,35 @@ def walk_element(element: dict[str, Any]) -> None: walk_element(element) -def update_element_arguments(element: dict[str, Any], arguments: dict[str, Any]) -> None: +def update_element_arguments( + element: dict[str, Any], + arguments: dict[str, Any], + contract: Optional["CommandContract"] = None, +) -> None: command_id = command_id_from_element(element) spec = get_command_spec(command_id) - existing = {argument["name"]: argument for argument in element.get("arguments", [])} + + # Keyed by name but keeping every occurrence: a multivalued parameter has several + # arguments under the same name, and collapsing them would drop all but the last. + existing: dict[str, list[dict[str, Any]]] = {} + order: list[str] = [] + for argument in element.get("arguments", []): + name = argument.get("name", "") + if name not in existing: + existing[name] = [] + order.append(name) + existing[name].append(argument) + for name, value in spec.normalize_argument_names(arguments).items(): - if isinstance(value, dict) and "data_source" in value: - source = value["data_source"] - argument_value = value.get("value") - else: - source = "CONSTANT" - argument_value = value - existing[name] = _make_argument(name, argument_value, source) + # Sending a parameter replaces its whole list, the way the UI's row editor does. + existing[name] = [ + _make_argument(name, argument_value, source) + for source, argument_value in _argument_occurrences(name, value, contract) + ] + if name not in order: + order.append(name) + spec.drop_superseded_aliases(existing) - element["arguments"] = list(existing.values()) + element["arguments"] = [ + argument for name in order if name in existing for argument in existing[name] + ] diff --git a/tools/ai_scriptless/persistence.py b/tools/ai_scriptless/persistence.py index 1137696..8c83209 100644 --- a/tools/ai_scriptless/persistence.py +++ b/tools/ai_scriptless/persistence.py @@ -107,7 +107,8 @@ async def _persist_script( ] if snapshot_comment: result["notes"].append( - "The comment labels the '' entry in list_snapshots (UI: Save with comment)." + "The comment labels the version this save created (UI: Save with comment). It shows on " + "'' only until the next save, after which it stays with its own history entry." ) else: result["notes"].append( diff --git a/tools/ai_scriptless/script.py b/tools/ai_scriptless/script.py index f52dc0e..eeffee1 100644 --- a/tools/ai_scriptless/script.py +++ b/tools/ai_scriptless/script.py @@ -10,7 +10,8 @@ find_step_path_for_element, insert_flow_element, move_element_by_path, - set_condition_expression, + find_condition_statement, + set_element_error_policy, set_element_enabled, update_flow_element_counts, ) @@ -98,8 +99,8 @@ def delete_element_by_path(self, step_path: StepPathInput) -> None: def set_element_enabled(self, step_path: StepPathInput, enabled: bool) -> None: set_element_enabled(self._data, step_path, enabled) - def set_condition_expression(self, step_path: StepPathInput, expression: str) -> None: - set_condition_expression(self._data, step_path, expression) + def set_element_error_policy(self, step_path: StepPathInput, error_policy: str) -> None: + set_element_error_policy(self._data, step_path, error_policy) def move_element_by_path( self, diff --git a/tools/ai_scriptless/tree.py b/tools/ai_scriptless/tree.py index 9cb46d0..b532dcc 100644 --- a/tools/ai_scriptless/tree.py +++ b/tools/ai_scriptless/tree.py @@ -173,17 +173,50 @@ def set_element_enabled(script: dict[str, Any], step_path: StepPathInput, enable if located is None: raise ValueError(f"step_path not found: {step_path}") _, _, element = located + # The UI marks an excluded step with both fields; active alone leaves status stale. element["active"] = enabled + element["status"] = None if enabled else "DISABLED" -def set_condition_expression(script: dict[str, Any], step_path: StepPathInput, expression: str) -> None: +CONDITION_STATEMENT_POLICY = "CATCH" + + +def set_element_error_policy(script: dict[str, Any], step_path: StepPathInput, error_policy: str) -> None: + """Set a step's on-failure behaviour (the UI's "On-fail Result"). + + CATCH is what makes a step the statement of the condition that follows it, which is + how a condition decides its branch — there is no expression to set. + """ located = find_element_by_path(script, step_path) if located is None: raise ValueError(f"step_path not found: {step_path}") _, _, element = located - if element.get("@type") != "IfStatement": - raise ValueError(f"step_path must reference an IfStatement: {step_path}") - element["expression"] = expression + if element.get("@type") not in ("Action", "Validation"): + raise ValueError( + f"step_path must reference a command, not a container: {step_path}" + ) + element["errorPolicy"] = error_policy + + +def find_condition_statement( + script: dict[str, Any], + step_path: StepPathInput, +) -> Optional[str]: + """Step path of the statement feeding an IfStatement, if it has one. + + The statement is the immediately preceding sibling carrying errorPolicy CATCH; the + UI renders it inside the condition but the script stores it as a sibling. + """ + located = find_element_by_path(script, step_path) + if located is None: + return None + siblings, index, element = located + if element.get("@type") != "IfStatement" or index == 0: + return None + previous = siblings[index - 1] + if previous.get("errorPolicy") != CONDITION_STATEMENT_POLICY: + return None + return find_step_path_for_element(script, previous) def move_element_by_path( diff --git a/tools/ai_scriptless/variables.py b/tools/ai_scriptless/variables.py index 59379ba..210c26c 100644 --- a/tools/ai_scriptless/variables.py +++ b/tools/ai_scriptless/variables.py @@ -25,11 +25,16 @@ def validate_variable_name(name: str) -> None: def _coerce_variable_value(variable_type: str, value: Any) -> Any: if variable_type == "boolean": + # A boolean has three states in the UI: Null, True and False. + if value is None or value == "": + return None if isinstance(value, bool): return value normalized = str(value).lower() + if normalized == "null": + return None if normalized not in ("true", "false"): - raise ValueError("boolean value must be true or false") + raise ValueError("boolean value must be true, false or null") return normalized == "true" if variable_type == "number": try: @@ -76,16 +81,48 @@ def build_variable_entry( } -def find_variable(script: dict[str, Any], variable_name: str) -> Optional[tuple[int, dict[str, Any]]]: - for index, variable in enumerate(script.get("variables", [])): - data = variable.get("data", {}) - if data.get("name") == variable_name: - return index, variable +# Where a declaration lives depends on "Set at runtime" (the UI checkbox): +# checked -> parameters[] as a Parameter (DUT is one), unchecked -> variables[] as a Variable. +# The UI's "Configure test variables" dialog lists both, parameters first. +VARIABLE_ARRAYS = ("parameters", "variables") + +DUT_NAME = "DUT" + + +def variable_array_for(set_at_runtime: bool) -> str: + return "parameters" if set_at_runtime else "variables" + + +def locate_variable( + script: dict[str, Any], + variable_name: str, +) -> Optional[tuple[str, int, dict[str, Any]]]: + """Find a declaration in either array: (array key, index, entry).""" + for array_key in VARIABLE_ARRAYS: + for index, variable in enumerate(script.get(array_key) or []): + if not isinstance(variable, dict): + continue + if variable.get("data", {}).get("name") == variable_name: + return array_key, index, variable return None +def find_variable(script: dict[str, Any], variable_name: str) -> Optional[tuple[int, dict[str, Any]]]: + located = locate_variable(script, variable_name) + if located is None: + return None + _array_key, index, variable = located + return index, variable + + def list_script_variables(script: dict[str, Any]) -> list[dict[str, Any]]: - return list(script.get("variables", [])) + """Every declaration, runtime parameters first, as the UI dialog lists them.""" + entries: list[dict[str, Any]] = [] + for array_key in VARIABLE_ARRAYS: + entries.extend( + variable for variable in (script.get(array_key) or []) if isinstance(variable, dict) + ) + return entries def add_script_variable( @@ -96,12 +133,12 @@ def add_script_variable( set_at_runtime: bool = False, ) -> dict[str, Any]: validate_variable_name(name) - if find_variable(script, name): - raise ValueError(f"variable already exists: {name}") - if name == "DUT": + if name == DUT_NAME: raise ValueError("DUT is a test parameter, not a script variable") + if locate_variable(script, name): + raise ValueError(f"variable already exists: {name}") entry = build_variable_entry(name, variable_type, value, set_at_runtime) - script.setdefault("variables", []).append(entry) + script.setdefault(variable_array_for(set_at_runtime), []).append(entry) return entry @@ -112,10 +149,14 @@ def modify_script_variable( variable_type: Optional[str] = None, set_at_runtime: Optional[bool] = None, ) -> dict[str, Any]: - located = find_variable(script, variable_name) + located = locate_variable(script, variable_name) if located is None: raise ValueError(f"variable not found: {variable_name}") - _, variable = located + array_key, index, variable = located + if variable_name == DUT_NAME: + raise ValueError( + "DUT is the device parameter execute_test fills in; it cannot be modified here" + ) current_type = _variable_type_from_data(variable.get("data", {})) target_type = variable_type or current_type if target_type not in SUPPORTED_VARIABLE_TYPES: @@ -128,15 +169,24 @@ def modify_script_variable( variable["data"] = build_variable_data(target_type, variable_name, target_value) if set_at_runtime is not None: variable["@type"] = "Parameter" if set_at_runtime else "Variable" + # Toggling "Set at runtime" moves the declaration to the other array. + target_array = variable_array_for(set_at_runtime) + if target_array != array_key: + script.get(array_key, []).pop(index) + script.setdefault(target_array, []).append(variable) return variable def delete_script_variable(script: dict[str, Any], variable_name: str) -> None: - located = find_variable(script, variable_name) + located = locate_variable(script, variable_name) if located is None: raise ValueError(f"variable not found: {variable_name}") - index, _ = located - script.get("variables", []).pop(index) + array_key, index, _variable = located + if variable_name == DUT_NAME: + raise ValueError( + "DUT is the device parameter every step binds handsetId to; deleting it breaks the test" + ) + script.get(array_key, []).pop(index) def _variable_type_from_data(data: dict[str, Any]) -> str: @@ -151,3 +201,44 @@ def _variable_type_from_data(data: dict[str, Any]) -> str: "TableData": "datatable", } return reverse.get(data_type, "string") + + +# Parameter dataType (command repository) -> variable data @type (script model). +# The UI only offers variables whose type matches the parameter being bound. +VARIABLE_DATA_TYPES_BY_PARAMETER_TYPE = { + "STRING": frozenset({"StringData"}), + "INTEGER": frozenset({"IntegerData"}), + "NUMBER": frozenset({"IntegerData"}), + "BOOLEAN": frozenset({"BooleanData"}), + "HANDSET": frozenset({"HandsetData"}), + "MEDIA": frozenset({"MediaData"}), + "TABLE": frozenset({"TableData"}), +} + + +def variable_type_label(data: dict[str, Any]) -> str: + """Our vocabulary for a variable's data type (string, number, device, ...).""" + return _variable_type_from_data(data) + + +def bindable_values(script: dict[str, Any]) -> dict[str, dict[str, Any]]: + """Everything an argument can bind to, by name. + + Script variables plus the test parameters (DUT lives in parameters[], not + variables[]), which is how the UI lists them together. + """ + bindable: dict[str, dict[str, Any]] = {} + for entry in list(script.get("parameters", [])) + list(script.get("variables", [])): + data = entry.get("data", {}) if isinstance(entry, dict) else {} + name = data.get("name") + if name: + bindable[name] = data + return bindable + + +def describe_bindable_values(script: dict[str, Any]) -> str: + described = [ + f"{name} ({variable_type_label(data)})" + for name, data in sorted(bindable_values(script).items()) + ] + return ", ".join(described) if described else "none" diff --git a/tools/ai_scriptless_manager.py b/tools/ai_scriptless_manager.py index 637cc7f..3380720 100644 --- a/tools/ai_scriptless_manager.py +++ b/tools/ai_scriptless_manager.py @@ -1,3 +1,4 @@ +import copy import json from typing import Optional, Any, Dict from urllib.parse import quote @@ -12,7 +13,7 @@ from formatters.ai_scriptless import format_ai_scriptless_tests, \ format_ai_scriptless_tests_filter_values, command_selection_policy_info, \ format_command_catalog, format_command_definitions, format_snapshots_list, \ - format_test_structure, format_test_variables + format_step_detail, format_test_structure, format_test_variables from models.manager import Manager from models.result import BaseResult, PaginationResult from telemetry import run_tool @@ -26,7 +27,7 @@ build_move_test_body, build_snapshot_search_body, command_id_from_element, - declared_parameters, + command_contract, delete_script_variable, delete_element_by_path, empty_mandatory_note, @@ -34,20 +35,29 @@ find_element_by_path, find_step_path_for_element, insert_flow_element, + bindable_values, + describe_bindable_values, + list_script_variables, + variable_type_label, load_and_mutate, modify_script_variable, move_element_by_path, new_empty_script, + normalize_if_statement_aliases, persist_script, script_write_lock, - set_condition_expression, + find_condition_statement, + set_element_error_policy, set_element_enabled, split_item_key, item_key_file_name, format_test_ui_location, update_element_arguments, validate_argument_names, + validate_argument_values, + validate_variable_bindings, ) +from tools.ai_scriptless.definitions import SUPPORTED_ERROR_POLICIES from tools.utils import api_request, format_sanitized_traceback, normalize_action_args STEP_PATH_REFRESH_NOTES = [ @@ -56,6 +66,13 @@ "do not reuse step_path values from this response.", ] +CONDITION_STATEMENT_HINT = ( + "A condition has no expression: the branch taken depends on the result of the step right " + "before it, which must carry errorPolicy CATCH (the UI shows that step as the condition's " + "'Statement'). Add the deciding command before the condition and mark it with " + "set_command_error_policy(error_policy='CATCH'). An expression is dropped by Perfecto." +) + CMD_ARGUMENTS_COLLISION_HINT = ( "The ai_user-action command declares a parameter named 'action', which collides with the action key " "of this tool: command arguments must stay nested inside 'cmd_arguments', never flattened into args. " @@ -267,6 +284,36 @@ async def view_test_structure(self, test_id: str) -> BaseResult: result_formatter_params={"item_key": test_id}) return _append_ui_access_info(result, self.token.cloud_name, test_id) + @token_verify + async def view_test_step(self, test_id: str, step_path: str) -> BaseResult: + if not test_id: + return BaseResult(error="test_id is required (itemKey from list_tests)") + if not step_path: + return BaseResult(error="step_path is required (from view_test_structure)") + + payload_result = await fetch_script_payload(self.token, test_id) + if payload_result.error: + return payload_result + payload = payload_result.result if isinstance(payload_result.result, dict) else {} + script = copy.deepcopy(payload.get("script", {})) + normalize_if_statement_aliases(script) + located = find_element_by_path(script, step_path) + if located is None: + return BaseResult(error=f"step_path not found: {step_path} (call view_test_structure)") + _, _, element = located + + detail = format_step_detail( + element, + item_key=test_id, + step_path=step_path, + # The script payload already carries the definitions of the commands it uses. + command_definitions=payload.get("commandDefinitions"), + statement_step_path=find_condition_statement(script, step_path), + ) + if detail.type == "IfStatement" and detail.statement_step_path is None: + detail.notes.append(CONDITION_STATEMENT_HINT) + return _append_ui_access_info(BaseResult(result=detail), self.token.cloud_name, test_id) + @token_verify async def add_command( self, @@ -281,15 +328,22 @@ async def add_command( if not command_id: return BaseResult(error="command_id is required (from list_commands)") - declared = await declared_parameters(self.token, command_id) - names_error = validate_argument_names(command_id, cmd_arguments, declared) + contract = await command_contract(self.token, command_id) + names_error = validate_argument_names(command_id, cmd_arguments, contract) if names_error: return BaseResult(error=names_error) + values_error = validate_argument_values(command_id, cmd_arguments, contract) + if values_error: + return BaseResult(error=values_error) - element = build_flow_element(command_id, cmd_arguments) + element = build_flow_element(command_id, cmd_arguments, contract) inserted_path: dict[str, Optional[str]] = {"step_path": None} def mutator(script: dict[str, Any]) -> None: + # Variable bindings need the script: the variable must exist with a matching type. + bindings_error = validate_variable_bindings(command_id, cmd_arguments, contract, script) + if bindings_error: + raise ValueError(bindings_error) insert_flow_element(script, element, after_path=after_path, parent_path=parent_path) inserted_path["step_path"] = find_step_path_for_element(script, element) @@ -298,7 +352,7 @@ def mutator(script: dict[str, Any]) -> None: return result result.result["step_path"] = inserted_path["step_path"] result.result["command_id"] = command_id - empty_note = empty_mandatory_note(command_id, cmd_arguments, declared) + empty_note = empty_mandatory_note(command_id, cmd_arguments, contract) if empty_note: result.result.setdefault("notes", []).append(empty_note) return _append_step_path_refresh_notes(result) @@ -319,11 +373,17 @@ async def mutator(script: dict[str, Any]) -> None: _, _, element = located # command_id is only known after locating the step, so validation happens here. command_id = command_id_from_element(element) - declared = await declared_parameters(self.token, command_id) - names_error = validate_argument_names(command_id, cmd_arguments, declared) + contract = await command_contract(self.token, command_id) + names_error = validate_argument_names(command_id, cmd_arguments, contract) if names_error: raise ValueError(names_error) - update_element_arguments(element, cmd_arguments) + values_error = validate_argument_values(command_id, cmd_arguments, contract) + if values_error: + raise ValueError(values_error) + bindings_error = validate_variable_bindings(command_id, cmd_arguments, contract, script) + if bindings_error: + raise ValueError(bindings_error) + update_element_arguments(element, cmd_arguments, contract) return _append_step_path_refresh_notes( await load_and_mutate(self.token, test_id, mutator) @@ -449,15 +509,43 @@ async def add_loop( count: int = 1, after_path: Optional[str] = None, parent_path: Optional[str] = None, + variable: Optional[str] = None, ) -> BaseResult: if not test_id: return BaseResult(error="test_id is required") - if count < 1: + if not variable and count < 1: return BaseResult(error="count must be at least 1") - element = build_loop(count) + + if variable: + # The UI only offers number variables here, so check before writing. + payload_result = await fetch_script_payload(self.token, test_id) + if payload_result.error: + return payload_result + script = payload_result.result.get("script", {}) if payload_result.result else {} + bindable = bindable_values(script) + data = bindable.get(variable) + if data is None: + return BaseResult( + error=( + f"variable '{variable}' is not defined on this test. " + f"Defined: {describe_bindable_values(script)}." + ) + ) + if variable_type_label(data) != "number": + return BaseResult( + error=( + f"a loop counts with a number variable; '{variable}' is a " + f"{variable_type_label(data)}." + ) + ) + + element = build_loop(count, variable) result = await self._add_structure(test_id, element, "Loop", after_path, parent_path) if not result.error: - result.result["count"] = count + if variable: + result.result["variable"] = variable + else: + result.result["count"] = count return result @token_verify @@ -471,29 +559,42 @@ async def add_condition( ) -> BaseResult: if not test_id: return BaseResult(error="test_id is required") - element = build_if_statement(expression, label) + if expression: + return BaseResult(error=CONDITION_STATEMENT_HINT) + element = build_if_statement(label) result = await self._add_structure(test_id, element, "IfStatement", after_path, parent_path) - if not result.error and expression: - result.result["expression"] = expression + if not result.error: + result.result.setdefault("notes", []).append(CONDITION_STATEMENT_HINT) return result @token_verify - async def set_condition_expression(self, test_id: str, step_path: str, expression: str) -> BaseResult: + async def set_command_error_policy(self, test_id: str, step_path: str, error_policy: str) -> BaseResult: if not test_id: return BaseResult(error="test_id is required") if not step_path: - return BaseResult(error="step_path is required (IfStatement path from view_test_structure)") - if not expression: - return BaseResult(error="expression is required") + return BaseResult(error="step_path is required (from view_test_structure)") + policy = str(error_policy or "").upper() + if policy not in SUPPORTED_ERROR_POLICIES: + return BaseResult( + error=( + f"error_policy must be one of {', '.join(sorted(SUPPORTED_ERROR_POLICIES))}, " + f"got {error_policy!r}." + ) + ) def mutator(script: dict[str, Any]) -> None: - set_condition_expression(script, step_path, expression) + set_element_error_policy(script, step_path, policy) result = await load_and_mutate(self.token, test_id, mutator) if result.error: return result result.result["step_path"] = step_path - result.result["expression"] = expression + result.result["error_policy"] = policy + if policy == "CATCH": + result.result.setdefault("notes", []).append( + "This step now feeds the condition that follows it; the UI shows it as that " + "condition's Statement." + ) return _append_step_path_refresh_notes(result) @token_verify @@ -612,7 +713,9 @@ async def list_test_variables(self, test_id: str) -> BaseResult: if payload_result.error: return payload_result script = payload_result.result.get("script", {}) - variables = format_test_variables(script.get("variables", [])) + # Runtime parameters live in parameters[] and plain variables in variables[]; the UI + # dialog lists both, so reading only one array hides half the declarations. + variables = format_test_variables(list_script_variables(script)) return BaseResult(result=variables) @token_verify @@ -629,8 +732,15 @@ async def add_test_variable( if not name: return BaseResult(error="name is required") + stored_value = value + if variable_type == "secured_string" and value: + encrypted = await self._encrypt_secured_value(value) + if encrypted.error: + return encrypted + stored_value = encrypted.result + def mutator(script: dict[str, Any]) -> None: - add_script_variable(script, name, variable_type, value, set_at_runtime) + add_script_variable(script, name, variable_type, stored_value, set_at_runtime) result = await load_and_mutate(self.token, test_id, mutator) if result.error: @@ -638,8 +748,27 @@ def mutator(script: dict[str, Any]) -> None: result.result["name"] = name result.result["type"] = variable_type result.result["set_at_runtime"] = set_at_runtime + if variable_type == "secured_string": + # Never echo either the plaintext or the ciphertext back. + result.result["value"] = "" return result + async def _encrypt_secured_value(self, value: Any) -> BaseResult: + """Encrypt through the same endpoint the UI's lock button calls.""" + encrypt_url = perfecto.get_ai_scriptless_api_url(self.token.cloud_name) + encrypt_url = encrypt_url + f"/script/variable/encrypt?value={quote(str(value), safe='')}" + result = await api_request(self.token, "GET", endpoint=encrypt_url) + if result.error: + return BaseResult(error=f"Could not encrypt the secured value: {result.error}") + ciphertext = result.result + if isinstance(ciphertext, dict): + ciphertext = ciphertext.get("value") or ciphertext.get("result") + if not isinstance(ciphertext, str) or not ciphertext: + return BaseResult( + error="Could not encrypt the secured value: unexpected response from Perfecto" + ) + return BaseResult(result=ciphertext) + @token_verify async def modify_test_variable( self, @@ -656,8 +785,15 @@ async def modify_test_variable( if value is None and variable_type is None and set_at_runtime is None: return BaseResult(error="At least one of value, variable_type, or set_at_runtime is required") + stored_value = value + if variable_type == "secured_string" and value is not None: + encrypted = await self._encrypt_secured_value(value) + if encrypted.error: + return encrypted + stored_value = encrypted.result + def mutator(script: dict[str, Any]) -> None: - modify_script_variable(script, name, value, variable_type, set_at_runtime) + modify_script_variable(script, name, stored_value, variable_type, set_at_runtime) result = await load_and_mutate(self.token, test_id, mutator) if result.error: @@ -709,6 +845,14 @@ def register(mcp, token: Optional[PerfectoToken]): - view_test_structure: View the hierarchical structure of an AI Scriptless test. Each step has step_path (dot-separated positional path, e.g. 0, 2.0, 5.b0.1). args(dict): Dictionary with the following required parameters: test_id (str): Test itemKey from list_tests (e.g. PRIVATE:My Folder/My Test.xml). +- view_test_step: View the full configuration of one step (view_test_structure is the high-level tree; this is the detail). + Returns every persisted argument with its current value and data_source, joined with what the command declares + (parameter_type, mandatory, allowed_values, value_range, allowed_data_sources), plus unset_parameters: the declared + parameters the step does not set yet. Read it before modify_command to know the exact keys and accepted values. + Containers also report label, expression, loop_count and the step_path of their direct children. + args(dict): Dictionary with the following required parameters: + test_id (str): Test itemKey from list_tests. + step_path (str): Step path from view_test_structure (e.g. 0, 2.0, 5.b0.1). - list_commands: List available AI Scriptless commands from the command repository. Returns the catalog in result and command selection policy in info (read info before add_command when authoring tests). args(dict): Dictionary with the following optional parameters: @@ -725,7 +869,13 @@ def register(mcp, token: Optional[PerfectoToken]): cmd_arguments (dict, optional): Command argument names to values. Keys must be parameter names from get_command_definitions (mandatory_parameters / optional_parameters); undeclared keys are rejected. Values are constants by default. To point an argument at a script variable instead of a constant, - pass {"data_source": "VARIABLE", "value": ""} (see list_test_variables). + pass {"data_source": "VARIABLE", "value": ""} (see list_test_variables), or bind it to + a DataTable column with {"data_source": "DATATABLE", "table_name": "
", "column": ""}. + A parameter only accepts the data sources listed in allowed_data_sources by view_test_step. + A multivalued parameter (max_occurrences > 1, e.g. the 'value' of text_concat which needs at least + two) takes a list, in order, each item a constant or a binding of its own: + {"value": ["Hello", {"data_source": "VARIABLE", "value": "name"}]}. Sending a multivalued parameter + replaces its whole list, the way the UI's row editor does. Never flatten these keys to the top level of args: ai_user-action declares a parameter named 'action', which would collide with the action key of this tool. Always nest them in cmd_arguments, e.g. {"action": "add_command", "args": {"test_id": "...", "command_id": "ai_user-action", @@ -775,21 +925,28 @@ def register(mcp, token: Optional[PerfectoToken]): - add_loop: Add a Loop container and persist. args(dict): Dictionary with the following parameters: test_id (str, required): Test itemKey from list_tests. - count (int, default=1): RepeatIterator count. + count (int, default=1): How many times to repeat (ignored when variable is given). + variable (str, optional): Name of a number variable whose value decides the iterations, + instead of a fixed count (UI: the Loop editor's Variable mode). after_path (str, optional): Insert after this step path. parent_path (str, optional): Insert inside a container step path. - add_condition: Add an IfStatement condition with Then/Else branches and persist. + A condition has no expression. The branch taken depends on the result of the step immediately + before it, which must carry errorPolicy CATCH; the UI shows that step as the condition's + 'Statement'. So: add the deciding command (typically a validation or checkpoint), mark it with + set_command_error_policy(error_policy='CATCH'), then add the condition after it. args(dict): Dictionary with the following parameters: test_id (str, required): Test itemKey from list_tests. - expression (str, optional): Condition expression. label (str, optional): Condition label. after_path (str, optional): Insert after this step path. parent_path (str, optional): Insert inside a container step path. -- set_condition_expression: Set the expression on an IfStatement and persist. +- set_command_error_policy: Set what a step does when it fails ('On-fail Result' in the UI) and persist. args(dict): Dictionary with the following required parameters: test_id (str): Test itemKey from list_tests. - step_path (str): IfStatement path from view_test_structure (e.g. 5). - expression (str): Condition expression. + step_path (str): Step path of a command (not a container) from view_test_structure. + error_policy (str, values=['ABORT', 'IGNORE', 'BREAK', 'CONTINUE', 'CATCH']): ABORT ends the run, + IGNORE reports the failure and goes on, BREAK and CONTINUE act on the enclosing loop (abort + outside one), CATCH feeds the result to the condition that follows the step. - move_command: Move a step to a new position and persist. args(dict): Dictionary with the following required parameters: test_id (str): Test itemKey from list_tests. @@ -812,39 +969,60 @@ def register(mcp, token: Optional[PerfectoToken]): - view_snapshot: View the hierarchical structure of a historical snapshot (same format as view_test_structure). args(dict): Dictionary with the following required parameters: snapshot_id (str): UUID key from list_snapshots (not ''; use view_test_structure for the live script). -- list_test_variables: List script variables configured on a test (distinct from DUT parameters). +- list_test_variables: List everything the test declares, exactly as the UI's Configure test variables dialog: + runtime parameters first (set_at_runtime=true, including the DUT device parameter), then plain variables. args(dict): Dictionary with the following required parameters: test_id (str): Test itemKey from list_tests. -- add_test_variable: Add a script variable and persist. +- add_test_variable: Add a script variable or runtime parameter and persist. args(dict): Dictionary with the following required parameters: test_id (str): Test itemKey from list_tests. - name (str): Variable name (letters, numbers, underscore; cannot start with a number). + name (str): Variable name (letters, numbers, underscore; cannot start with a number). Must be unique + across runtime parameters and variables alike; Perfecto rejects a name declared twice. variable_type (str, default='string', values=['string', 'secured_string', 'number', 'boolean']): Variable type. - value (any, default=''): Variable value. - set_at_runtime (bool, default=false): When true, value is supplied at execution time. + value (any, default=''): Variable value. For secured_string, pass the plaintext: it is encrypted + through Perfecto before being stored (the UI's lock button) and never echoed back. + set_at_runtime (bool, default=false): True makes it a runtime variable: the stored value is only the + default, it is supplied when the run starts (UI: the 'Set at runtime' checkbox, then the 'Enter + runtime values' dialog; execute_test is that same channel, the way DUT receives its device) and it + may change during the execution — a command parameter declared inOutBehavior OUT writes its result + into the variable it names. False makes the value constant for the whole run. - modify_test_variable: Update a script variable and persist. args(dict): Dictionary with the following required parameters: test_id (str): Test itemKey from list_tests. - name (str): Existing variable name. + name (str): Existing variable name (runtime parameters included; DUT cannot be modified). value (any, optional): New value. variable_type (str, optional): New type (string, secured_string, number, boolean). - set_at_runtime (bool, optional): Toggle runtime parameter behavior. + set_at_runtime (bool, optional): Toggle between runtime parameter and variable. - delete_test_variable: Remove a script variable and persist. args(dict): Dictionary with the following required parameters: test_id (str): Test itemKey from list_tests. - name (str): Variable name to delete. + name (str): Variable name to delete (runtime parameters included; DUT cannot be deleted). Hints: - LICENSE: AI Scriptless actions require a Perfecto AI license on your cloud (administrator opt-in via feature toggle). Without it, AI commands and related MCP operations will not work. Desktop web test authoring additionally requires the Desktop Web license. - COVERAGE: DataTables, Scheduler (scheduled jobs), Embedded tests, and other advanced UI capabilities (folder management, rename test, restore snapshot, download as Appium, AI Assistant, Object Spy, per-step error policy, etc.) are not yet supported by this MCP tool. - HELP: For product behavior and workarounds, use the perfecto_help tool: Filter by category_id='perfecto', subcategory_id_list=['ide']. - UI_ACCESS: No per-test URL exists. Only UI entry: cloud_url/lab/scriptless-mobile/ (cloud_url from perfecto_user read_user). For debugging or unsupported MCP tasks, link the lab URL and tell the user to open the test via Tests → Open or Manage tests using the folder tree and test name from list_tests (itemKey is MCP-only; the UI shows folders and names, not itemKey). Never invent other scriptless URLs. - When authoring or editing test steps, call list_commands first and follow the command selection policy in the info field. +- Before editing an existing step, call view_test_step with its step_path: view_test_structure is a high-level tree and + does not show argument values, so it is not enough to know what to change. - cmd_arguments keys are validated against the command definitions before saving: an undeclared name is rejected with the list of valid parameter names instead of being persisted as a step argument that Perfecto ignores at runtime. +- Values are validated too: a number outside the declared range, a value outside allowed_values, a non-boolean on a + boolean parameter, or a data_source the parameter does not accept are all rejected with the accepted options. + Numbers and booleans are normalized to the spelling Perfecto persists, so passing 30 or "30" is equivalent. +- A VARIABLE binding is checked against the test: the variable must exist and its type must match the parameter + (a string variable cannot feed a Number parameter), the same rule the UI enforces by only offering compatible + variables. The error lists the variables the test defines with their types. +- A step's failure semantics also come from the command definition: Action steps abort the test (ABORT) while + Validation steps are only reported (IGNORE). No need to set it, add_command applies what the command declares. - step_path is a dot-separated positional path without spaces (0-based indices; b0=Then branch, b1=Else). Example: root step 3 is "3"; first step inside Then of condition at 5 is "5.b0.0". Perfecto does not persist paths; they change when steps are inserted, moved, or deleted. Always call view_test_structure before the next structure edit; do not reuse step_path from a previous mutation response. - Use parent_path on add_command with the step_path of a LogicalStep, Loop, or Branch from view_test_structure. - Use add_logical_step, add_loop, and add_condition to build control-flow structures matching the UI toolbar Group, Loop, and Condition actions. -- Script variables (list_test_variables, add/modify/delete_test_variable) are stored in script.variables[] and are distinct from the DUT parameter in script.parameters[]. +- A declaration lives in one of two places depending on 'Set at runtime': runtime parameters in script.parameters[] + (as Parameter, where DUT lives) and fixed-value variables in script.variables[] (as Variable). The variable actions + cover both, and toggling set_at_runtime moves the declaration from one to the other. +- A VARIABLE binding on cmd_arguments can target any of them as long as the type matches the parameter + (a Number parameter needs a number variable), which is what the UI's variable picker filters by. - Snapshot behavior: every save creates a UUID history entry; comment on save_test labels ''. See list_snapshots notes for details. - Edits persist immediately via the internal draft→script pipeline (each persist also adds snapshot history; save_test is available to re-persist unchanged content). - IMPORTANT: Always call list_filter_values first to get valid filter values before using any filters in list_tests. @@ -883,6 +1061,11 @@ async def _dispatch(): args.get("device_under_test", {})) case "view_test_structure": return await ai_scriptless_manager.view_test_structure(args.get("test_id", "")) + case "view_test_step": + return await ai_scriptless_manager.view_test_step( + args.get("test_id", ""), + args.get("step_path", ""), + ) case "list_commands": return await ai_scriptless_manager.list_commands(args.get("checkpoint", False)) case "get_command_definitions": @@ -944,6 +1127,7 @@ async def _dispatch(): args.get("count", 1), args.get("after_path"), args.get("parent_path"), + args.get("variable"), ) case "add_condition": return await ai_scriptless_manager.add_condition( @@ -954,10 +1138,13 @@ async def _dispatch(): args.get("parent_path"), ) case "set_condition_expression": - return await ai_scriptless_manager.set_condition_expression( + # Kept so an agent that learned the old action gets the mechanism, not silence. + return BaseResult(error=CONDITION_STATEMENT_HINT) + case "set_command_error_policy": + return await ai_scriptless_manager.set_command_error_policy( args.get("test_id", ""), args.get("step_path", ""), - args.get("expression", ""), + args.get("error_policy", ""), ) case "move_command": return await ai_scriptless_manager.move_command(