Merge branch 'main' into review-improve

Author: xlight05

workspaces/ballerina/ballerina-core/src/rpc-types/ai-panel/index.ts

@@ -39,6 +39,8 @@ import {
     TaskDeclineRequest,
     ConnectorSpecRequest,
     ConnectorSpecCancelRequest,
+    ConfigurationProvideRequest,
+    ConfigurationCancelRequest,
     UIChatMessage,
     CheckpointInfo,
     AbortAIGenerationRequest,
@@ -93,6 +95,8 @@ export interface AIPanelAPI {
     declineTask: (params: TaskDeclineRequest) => Promise<void>;
     provideConnectorSpec: (params: ConnectorSpecRequest) => Promise<void>;
     cancelConnectorSpec: (params: ConnectorSpecCancelRequest) => Promise<void>;
+    provideConfiguration: (params: ConfigurationProvideRequest) => Promise<void>;
+    cancelConfiguration: (params: ConfigurationCancelRequest) => Promise<void>;
     // ==================================
     // Chat State Management
     // ==================================

workspaces/ballerina/ballerina-core/src/rpc-types/ai-panel/interfaces.ts

@@ -437,6 +437,16 @@ export interface ConnectorSpecCancelRequest {
     comment?: string;
 }
 
+export interface ConfigurationProvideRequest {
+    requestId: string;
+    configValues: Record<string, string>;
+}
+
+export interface ConfigurationCancelRequest {
+    requestId: string;
+    comment?: string;
+}
+
 export type ErrorCode = {
     code: number;
     message: string;

workspaces/ballerina/ballerina-core/src/rpc-types/ai-panel/rpc-type.ts

@@ -41,6 +41,8 @@ import {
     TaskDeclineRequest,
     ConnectorSpecRequest,
     ConnectorSpecCancelRequest,
+    ConfigurationProvideRequest,
+    ConfigurationCancelRequest,
     UIChatMessage,
     CheckpointInfo,
     AbortAIGenerationRequest,
@@ -85,6 +87,8 @@ export const approveTask: RequestType<ApproveTaskRequest, void> = { method: `${_
 export const declineTask: RequestType<TaskDeclineRequest, void> = { method: `${_preFix}/declineTask` };
 export const provideConnectorSpec: RequestType<ConnectorSpecRequest, void> = { method: `${_preFix}/provideConnectorSpec` };
 export const cancelConnectorSpec: RequestType<ConnectorSpecCancelRequest, void> = { method: `${_preFix}/cancelConnectorSpec` };
+export const provideConfiguration: RequestType<ConfigurationProvideRequest, void> = { method: `${_preFix}/provideConfiguration` };
+export const cancelConfiguration: RequestType<ConfigurationCancelRequest, void> = { method: `${_preFix}/cancelConfiguration` };
 export const getChatMessages: NotificationType<void> = { method: `${_preFix}/getChatMessages` };
 export const getCheckpoints: NotificationType<void> = { method: `${_preFix}/getCheckpoints` };
 export const restoreCheckpoint: RequestType<RestoreCheckpointRequest, void> = { method: `${_preFix}/restoreCheckpoint` };

workspaces/ballerina/ballerina-core/src/state-machine-types.ts

@@ -103,7 +103,8 @@ export enum MACHINE_VIEW {
     ServiceFunctionForm = "Service Function Form",
     BISamplesView = "BI Samples View",
     ReviewMode = "Review Mode SKIP",
-    EvalsetViewer = "Evalset Viewer SKIP"
+    EvalsetViewer = "Evalset Viewer SKIP",
+    ConfigurationCollector = "Configuration Collector"
 }
 
 export interface MachineEvent {
@@ -168,7 +169,20 @@ export interface ArtifactData {
     identifier?: string;
 }
 
+export interface ConfigurationCollectorMetadata {
+    requestId: string;
+    variables: Array<{
+        name: string;
+        description: string;
+        type?: "string" | "int";
+    }>;
+    existingValues?: Record<string, string>;
+    message: string;
+    isTestConfig?: boolean;
+}
+
 export interface AgentMetadata {
+    configurationCollector?: ConfigurationCollectorMetadata;
 }
 
 export interface ApprovalOverlayState {
@@ -263,7 +277,7 @@ export interface EvalsetTrace {
     iterations: EvalIteration[];
     output: EvalChatAssistantMessage | any;
     tools: EvalToolSchema[];
-    toolCalls: EvalFunctionCall[];
+    toolCalls?: EvalFunctionCall[];
     startTime: string;
     endTime: string;
 }
@@ -326,6 +340,7 @@ export type ChatNotify =
     | TaskApprovalRequest
     | GeneratedSourcesEvent
     | ConnectorGenerationNotification
+    | ConfigurationCollectionEvent
     | CodeReviewActions
     | PlanUpdated;
 
@@ -446,6 +461,24 @@ export interface ConnectorGenerationNotification {
     message: string;
 }
 
+export interface ConfigurationCollectionEvent {
+    type: "configuration_collection_event";
+    requestId: string;
+    stage: "creating_file" | "collecting" | "done" | "skipped" | "error";
+    variables?: Array<{
+        name: string;
+        description: string;
+        type?: "string" | "int";
+    }>;
+    existingValues?: Record<string, string>;
+    message: string;
+    isTestConfig?: boolean;
+    error?: {
+        message: string;
+        code: string;
+    };
+}
+
 export interface CodeReviewActions {
     type: "review_actions";
 }

workspaces/ballerina/ballerina-extension/package.json

@@ -753,6 +753,30 @@
                 "icon": "$(clear-all)",
                 "hidden": true,
                 "enablement": "isSupportedProject"
+            },
+            {
+                "command": "ballerina.createNewEvalset",
+                "title": "Create New Evalset",
+                "category": "Ballerina",
+                "icon": "$(add)"
+            },
+            {
+                "command": "ballerina.createNewThread",
+                "title": "Add Thread",
+                "category": "Ballerina",
+                "icon": "$(add)"
+            },
+            {
+                "command": "ballerina.deleteEvalset",
+                "title": "Delete Evalset",
+                "category": "Ballerina",
+                "icon": "$(trash)"
+            },
+            {
+                "command": "ballerina.deleteThread",
+                "title": "Delete Thread",
+                "category": "Ballerina",
+                "icon": "$(trash)"
             }
         ],
         "views": {
@@ -865,6 +889,28 @@
                     "when": "view == ballerina-traceView && ballerina.tracingEnabled && !ballerina.tracesEmpty",
                     "group": "navigation",
                     "title": "Clear Traces"
+                },
+                {
+                    "command": "ballerina.createNewEvalset",
+                    "when": "view == ballerina-evalsets",
+                    "group": "navigation"
+                }
+            ],
+            "view/item/context": [
+                {
+                    "command": "ballerina.createNewThread",
+                    "when": "view == ballerina-evalsets && viewItem == evalsetFile",
+                    "group": "inline"
+                },
+                {
+                    "command": "ballerina.deleteEvalset",
+                    "when": "view == ballerina-evalsets && viewItem == evalsetFile",
+                    "group": "inline"
+                },
+                {
+                    "command": "ballerina.deleteThread",
+                    "when": "view == ballerina-evalsets && viewItem == evalsetThread",
+                    "group": "inline"
                 }
             ],
             "BI.test.add.submenu": [

workspaces/ballerina/ballerina-extension/src/features/ai/agent/np/prompts.ts

@@ -14,6 +14,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
+import { CONFIG_COLLECTOR_TOOL } from "../tools/config-collector";
+
 export const REQUIREMENTS_DOCUMENT_KEY: string = "user_requirements_file";
 
 export function getRequirementAnalysisCodeGenPrefix(requirementAnalysisDocument: string) {
@@ -93,6 +95,12 @@ You are an expert test automation engineer specializing in generating test artif
 
 2. Create Ballerina test module with:
    - Network client configurations (if required, e.g., HTTP, GraphQL, WebSocket, etc.)
+   - Setup test configuration (if main code uses it):
+     * If main code uses configuration values (API keys, tokens, passwords), use ${CONFIG_COLLECTOR_TOOL}
+     * Set mode: "collect", isTestConfig: true
+     * Tell user: "Setting up test configuration based on main configuration"
+     * The tool will automatically handle reading from main config and saving to tests/Config.toml
+     * Example: { mode: "collect", variables: [{ name: "API_KEY", description: "Stripe API key", type: "string" }], isTestConfig: true }
    - Test data factories
    - Reusable validation functions
 

workspaces/ballerina/ballerina-extension/src/features/ai/agent/prompts.ts

@@ -20,6 +20,7 @@ import { LIBRARY_SEARCH_TOOL } from "./tools/library-search";
 import { TASK_WRITE_TOOL_NAME } from "./tools/task-writer";
 import { FILE_BATCH_EDIT_TOOL_NAME, FILE_SINGLE_EDIT_TOOL_NAME, FILE_WRITE_TOOL_NAME } from "./tools/text-editor";
 import { CONNECTOR_GENERATOR_TOOL } from "./tools/connector-generator";
+import { CONFIG_COLLECTOR_TOOL } from "./tools/config-collector";
 import { getLanglibInstructions } from "../utils/libs/langlibs";
 import { formatCodebaseStructure, formatCodeContext } from "./utils";
 import { GenerateAgentCodeRequest, OperationType, ProjectSource } from "@wso2/ballerina-core";
@@ -152,6 +153,11 @@ ${getLanglibInstructions()}
 
 ## Code Structure
 - Define required configurables for the query. Use only string, int, decimal, boolean types in configurable variables.
+- For sensitive configuration values (API keys, tokens, passwords), use ${CONFIG_COLLECTOR_TOOL} in COLLECT mode. Variable names are converted to lowercase without underscores in Config.toml. You MUST use the exact Config.toml names in your Ballerina configurables to avoid runtime errors.
+- When generating tests that need configuration values:
+  - Use COLLECT mode with isTestConfig: true
+  - The tool will automatically read existing values from Config.toml (if exists), ask user to reuse or modify for testing, and save to tests/Config.toml
+  - Example: { mode: "collect", variables: [...], isTestConfig: true }
 - Initialize any necessary clients with the correct configuration based on the retrieved libraries at the module level (before any function or service declarations).
 - Implement the main function OR service to address the query requirements.
 
@@ -181,7 +187,7 @@ ${getLanglibInstructions()}
     )} tools. The complete existing source code will be provided in the <existing_code> section of the user prompt.
 - When making replacements inside an existing file, provide the **exact old string** and the **exact new string** with all newlines, spaces, and indentation, being mindful to replace nearby occurrences together to minimize the number of tool calls.
 - Do NOT create a new markdown file to document each change or summarize your work unless specifically requested by the user.
-- Do not add/modify toml files (Config.toml/Ballerina.toml/Dependencies.toml) as you don't have access to those files.
+- Do not manually add/modify toml files (Ballerina.toml/Dependencies.toml). For Config.toml configuration management, use ${CONFIG_COLLECTOR_TOOL}.
 - Prefer modifying existing bal files over creating new files unless explicitly asked to create a new file in the query.
 
 ${getNPSuffix(projects, op)}

workspaces/ballerina/ballerina-extension/src/features/ai/agent/tool-registry.ts

@@ -40,6 +40,7 @@ import { GenerationType } from '../utils/libs/libraries';
 import { getHealthcareLibraryProviderTool, HEALTHCARE_LIBRARY_PROVIDER_TOOL } from './tools/healthcare-library';
 import { createConnectorGeneratorTool, CONNECTOR_GENERATOR_TOOL } from './tools/connector-generator';
 import { LIBRARY_SEARCH_TOOL, getLibrarySearchTool } from './tools/library-search';
+import { createConfigCollectorTool, CONFIG_COLLECTOR_TOOL } from './tools/config-collector';
 
 export interface ToolRegistryOptions {
     eventHandler: CopilotEventHandler;
@@ -79,6 +80,14 @@ export function createToolRegistry(opts: ToolRegistryOptions) {
             projects[0].projectName,
             modifiedFiles
         ),
+        [CONFIG_COLLECTOR_TOOL]: createConfigCollectorTool(
+            eventHandler,
+            {
+                tempPath: tempProjectPath,
+                workspacePath: workspaceId
+            },
+            modifiedFiles
+        ),
         [FILE_WRITE_TOOL_NAME]: createWriteTool(
             createWriteExecute(eventHandler, tempProjectPath, modifiedFiles)
         ),