Assertions are bery inconsistent #87
Description
I'm testing the detox copilot but is doing things inconsistent, even if the task is very simple.
Here is my handler:
import OpenAI from 'openai';
import type { ChatCompletionMessageParam } from 'openai/resources/chat/completions';
class OpenAIPromptHandler {
private readonly openai = new OpenAI({
apiKey: process.env.EXPO_PUBLIC_OPENAI_API_KEY,
});
// GPT-3.5-turbo can handle up to 4K tokens in context window
private readonly MAX_PROMPT_LENGTH = 4000; // Use most of the available context window
private readonly MAX_TOKENS = 256; // Reasonable limit for response tokens
private readonly SYSTEM_PROMPT = `You are a Detox E2E test assistant for React Native. You MUST ONLY generate Detox commands.
CORRECT Detox patterns to use:
expect(element(by.text("Welcome"))).toBeVisible()
expect(element(by.id("button"))).toExist()
expect(element(by.id("input"))).toHaveText("text")
await element(by.text("Submit")).tap()
INCORRECT patterns (DO NOT USE):
❌ onView(withText("text")) // This is Espresso
❌ cy.get("[data-test=button]") // This is Cypress
❌ await page.locator("text").click() // This is Playwright
❌ import statements or setup code
❌ comments or explanations
Rules:
1. Return ONLY the Detox command
2. No imports, no comments, no setup
3. No code blocks or markdown
4. Keep exact text/labels from the prompt
Example inputs and outputs:
Input: "Verify that the Welcome message is visible"
Output: expect(element(by.text("Welcome"))).toBeVisible()
Input: "Check if Submit button exists"
Output: expect(element(by.text("Submit"))).toExist()
Input: 'Verify that the "Hello!" message is displayed'
Output: expect(element(by.text("Hello!"))).toBeVisible()`;
async runPrompt(prompt: string): Promise<string> {
const truncatedPrompt =
prompt.length > this.MAX_PROMPT_LENGTH
? prompt.substring(0, this.MAX_PROMPT_LENGTH) + '...(truncated)'
: prompt;
const messages: ChatCompletionMessageParam[] = [
{
role: 'system',
content: this.SYSTEM_PROMPT,
},
{ role: 'user', content: truncatedPrompt },
];
try {
const response = await this.openai.chat.completions.create({
model: 'gpt-3.5-turbo',
messages: messages,
max_tokens: this.MAX_TOKENS,
temperature: 0.1,
});
return (response.choices[0].message.content ?? '').trim();
} catch (error: any) {
console.error('OpenAI API Error:', error);
throw new Error(
`Failed to generate test commands: ${error?.message || 'Unknown error'}`,
);
}
}
isSnapshotImageSupported() {
return true;
}
}
export default OpenAIPromptHandler;
The test:
import { copilot, expect } from 'detox';
import OpenAIPromptHandler from './OpenAIPromptHandler';
describe('Home Screen', () => {
beforeAll(async () => {
await device.launchApp();
const promptHandler = new OpenAIPromptHandler();
copilot.init(promptHandler);
});
beforeEach(async () => {
await device.reloadReactNative();
});
it('Should render home view', async () => {
await copilot.perform('Verify that the "Welcome!" message is displayed');
});
});
And the result:
