fix: screenshot output in web extractor

Author: yuyutaotao

packages/midscene/src/ai-model/service-caller/index.ts

@@ -19,7 +19,6 @@ import {
   MIDSCENE_API_TYPE,
   MIDSCENE_AZURE_OPENAI_INIT_CONFIG_JSON,
   MIDSCENE_AZURE_OPENAI_SCOPE,
-  MIDSCENE_DANGEROUSLY_PRINT_ALL_CONFIG,
   MIDSCENE_DEBUG_AI_PROFILE,
   MIDSCENE_LANGSMITH_DEBUG,
   MIDSCENE_MODEL_NAME,
@@ -31,7 +30,6 @@ import {
   OPENAI_BASE_URL,
   OPENAI_MAX_TOKENS,
   OPENAI_USE_AZURE,
-  allAIConfig,
   getAIConfig,
   getAIConfigInJson,
 } from '../../env';

packages/web-integration/tests/unit-test/fixtures/web-extractor/output.png

@@ -3,7 +3,11 @@ import { parseContextFromWebPage } from '@/common/utils';
 import StaticPage from '@/playground/static-page';
 import type { WebElementInfo } from '@/web-element';
 import { traverseTree } from '@midscene/shared/extractor';
-import { imageInfoOfBase64 } from '@midscene/shared/img';
+import {
+  compositeElementInfoImg,
+  imageInfoOfBase64,
+  saveBase64Image,
+} from '@midscene/shared/img';
 import { createServer } from 'http-server';
 import { beforeAll, describe, expect, it } from 'vitest';
 import { launchPage } from '../ai/web/puppeteer/utils';
@@ -39,19 +43,22 @@ describe(
         },
       });
 
-      const { content, tree } = await parseContextFromWebPage(page);
-      // TODO
-      // await generateExtractData(
-      //   page,
-      //   path.join(__dirname, 'fixtures/web-extractor'),
-      //   {
-      //     disableInputImage: false,
-      //     disableOutputImage: false,
-      //     disableOutputWithoutTextImg: true,
-      //     disableResizeOutputImg: true,
-      //     disableSnapshot: true,
-      //   },
-      // );
+      const { content, tree, screenshotBase64 } =
+        await parseContextFromWebPage(page);
+
+      const markedImg = await compositeElementInfoImg({
+        inputImgBase64: await page.screenshotBase64(),
+        elementsPositionInfo: content,
+      });
+
+      await saveBase64Image({
+        base64Data: screenshotBase64,
+        outputPath: join(pageDir, 'input.png'),
+      });
+      await saveBase64Image({
+        base64Data: markedImg,
+        outputPath: join(pageDir, 'output.png'),
+      });
 
       const list = content.map((item) => {
         return {