GhostBridge/test_with_files.py at main · 2823023323/GhostBridge · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
"""
GhostBridge 完整测试 - 保留测试文件
"""

import sys
from pathlib import Path
from dotenv import load_dotenv

# 加载环境变量
env_path = Path(r"C:\Users\ljy33\Desktop\GhostBridge\.env")
load_dotenv(env_path)

# 添加 GhostBridge 到 Python 路径
ghostbridge_path = Path(r"C:\Users\ljy33\Desktop\GhostBridge\GhostBridge")
if str(ghostbridge_path) not in sys.path:
    sys.path.insert(0, str(ghostbridge_path))

from ghostbridge.perception import add_tags_to_image, identify_element
from PIL import Image, ImageDraw


def create_test_ui_image():
    """创建一个模拟的 UI 界面截图"""
    img = Image.new('RGB', (800, 600), color='white')
    draw = ImageDraw.Draw(img)

    # 绘制登录表单
    draw.rectangle([250, 150, 550, 200], outline='gray', width=2)
    draw.text((270, 170), "用户名", fill='black')

    draw.rectangle([250, 210, 550, 260], outline='gray', width=2)
    draw.text((270, 230), "密码", fill='black')

    # 绘制登录按钮
    draw.rectangle([250, 300, 400, 350], fill='blue', outline='blue')
    draw.text((300, 320), "登录", fill='white')

    # 绘制注册按钮
    draw.rectangle([420, 300, 550, 350], outline='gray', width=2)
    draw.text((440, 320), "注册", fill='black')

    # 绘制标题
    draw.text((350, 100), "欢迎登录", fill='black')

    return img


def main():
    print("=" * 60)
    print("GhostBridge 完整测试")
    print("=" * 60)

    # 1. 创建测试图像
    print("\n1. 创建测试 UI 图像...")
    img = create_test_ui_image()
    test_image_path = Path("test_ui.png")
    img.save(test_image_path)
    print(f"   ✓ 已保存: {test_image_path}")

    # 2. 添加数字标签
    print("\n2. 为图像添加数字标签...")
    tagged_image = add_tags_to_image(str(test_image_path))
    print(f"   ✓ 已保存: {tagged_image}")

    # 3. 读取标签元数据
    print("\n3. 标签元数据:")
    import json
    meta_path = Path(f"{test_image_path.stem}_tags.json")
    if meta_path.exists():
        with open(meta_path, 'r', encoding='utf-8') as f:
            meta = json.load(f)

        for tag in meta['tags']:
            print(f"   标签 {tag['id']}: 中心坐标 {tag['center']}")

    # 4. 识别元素
    print("\n4. 识别 UI 元素...")

    test_cases = [
        "登录按钮",
        "注册按钮",
        "用户名输入框",
        "密码输入框"
    ]

    for instruction in test_cases:
        print(f"\n   识别: {instruction}")
        result = identify_element(tagged_image, instruction)

        tag_id = result['tag_id']
        center = result['center']

        print(f"   → 标签 ID: {tag_id}")
        print(f"   → 中心坐标: {center}")

        if center:
            print(f"   ✓ 识别成功！")
        else:
            print(f"   ⚠ 未找到坐标")

    # 5. 显示结果
    print("\n" + "=" * 60)
    print("测试完成！")
    print("=" * 60)
    print(f"\n生成的文件（已保留）:")
    print(f"  - 原始图像: {test_image_path}")
    print(f"  - 标签图像: {tagged_image}")
    print(f"  - 元数据: {meta_path}")

    print(f"\n💡 提示:")
    print(f"  1. 打开 {tagged_image} 查看添加的数字标签")
    print(f"  2. 打开 {meta_path} 查看标签坐标信息")
    print(f"  3. 对比原始图像和标签图像，理解标签位置")


if __name__ == "__main__":
    main()