Skip to content

3DSRBench和SiteBenchImage两个数据集评测的bug麻烦修复下,修复方式文中给出。 #1485

@yourlovedu

Description

@yourlovedu

评测过程中发现两个bug,麻烦有空帮忙修正下:
1.3DSRBench评测name_str未定义。
vlmeval/dataset/image_mcq.py文件中2576行的name_str应该得和其他的一样加上类似下面的代码

        name_str_map = {'chatgpt-0125': 'openai', 'gpt-4-0125': 'gpt4'}
        name_str = name_str_map[model] if model in name_str_map else model

2.SiteBenchImage读图时会有没扩展名的文件。
vlmeval/smp/file.py文件378行的parse_file函数应该参考https://github.com/EvolvingLMMs-Lab/VLMEvalKit/blob/EASI/vlmeval/smp/file.py
修改为如下

def _mime_from_pillow(p):
    from PIL import Image
    try:
        with Image.open(p) as im:
            fmt = (im.format or '').lower()
        return {
            'jpeg': 'image/jpeg', 'jpg': 'image/jpeg',
            'png': 'image/png',
            'webp': 'image/webp',
            'bmp': 'image/bmp',
            'gif': 'image/gif',
            'tiff': 'image/tiff', 'tif': 'image/tiff',
        }.get(fmt, 'unknown')
    except Exception:
        return 'unknown'


def parse_file(s):
    if osp.exists(s) and s != '.':
        assert osp.isfile(s)
        suffix = osp.splitext(s)[1].lower()
        # 添加对webp的支持
        if suffix == '.webp':
            return ('image/webp', s)
        mime = mimetypes.types_map.get(suffix, 'unknown')
        if mime == 'unknown':
            mime = _mime_from_pillow(s)
        return (mime, s)
    elif s.startswith('data:image/'):
        # To be compatible with OPENAI base64 format
        content = s[11:]
        mime = content.split(';')[0]
        content = ';'.join(content.split(';')[1:])
        dname = osp.join(LMUDataRoot(), 'files')
        assert content.startswith('base64,')
        b64 = content[7:]
        os.makedirs(dname, exist_ok=True)
        tgt = osp.join(dname, md5(b64) + '.png')
        decode_base64_to_image_file(b64, tgt)
        return parse_file(tgt)
    elif validators.url(s):
        suffix = osp.splitext(s)[1].lower()
        # 添加对webp的支持
        if suffix == '.webp':
            mime = 'image/webp'
        elif suffix in mimetypes.types_map:
            mime = mimetypes.types_map[suffix]
            dname = osp.join(LMUDataRoot(), 'files')
            os.makedirs(dname, exist_ok=True)
            tgt = osp.join(dname, md5(s) + suffix)
            download_file(s, tgt)
            return (mime, tgt)
        else:
            return ('url', s)

    else:
        return (None, s)

Metadata

Metadata

Assignees

No one assigned

    Labels

    No labels
    No labels

    Type

    No type
    No fields configured for issues without a type.

    Projects

    No projects

    Milestone

    No milestone

    Relationships

    None yet

    Development

    No branches or pull requests

    Issue actions