I tried to generate the onnx and quantize version using the following code. But it failed.

import torch
import os
import onnx
import logging
import argparse
import json
from pathlib import Path
from onnxruntime.quantization import quantize_dynamic, QuantType
# 导入官方模块
from kokoro import KModel, KPipeline
from kokoro.model import KModelForONNX

# --- 0. 配置和日志设定 ---
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

# --- 1. 命令行参数解析 ---

def parse_args():
    """解析命令行参数，设置文件路径和输出目录。"""
    parser = argparse.ArgumentParser(description="Kokoro PyTorch 模型到 ONNX 及其量化版本导出工具。")
    # 更改参数名以匹配官方脚本的习惯，但保持默认值不变
    parser.add_argument(
        "--checkpoint_path", 
        "-p",
        type=str, 
        default="kokoro-v1_1-zh.pth", 
        help="PyTorch 权重文件 (.pth) 的路径。"
    )
    parser.add_argument(
        "--config_file", 
        "-c",
        type=str, 
        default="config.json", 
        help="模型配置文件 (.json) 的路径。"
    )
    parser.add_argument(
        "--output_dir", 
        "-o",
        type=str, 
        default="./", 
        help="ONNX 模型输出目录。"
    )
    return parser.parse_args()

# --- 2. 模型加载和准备 (按照官方源码修正) ---

def load_kokoro_model(pth_path, config_path):
    """
    根据官方源码的模式，使用文件路径直接初始化 KModel，再用其初始化 KModelForONNX 包装器。
    """
    pth_file = Path(pth_path)
    config_file = Path(config_path)

    if not pth_file.exists() or not config_file.exists():
        logger.error(f"文件未找到: pth={pth_path}, config={config_path}")
        raise FileNotFoundError("请检查模型权重和配置文件路径是否正确。")

    logger.info(f"使用官方方法初始化 KModel: config={config_path}, checkpoint={pth_path}")
    
    # 1. 初始化核心模型 KModel (使用文件路径，而不是配置字典)
    # 官方源码：kmodel = KModel(config=config_file, model=checkpoint_path, disable_complex=True)
    kmodel = KModel(config=config_path, model=pth_path, disable_complex=True) 
    logger.info("✅ KModel 核心模型实例化和权重加载成功。")
        
    # 2. 实例化 ONNX 包装器 KModelForONNX
    model_for_onnx = KModelForONNX(kmodel).eval()
    logger.info("✅ KModelForONNX 包装器实例化成功。")

    # 3. 读取配置，用于后续的动态轴和 FP16 转换的 STYLE_DIM
    with open(config_file, 'r', encoding='utf-8') as f:
        config_data = json.load(f)
        
    return model_for_onnx, config_data

# --- 3. ONNX 导出函数 (FP32) ---

# 根据官方源码 export_onnx 函数修改输入参数，不再使用 text_lengths
def export_to_onnx_fp32(model, output_path, config_data):
    """
    将 PyTorch 模型导出为 FP32 ONNX 模型。
    输入参数修改为官方 export_onnx 中的 (input_ids, style, speed)。
    """
    logger.info(f"\n--- 1. 开始导出 FP32 ONNX 模型到 {output_path}...")
    
    # 从 config_data 中获取风格维度
    STYLE_DIM = config_data.get('style_dim', 256) 
    
    # 构造 Dummy Input: (input_ids, style, speed) - 匹配官方源码
    # input_ids: [1, L]
    input_ids = torch.randint(1, 100, (48,)).numpy()
    dummy_input_ids = torch.LongTensor([[0, *input_ids, 0]]) # [1, 50] (包含首尾 token)
    # style: [1, STYLE_DIM]
    dummy_style = torch.randn(1, STYLE_DIM, dtype=torch.float32)
    # speed: [1]
    dummy_speed = torch.IntTensor([1]) # 注意这里使用 IntTensor 匹配官方源码

    dummy_input = (dummy_input_ids, dummy_style, dummy_speed)
    
    input_names = ["input_ids", "style", "speed"]
    output_names = ["waveform", "duration"] # 匹配官方源码的输出名
    
    # 定义动态轴 (匹配官方源码)
    dynamic_axes = {
        "input_ids": {1: "input_ids_len"},
        "waveform": {0: "num_samples"}, # 官方源码只将 waveform 的第一个轴设为动态
    }

    try:
        torch.onnx.export(
            model,
            dummy_input,
            output_path,
            export_params=True,
            opset_version=17, 
            do_constant_folding=True,
            input_names=input_names,
            output_names=output_names,
            dynamic_axes=dynamic_axes,
            verbose=False,
        )
        logger.info(f"✅ FP32 ONNX 模型导出成功: {output_path}")
        return True
    except Exception as e:
        logger.error(f"❌ FP32 ONNX 导出失败: {e}")
        return False

# --- 4. 量化函数 (INT8 动态量化) ---

# ... (quantize_dynamic_onnx 保持不变) ...
def quantize_dynamic_onnx(fp32_path, int8_path):
    """使用 ONNX Runtime Tools 进行动态 INT8 量化。"""
    logger.info(f"\n--- 2. 开始 INT8 动态量化到 {int8_path}...")
    try:
        quantize_dynamic(
            model_input=fp32_path,
            model_output=int8_path,
            optimize_model=True, 
            per_channel=False,    
            weight_type=QuantType.QInt8, 
        )
        logger.info(f"✅ INT8 动态量化模型生成成功: {int8_path}")
        return True
    except Exception as e:
        logger.error(f"❌ INT8 动态量化失败: {e}")
        return False

# --- 5. 半精度优化/转换 (FP16) ---

# ... (convert_to_fp16_onnx 保持不变，但输入数据需要匹配新的格式) ...
def convert_to_fp16_onnx(fp32_path, fp16_path, config_data):
    """将 ONNX 模型转换为 FP16 格式（依赖 onnxsim 库）。"""
    logger.info(f"\n--- 3. 开始 FP16 半精度转换到 {fp16_path}...")
    
    try:
        import onnxsim
        STYLE_DIM = config_data.get('style_dim', 256) 
        
        # 构造输入数据字典 (匹配 export_to_onnx_fp32 中的格式)
        input_data = {
            "input_ids": torch.LongTensor([[0] * 50]).numpy(), # [1, 50]
            "style": torch.randn(1, STYLE_DIM).numpy(),
            "speed": torch.IntTensor([1]).numpy()
        }
        
        onnx_model = onnx.load(fp32_path)
        
        # 简化并转换为 FP16
        model_fp16, check_ok = onnxsim.simplify(
            onnx_model, 
            input_data=input_data, 
            dynamic_input_shape=True, 
            convert_to_fp16=True
        )

        if not check_ok:
            logger.error("FP16 转换简化未能通过一致性检查。")
            return False

        onnx.save(model_fp16, fp16_path)
        logger.info(f"✅ FP16 优化模型生成成功: {fp16_path}")
        return True

    except ImportError:
        logger.warning("❌ onnxsim 库未安装。跳过 FP16 转换。请运行 'pip install onnxsim'。")
        return False
    except Exception as e:
        logger.error(f"❌ FP16 转换失败: {e}")
        return False


# --- 6. 执行主流程 ---

if __name__ == '__main__':
    args = parse_args()
    
    output_dir = Path(args.output_dir)
    output_dir.mkdir(exist_ok=True)
    
    FP32_ONNX_PATH = output_dir / "kokoro_fp32.onnx"
    INT8_ONNX_PATH = output_dir / "kokoro_int8_dynamic_quant.onnx"
    FP16_ONNX_PATH = output_dir / "kokoro_fp16_optimized.onnx"
    
    try:
        # 1. 核心模型初始化 (使用官方方法)
        model, config_data = load_kokoro_model(args.checkpoint_path, args.config_file)
    except Exception:
        # 错误已在 load_kokoro_model 中详细记录
        exit()

    # 2. 导出 FP32 基准模型
    if export_to_onnx_fp32(model, FP32_ONNX_PATH, config_data):
        
        # 3. 动态量化到 INT8
        quantize_dynamic_onnx(FP32_ONNX_PATH, INT8_ONNX_PATH)

        # 4. 优化到 FP16
        convert_to_fp16_onnx(FP32_ONNX_PATH, FP16_ONNX_PATH, config_data)
    
    logger.info("\n==============================================")
    logger.info("🎉 所有转换和量化流程完成！请检查输出目录。")
    logger.info("==============================================")

Details

(kokoro-82m-v1-1-zh) C:\Users\admin\code\other\Kokoro-82M-v1.1-zh>python kokoro_export.py
INFO:__main__:使用官方方法初始化 KModel: config=config.json, checkpoint=kokoro-v1_1-zh.pth
WARNING: Defaulting repo_id to hexgrad/Kokoro-82M. Pass repo_id='hexgrad/Kokoro-82M' to suppress this warning.
C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\rnn.py:123: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1
  warnings.warn(
C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\utils\weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)
INFO:__main__:✅ KModel 核心模型实例化和权重加载成功。
INFO:__main__:✅ KModelForONNX 包装器实例化成功。
INFO:__main__:
--- 1. 开始导出 FP32 ONNX 模型到 kokoro_fp32.onnx...
C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\kokoro_export.py:110: UserWarning: # 'dynamic_axes' is not recommended when dynamo=True, and may lead to 'torch._dynamo.exc.UserError: Constraints violated.' Supply the 'dynamic_shapes' argument instead if export is unsuccessful.
  torch.onnx.export(
W1209 15:35:38.378000 20636 .venv\Lib\site-packages\torch\onnx\_internal\exporter\_compat.py:114] Setting ONNX exporter to use operator set version 18 because the requested opset_version 17 is a lower version than we have implementations for. Automatic version conversion will be performed, which may not be successful at converting to the requested version. If version conversion is unsuccessful, the opset version of the exported model will be kept at 18. Please consider setting opset_version >=18 to leverage latest ONNX features
W1209 15:35:38.767000 20636 .venv\Lib\site-packages\torch\onnx\_internal\exporter\_registration.py:107] torchvision is not installed. Skipping torchvision::nms



def forward(self, arg0_1: "f32[178, 128]", arg1_1: "f32[512, 128]", arg2_1: "f32[2, 128]", arg3_1: "f32[128]", arg4_1: "f32[128]", arg5_1: "f32[768, 128]", arg6_1: "f32[768]", arg7_1: "f32[768]", arg8_1: "f32[768]", arg9_1: "f32[768, 768]", arg10_1: "f32[768]", arg11_1: "f32[768, 768]", arg12_1: "f32[768]", arg13_1: "f32[768, 768]", arg14_1: "f32[768]", arg15_1: "f32[768, 768]", arg16_1: "f32[768]", arg17_1: "f32[768]", arg18_1: "f32[768]", arg19_1: "f32[2048, 768]", arg20_1: "f32[2048]", arg21_1: "f32[768, 2048]", arg22_1: "f32[768]", arg23_1: "f32[768, 768]", arg24_1: "f32[768]", arg25_1: "f32[512, 768]", arg26_1: "f32[512]", arg27_1: "f32[1024, 640]", arg28_1: "f32[1024, 256]", arg29_1: "f32[1024]", arg30_1: "f32[1024]", arg31_1: "f32[1024, 640]", arg32_1: "f32[1024, 256]", arg33_1: "f32[1024]", arg34_1: "f32[1024]", arg35_1: "f32[1024, 128]", arg36_1: "f32[1024]", arg37_1: "f32[1024, 640]", arg38_1: "f32[1024, 256]", arg39_1: "f32[1024]", arg40_1: "f32[1024]", arg41_1: "f32[1024, 640]", arg42_1: "f32[1024, 256]", arg43_1: "f32[1024]", arg44_1: "f32[1024]", arg45_1: "f32[1024, 128]", arg46_1: "f32[1024]", arg47_1: "f32[1024, 640]", arg48_1: "f32[1024, 256]", arg49_1: "f32[1024]", arg50_1: "f32[1024]", arg51_1: "f32[1024, 640]", arg52_1: "f32[1024, 256]", arg53_1: "f32[1024]", arg54_1: "f32[1024]", arg55_1: "f32[1024, 128]", arg56_1: "f32[1024]", arg57_1: "f32[1024, 640]", arg58_1: "f32[1024, 256]", arg59_1: "f32[1024]", arg60_1: "f32[1024]", arg61_1: "f32[1024, 640]", arg62_1: "f32[1024, 256]", arg63_1: "f32[1024]", arg64_1: "f32[1024]", arg65_1: "f32[50, 512]", arg66_1: "f32[50]", arg67_1: "f32[1024, 640]", arg68_1: "f32[1024, 256]", arg69_1: "f32[1024]", arg70_1: "f32[1024]", arg71_1: "f32[1024, 640]", arg72_1: "f32[1024, 256]", arg73_1: "f32[1024]", arg74_1: "f32[1024]", arg75_1: "f32[512]", arg76_1: "f32[512, 1, 1]", arg77_1: "f32[512, 512, 3]", arg78_1: "f32[512]", arg79_1: "f32[512, 1, 1]", arg80_1: "f32[512, 512, 3]", arg81_1: "f32[512]", arg82_1: "f32[512]", arg83_1: "f32[1024, 128]", arg84_1: "f32[1024]", arg85_1: "f32[512]", arg86_1: "f32[512]", arg87_1: "f32[1024, 128]", arg88_1: "f32[1024]", arg89_1: "f32[256]", arg90_1: "f32[256, 1, 1]", arg91_1: "f32[256, 512, 3]", arg92_1: "f32[256]", arg93_1: "f32[256, 1, 1]", arg94_1: "f32[256, 256, 3]", arg95_1: "f32[512]", arg96_1: "f32[512]", arg97_1: "f32[1024, 128]", arg98_1: "f32[1024]", arg99_1: "f32[256]", arg100_1: "f32[256]", arg101_1: "f32[512, 128]", arg102_1: "f32[512]", arg103_1: "f32[256, 1, 1]", arg104_1: "f32[256, 512, 1]", arg105_1: "f32[512]", arg106_1: "f32[512, 1, 1]", arg107_1: "f32[512, 1, 3]", arg108_1: "f32[256]", arg109_1: "f32[256, 1, 1]", arg110_1: "f32[256, 256, 3]", arg111_1: "f32[256]", arg112_1: "f32[256, 1, 1]", arg113_1: "f32[256, 256, 3]", arg114_1: "f32[256]", arg115_1: "f32[256]", arg116_1: "f32[512, 128]", arg117_1: "f32[512]", arg118_1: "f32[256]", arg119_1: "f32[256]", arg120_1: "f32[512, 128]", arg121_1: "f32[512]", arg122_1: "f32[512]", arg123_1: "f32[512, 1, 1]", arg124_1: "f32[512, 512, 3]", arg125_1: "f32[512]", arg126_1: "f32[512, 1, 1]", arg127_1: "f32[512, 512, 3]", arg128_1: "f32[512]", arg129_1: "f32[512]", arg130_1: "f32[1024, 128]", arg131_1: "f32[1024]", arg132_1: "f32[512]", arg133_1: "f32[512]", arg134_1: "f32[1024, 128]", arg135_1: "f32[1024]", arg136_1: "f32[256]", arg137_1: "f32[256, 1, 1]", arg138_1: "f32[256, 512, 3]", arg139_1: "f32[256]", arg140_1: "f32[256, 1, 1]", arg141_1: "f32[256, 256, 3]", arg142_1: "f32[512]", arg143_1: "f32[512]", arg144_1: "f32[1024, 128]", arg145_1: "f32[1024]", arg146_1: "f32[256]", arg147_1: "f32[256]", arg148_1: "f32[512, 128]", arg149_1: "f32[512]", arg150_1: "f32[256, 1, 1]", arg151_1: "f32[256, 512, 1]", arg152_1: "f32[512]", arg153_1: "f32[512, 1, 1]", arg154_1: "f32[512, 1, 3]", arg155_1: "f32[256]", arg156_1: "f32[256, 1, 1]", arg157_1: "f32[256, 256, 3]", arg158_1: "f32[256]", arg159_1: "f32[256, 1, 1]", arg160_1: "f32[256, 256, 3]", arg161_1: "f32[256]", arg162_1: "f32[256]", arg163_1: "f32[512, 128]", arg164_1: "f32[512]", arg165_1: "f32[256]", arg166_1: "f32[256]", arg167_1: "f32[512, 128]", arg168_1: "f32[512]", arg169_1: "f32[1, 256, 1]", arg170_1: "f32[1]", arg171_1: "f32[1, 256, 1]", arg172_1: "f32[1]", arg173_1: "f32[178, 512]", arg174_1: "f32[512]", arg175_1: "f32[512, 1, 1]", arg176_1: "f32[512, 512, 5]", arg177_1: "f32[512]", arg178_1: "f32[512]", arg179_1: "f32[512]", arg180_1: "f32[512, 1, 1]", arg181_1: "f32[512, 512, 5]", arg182_1: "f32[512]", arg183_1: "f32[512]", arg184_1: "f32[512]", arg185_1: "f32[512, 1, 1]", arg186_1: "f32[512, 512, 5]", arg187_1: "f32[512]", arg188_1: "f32[512]", arg189_1: "f32[1024, 512]", arg190_1: "f32[1024, 256]", arg191_1: "f32[1024]", arg192_1: "f32[1024]", arg193_1: "f32[1024, 512]", arg194_1: "f32[1024, 256]", arg195_1: "f32[1024]", arg196_1: "f32[1024]", arg197_1: "f32[1024]", arg198_1: "f32[1024, 1, 1]", arg199_1: "f32[1024, 514, 3]", arg200_1: "f32[1024]", arg201_1: "f32[1024, 1, 1]", arg202_1: "f32[1024, 1024, 3]", arg203_1: "f32[514]", arg204_1: "f32[514]", arg205_1: "f32[1028, 128]", arg206_1: "f32[1028]", arg207_1: "f32[1024]", arg208_1: "f32[1024]", arg209_1: "f32[2048, 128]", arg210_1: "f32[2048]", arg211_1: "f32[1024, 1, 1]", arg212_1: "f32[1024, 514, 1]", arg213_1: "f32[1024]", arg214_1: "f32[1024, 1, 1]", arg215_1: "f32[1024, 1090, 3]", arg216_1: "f32[1024]", arg217_1: "f32[1024, 1, 1]", arg218_1: "f32[1024, 1024, 3]", arg219_1: "f32[1090]", arg220_1: "f32[1090]", arg221_1: "f32[2180, 128]", arg222_1: "f32[2180]", arg223_1: "f32[1024]", arg224_1: "f32[1024]", arg225_1: "f32[2048, 128]", arg226_1: "f32[2048]", arg227_1: "f32[1024, 1, 1]", arg228_1: "f32[1024, 1090, 1]", arg229_1: "f32[1024]", arg230_1: "f32[1024, 1, 1]", arg231_1: "f32[1024, 1090, 3]", arg232_1: "f32[1024]", arg233_1: "f32[1024, 1, 1]", arg234_1: "f32[1024, 1024, 3]", arg235_1: "f32[1090]", arg236_1: "f32[1090]", arg237_1: "f32[2180, 128]", arg238_1: "f32[2180]", arg239_1: "f32[1024]", arg240_1: "f32[1024]", arg241_1: "f32[2048, 128]", arg242_1: "f32[2048]", arg243_1: "f32[1024, 1, 1]", arg244_1: "f32[1024, 1090, 1]", arg245_1: "f32[1024]", arg246_1: "f32[1024, 1, 1]", arg247_1: "f32[1024, 1090, 3]", arg248_1: "f32[1024]", arg249_1: "f32[1024, 1, 1]", arg250_1: "f32[1024, 1024, 3]", arg251_1: "f32[1090]", arg252_1: "f32[1090]", arg253_1: "f32[2180, 128]", arg254_1: "f32[2180]", arg255_1: "f32[1024]", arg256_1: "f32[1024]", arg257_1: "f32[2048, 128]", arg258_1: "f32[2048]", arg259_1: "f32[1024, 1, 1]", arg260_1: "f32[1024, 1090, 1]", arg261_1: "f32[512]", arg262_1: "f32[512, 1, 1]", arg263_1: "f32[512, 1090, 3]", arg264_1: "f32[512]", arg265_1: "f32[512, 1, 1]", arg266_1: "f32[512, 512, 3]", arg267_1: "f32[1090]", arg268_1: "f32[1090]", arg269_1: "f32[2180, 128]", arg270_1: "f32[2180]", arg271_1: "f32[512]", arg272_1: "f32[512]", arg273_1: "f32[1024, 128]", arg274_1: "f32[1024]", arg275_1: "f32[512, 1, 1]", arg276_1: "f32[512, 1090, 1]", arg277_1: "f32[1090]", arg278_1: "f32[1090, 1, 1]", arg279_1: "f32[1090, 1, 3]", arg280_1: "f32[1]", arg281_1: "f32[1, 1, 1]", arg282_1: "f32[1, 1, 3]", arg283_1: "f32[1]", arg284_1: "f32[1, 1, 1]", arg285_1: "f32[1, 1, 3]", arg286_1: "f32[64]", arg287_1: "f32[64, 1, 1]", arg288_1: "f32[64, 512, 1]", arg289_1: "f32[1, 9]", arg290_1: "f32[1]", arg291_1: "f32[256, 22, 12]", arg292_1: "f32[256]", arg293_1: "f32[128, 22, 1]", arg294_1: "f32[128]", arg295_1: "f32[256]", arg296_1: "f32[256, 1, 1]", arg297_1: "f32[256, 256, 7]", arg298_1: "f32[256]", arg299_1: "f32[256, 1, 1]", arg300_1: "f32[256, 256, 7]", arg301_1: "f32[256]", arg302_1: "f32[256, 1, 1]", arg303_1: "f32[256, 256, 7]", arg304_1: "f32[256]", arg305_1: "f32[256, 1, 1]", arg306_1: "f32[256, 256, 7]", arg307_1: "f32[256]", arg308_1: "f32[256, 1, 1]", arg309_1: "f32[256, 256, 7]", arg310_1: "f32[256]", arg311_1: "f32[256, 1, 1]", arg312_1: "f32[256, 256, 7]", arg313_1: "f32[256]", arg314_1: "f32[256]", arg315_1: "f32[512, 128]", arg316_1: "f32[512]", arg317_1: "f32[256]", arg318_1: "f32[256]", arg319_1: "f32[512, 128]", arg320_1: "f32[512]", arg321_1: "f32[256]", arg322_1: "f32[256]", arg323_1: "f32[512, 128]", arg324_1: "f32[512]", arg325_1: "f32[256]", arg326_1: "f32[256]", arg327_1: "f32[512, 128]", arg328_1: "f32[512]", arg329_1: "f32[256]", arg330_1: "f32[256]", arg331_1: "f32[512, 128]", arg332_1: "f32[512]", arg333_1: "f32[256]", arg334_1: "f32[256]", arg335_1: "f32[512, 128]", arg336_1: "f32[512]", arg337_1: "f32[1, 256, 1]", arg338_1: "f32[1, 256, 1]", arg339_1: "f32[1, 256, 1]", arg340_1: "f32[1, 256, 1]", arg341_1: "f32[1, 256, 1]", arg342_1: "f32[1, 256, 1]", arg343_1: "f32[128]", arg344_1: "f32[128, 1, 1]", arg345_1: "f32[128, 128, 11]", arg346_1: "f32[128]", arg347_1: "f32[128, 1, 1]", arg348_1: "f32[128, 128, 11]", arg349_1: "f32[128]", arg350_1: "f32[128, 1, 1]", arg351_1: "f32[128, 128, 11]", arg352_1: "f32[128]", arg353_1: "f32[128, 1, 1]", arg354_1: "f32[128, 128, 11]", arg355_1: "f32[128]", arg356_1: "f32[128, 1, 1]", arg357_1: "f32[128, 128, 11]", arg358_1: "f32[128]", arg359_1: "f32[128, 1, 1]", arg360_1: "f32[128, 128, 11]", arg361_1: "f32[128]", arg362_1: "f32[128]", arg363_1: "f32[256, 128]", arg364_1: "f32[256]", arg365_1: "f32[128]", arg366_1: "f32[128]", arg367_1: "f32[256, 128]", arg368_1: "f32[256]", arg369_1: "f32[128]", arg370_1: "f32[128]", arg371_1: "f32[256, 128]", arg372_1: "f32[256]", arg373_1: "f32[128]", arg374_1: "f32[128]", arg375_1: "f32[256, 128]", arg376_1: "f32[256]", arg377_1: "f32[128]", arg378_1: "f32[128]", arg379_1: "f32[256, 128]", arg380_1: "f32[256]", arg381_1: "f32[128]", arg382_1: "f32[128]", arg383_1: "f32[256, 128]", arg384_1: "f32[256]", arg385_1: "f32[1, 128, 1]", arg386_1: "f32[1, 128, 1]", arg387_1: "f32[1, 128, 1]", arg388_1: "f32[1, 128, 1]", arg389_1: "f32[1, 128, 1]", arg390_1: "f32[1, 128, 1]", arg391_1: "f32[256]", arg392_1: "f32[512, 1, 1]", arg393_1: "f32[512, 256, 20]", arg394_1: "f32[128]", arg395_1: "f32[256, 1, 1]", arg396_1: "f32[256, 128, 12]", arg397_1: "f32[256]", arg398_1: "f32[256, 1, 1]", arg399_1: "f32[256, 256, 3]", arg400_1: "f32[256]", arg401_1: "f32[256, 1, 1]", arg402_1: "f32[256, 256, 3]", arg403_1: "f32[256]", arg404_1: "f32[256, 1, 1]", arg405_1: "f32[256, 256, 3]", arg406_1: "f32[256]", arg407_1: "f32[256, 1, 1]", arg408_1: "f32[256, 256, 3]", arg409_1: "f32[256]", arg410_1: "f32[256, 1, 1]", arg411_1: "f32[256, 256, 3]", arg412_1: "f32[256]", arg413_1: "f32[256, 1, 1]", arg414_1: "f32[256, 256, 3]", arg415_1: "f32[256]", arg416_1: "f32[256]", arg417_1: "f32[512, 128]", arg418_1: "f32[512]", arg419_1: "f32[256]", arg420_1: "f32[256]", arg421_1: "f32[512, 128]", arg422_1: "f32[512]", arg423_1: "f32[256]", arg424_1: "f32[256]", arg425_1: "f32[512, 128]", arg426_1: "f32[512]", arg427_1: "f32[256]", arg428_1: "f32[256]", arg429_1: "f32[512, 128]", arg430_1: "f32[512]", arg431_1: "f32[256]", arg432_1: "f32[256]", arg433_1: "f32[512, 128]", arg434_1: "f32[512]", arg435_1: "f32[256]", arg436_1: "f32[256]", arg437_1: "f32[512, 128]", arg438_1: "f32[512]", arg439_1: "f32[1, 256, 1]", arg440_1: "f32[1, 256, 1]", arg441_1: "f32[1, 256, 1]", arg442_1: "f32[1, 256, 1]", arg443_1: "f32[1, 256, 1]", arg444_1: "f32[1, 256, 1]", arg445_1: "f32[256]", arg446_1: "f32[256, 1, 1]", arg447_1: "f32[256, 256, 7]", arg448_1: "f32[256]", arg449_1: "f32[256, 1, 1]", arg450_1: "f32[256, 256, 7]", arg451_1: "f32[256]", arg452_1: "f32[256, 1, 1]", arg453_1: "f32[256, 256, 7]", arg454_1: "f32[256]", arg455_1: "f32[256, 1, 1]", arg456_1: "f32[256, 256, 7]", arg457_1: "f32[256]", arg458_1: "f32[256, 1, 1]", arg459_1: "f32[256, 256, 7]", arg460_1: "f32[256]", arg461_1: "f32[256, 1, 1]", arg462_1: "f32[256, 256, 7]", arg463_1: "f32[256]", arg464_1: "f32[256]", arg465_1: "f32[512, 128]", arg466_1: "f32[512]", arg467_1: "f32[256]", arg468_1: "f32[256]", arg469_1: "f32[512, 128]", arg470_1: "f32[512]", arg471_1: "f32[256]", arg472_1: "f32[256]", arg473_1: "f32[512, 128]", arg474_1: "f32[512]", arg475_1: "f32[256]", arg476_1: "f32[256]", arg477_1: "f32[512, 128]", arg478_1: "f32[512]", arg479_1: "f32[256]", arg480_1: "f32[256]", arg481_1: "f32[512, 128]", arg482_1: "f32[512]", arg483_1: "f32[256]", arg484_1: "f32[256]", arg485_1: "f32[512, 128]", arg486_1: "f32[512]", arg487_1: "f32[1, 256, 1]", arg488_1: "f32[1, 256, 1]", arg489_1: "f32[1, 256, 1]", arg490_1: "f32[1, 256, 1]", arg491_1: "f32[1, 256, 1]", arg492_1: "f32[1, 256, 1]", arg493_1: "f32[256]", arg494_1: "f32[256, 1, 1]", arg495_1: "f32[256, 256, 11]", arg496_1: "f32[256]", arg497_1: "f32[256, 1, 1]", arg498_1: "f32[256, 256, 11]", arg499_1: "f32[256]", arg500_1: "f32[256, 1, 1]", arg501_1: "f32[256, 256, 11]", arg502_1: "f32[256]", arg503_1: "f32[256, 1, 1]", arg504_1: "f32[256, 256, 11]", arg505_1: "f32[256]", arg506_1: "f32[256, 1, 1]", arg507_1: "f32[256, 256, 11]", arg508_1: "f32[256]", arg509_1: "f32[256, 1, 1]", arg510_1: "f32[256, 256, 11]", arg511_1: "f32[256]", arg512_1: "f32[256]", arg513_1: "f32[512, 128]", arg514_1: "f32[512]", arg515_1: "f32[256]", arg516_1: "f32[256]", arg517_1: "f32[512, 128]", arg518_1: "f32[512]", arg519_1: "f32[256]", arg520_1: "f32[256]", arg521_1: "f32[512, 128]", arg522_1: "f32[512]", arg523_1: "f32[256]", arg524_1: "f32[256]", arg525_1: "f32[512, 128]", arg526_1: "f32[512]", arg527_1: "f32[256]", arg528_1: "f32[256]", arg529_1: "f32[512, 128]", arg530_1: "f32[512]", arg531_1: "f32[256]", arg532_1: "f32[256]", arg533_1: "f32[512, 128]", arg534_1: "f32[512]", arg535_1: "f32[1, 256, 1]", arg536_1: "f32[1, 256, 1]", arg537_1: "f32[1, 256, 1]", arg538_1: "f32[1, 256, 1]", arg539_1: "f32[1, 256, 1]", arg540_1: "f32[1, 256, 1]", arg541_1: "f32[128]", arg542_1: "f32[128, 1, 1]", arg543_1: "f32[128, 128, 3]", arg544_1: "f32[128]", arg545_1: "f32[128, 1, 1]", arg546_1: "f32[128, 128, 3]", arg547_1: "f32[128]", arg548_1: "f32[128, 1, 1]", arg549_1: "f32[128, 128, 3]", arg550_1: "f32[128]", arg551_1: "f32[128, 1, 1]", arg552_1: "f32[128, 128, 3]", arg553_1: "f32[128]", arg554_1: "f32[128, 1, 1]", arg555_1: "f32[128, 128, 3]", arg556_1: "f32[128]", arg557_1: "f32[128, 1, 1]", arg558_1: "f32[128, 128, 3]", arg559_1: "f32[128]", arg560_1: "f32[128]", arg561_1: "f32[256, 128]", arg562_1: "f32[256]", arg563_1: "f32[128]", arg564_1: "f32[128]", arg565_1: "f32[256, 128]", arg566_1: "f32[256]", arg567_1: "f32[128]", arg568_1: "f32[128]", arg569_1: "f32[256, 128]", arg570_1: "f32[256]", arg571_1: "f32[128]", arg572_1: "f32[128]", arg573_1: "f32[256, 128]", arg574_1: "f32[256]", arg575_1: "f32[128]", arg576_1: "f32[128]", arg577_1: "f32[256, 128]", arg578_1: "f32[256]", arg579_1: "f32[128]", arg580_1: "f32[128]", arg581_1: "f32[256, 128]", arg582_1: "f32[256]", arg583_1: "f32[1, 128, 1]", arg584_1: "f32[1, 128, 1]", arg585_1: "f32[1, 128, 1]", arg586_1: "f32[1, 128, 1]", arg587_1: "f32[1, 128, 1]", arg588_1: "f32[1, 128, 1]", arg589_1: "f32[128]", arg590_1: "f32[128, 1, 1]", arg591_1: "f32[128, 128, 7]", arg592_1: "f32[128]", arg593_1: "f32[128, 1, 1]", arg594_1: "f32[128, 128, 7]", arg595_1: "f32[128]", arg596_1: "f32[128, 1, 1]", arg597_1: "f32[128, 128, 7]", arg598_1: "f32[128]", arg599_1: "f32[128, 1, 1]", arg600_1: "f32[128, 128, 7]", arg601_1: "f32[128]", arg602_1: "f32[128, 1, 1]", arg603_1: "f32[128, 128, 7]", arg604_1: "f32[128]", arg605_1: "f32[128, 1, 1]", arg606_1: "f32[128, 128, 7]", arg607_1: "f32[128]", arg608_1: "f32[128]", arg609_1: "f32[256, 128]", arg610_1: "f32[256]", arg611_1: "f32[128]", arg612_1: "f32[128]", arg613_1: "f32[256, 128]", arg614_1: "f32[256]", arg615_1: "f32[128]", arg616_1: "f32[128]", arg617_1: "f32[256, 128]", arg618_1: "f32[256]", arg619_1: "f32[128]", arg620_1: "f32[128]", arg621_1: "f32[256, 128]", arg622_1: "f32[256]", arg623_1: "f32[128]", arg624_1: "f32[128]", arg625_1: "f32[256, 128]", arg626_1: "f32[256]", arg627_1: "f32[128]", arg628_1: "f32[128]", arg629_1: "f32[256, 128]", arg630_1: "f32[256]", arg631_1: "f32[1, 128, 1]", arg632_1: "f32[1, 128, 1]", arg633_1: "f32[1, 128, 1]", arg634_1: "f32[1, 128, 1]", arg635_1: "f32[1, 128, 1]", arg636_1: "f32[1, 128, 1]", arg637_1: "f32[128]", arg638_1: "f32[128, 1, 1]", arg639_1: "f32[128, 128, 11]", arg640_1: "f32[128]", arg641_1: "f32[128, 1, 1]", arg642_1: "f32[128, 128, 11]", arg643_1: "f32[128]", arg644_1: "f32[128, 1, 1]", arg645_1: "f32[128, 128, 11]", arg646_1: "f32[128]", arg647_1: "f32[128, 1, 1]", arg648_1: "f32[128, 128, 11]", arg649_1: "f32[128]", arg650_1: "f32[128, 1, 1]", arg651_1: "f32[128, 128, 11]", arg652_1: "f32[128]", arg653_1: "f32[128, 1, 1]", arg654_1: "f32[128, 128, 11]", arg655_1: "f32[128]", arg656_1: "f32[128]", arg657_1: "f32[256, 128]", arg658_1: "f32[256]", arg659_1: "f32[128]", arg660_1: "f32[128]", arg661_1: "f32[256, 128]", arg662_1: "f32[256]", arg663_1: "f32[128]", arg664_1: "f32[128]", arg665_1: "f32[256, 128]", arg666_1: "f32[256]", arg667_1: "f32[128]", arg668_1: "f32[128]", arg669_1: "f32[256, 128]", arg670_1: "f32[256]", arg671_1: "f32[128]", arg672_1: "f32[128]", arg673_1: "f32[256, 128]", arg674_1: "f32[256]", arg675_1: "f32[128]", arg676_1: "f32[128]", arg677_1: "f32[256, 128]", arg678_1: "f32[256]", arg679_1: "f32[1, 128, 1]", arg680_1: "f32[1, 128, 1]", arg681_1: "f32[1, 128, 1]", arg682_1: "f32[1, 128, 1]", arg683_1: "f32[1, 128, 1]", arg684_1: "f32[1, 128, 1]", arg685_1: "f32[22]", arg686_1: "f32[22, 1, 1]", arg687_1: "f32[22, 128, 7]", arg688_1: "i64[1, 512]", arg689_1: "i64[1, 512]", arg690_1: "f32[20]", arg691_1: "f32[11, 1, 20]", arg692_1: "f32[11, 1, 20]", arg693_1: "f32[11, 1, 20]", arg694_1: "f32[11, 1, 20]", arg695_1: "i64[1, s70]", arg696_1: "f32[1, 128]", arg697_1: "i32[1]"):
     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\kokoro\model.py:150 in forward, code: waveform, duration = self.kmodel.forward_with_tokens(input_ids, ref_s, speed)
    _set_grad_enabled = torch._C._set_grad_enabled(False);  _set_grad_enabled = None
    sym_size_int: "Sym(s70)" = torch.ops.aten.sym_size.int(arg695_1, 1)
    full: "i64[1]" = torch.ops.aten.full.default([1], sym_size_int, dtype = torch.int64, device = device(type='cpu'), pin_memory = False)
    max_1: "i64[]" = torch.ops.aten.max.default(full)
    item: "Sym(u0)" = torch.ops.aten.item.default(max_1);  max_1 = None
    arange: "i64[u0]" = torch.ops.aten.arange.default(item, device = device(type='cpu'), pin_memory = False);  item = None
    unsqueeze: "i64[1, u0]" = torch.ops.aten.unsqueeze.default(arange, 0)
    expand: "i64[1, u0]" = torch.ops.aten.expand.default(unsqueeze, [1, -1]);  unsqueeze = None
    type_as: "i64[1, u0]" = torch.ops.aten.type_as.default(expand, full);  expand = None
    add: "i64[1, u0]" = torch.ops.aten.add.Tensor(type_as, 1);  type_as = None
    unsqueeze_1: "i64[1, 1]" = torch.ops.aten.unsqueeze.default(full, 1);  full = None
    gt: "b8[1, u0]" = torch.ops.aten.gt.Tensor(add, unsqueeze_1);  add = unsqueeze_1 = None
    to: "b8[1, u0]" = torch.ops.aten.to.dtype_layout(gt, dtype = torch.bool, layout = torch.strided, device = device(type='cpu'));  gt = None
    bitwise_not: "b8[1, u0]" = torch.ops.aten.bitwise_not.default(to);  to = None
    to_1: "i32[1, u0]" = torch.ops.aten.to.dtype(bitwise_not, torch.int32);  bitwise_not = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:684 in forward, code: buffered_token_type_ids = self.embeddings.token_type_ids[:, :seq_length]
    slice_1: "i64[1, s70]" = torch.ops.aten.slice.Tensor(arg689_1, 1, None, sym_size_int);  arg689_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:685 in forward, code: buffered_token_type_ids_expanded = buffered_token_type_ids.expand(batch_size, seq_length)
    expand_1: "i64[1, s70]" = torch.ops.aten.expand.default(slice_1, [1, sym_size_int]);  slice_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:218 in forward, code: position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length]
    add_1: "Sym(s70)" = sym_size_int + 0
    slice_2: "i64[1, s70]" = torch.ops.aten.slice.Tensor(arg688_1, 1, 0, add_1);  arg688_1 = add_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\sparse.py:192 in forward, code: return F.embedding(
    embedding: "f32[1, s70, 128]" = torch.ops.aten.embedding.default(arg0_1, arg695_1, 0);  arg0_1 = arg695_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\sparse.py:192 in forward, code: return F.embedding(
    embedding_1: "f32[1, s70, 128]" = torch.ops.aten.embedding.default(arg2_1, expand_1);  arg2_1 = expand_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:235 in forward, code: embeddings = inputs_embeds + token_type_embeddings
    add_2: "f32[1, s70, 128]" = torch.ops.aten.add.Tensor(embedding, embedding_1);  embedding = embedding_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\sparse.py:192 in forward, code: return F.embedding(
    embedding_2: "f32[1, s70, 128]" = torch.ops.aten.embedding.default(arg1_1, slice_2);  arg1_1 = slice_2 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:238 in forward, code: embeddings += position_embeddings
    add_: "f32[1, s70, 128]" = torch.ops.aten.add_.Tensor(add_2, embedding_2);  add_2 = embedding_2 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\normalization.py:229 in forward, code: return F.layer_norm(
    layer_norm: "f32[1, s70, 128]" = torch.ops.aten.layer_norm.default(add_, [128], arg3_1, arg4_1, 1e-12);  add_ = arg3_1 = arg4_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\dropout.py:73 in forward, code: return F.dropout(input, self.p, self.training, self.inplace)
    dropout: "f32[1, s70, 128]" = torch.ops.aten.dropout.default(layer_norm, 0.0, False);  layer_norm = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:702 in forward, code: extended_attention_mask = _prepare_4d_attention_mask_for_sdpa(
    unsqueeze_2: "i32[1, 1, u0]" = torch.ops.aten.unsqueeze.default(to_1, 1);  to_1 = None
    unsqueeze_3: "i32[1, 1, 1, u0]" = torch.ops.aten.unsqueeze.default(unsqueeze_2, 2);  unsqueeze_2 = None
    sym_size_int_1: "Sym(u0)" = torch.ops.aten.sym_size.int(arange, 0);  arange = None
    le: "Sym(u0 <= 9223372036854775807)" = sym_size_int_1 <= 9223372036854775807;  le = None
    slice_3: "i32[1, 1, 1, u0]" = torch.ops.aten.slice.Tensor(unsqueeze_3, 3, 0, 9223372036854775807);  unsqueeze_3 = None
    expand_2: "i32[1, 1, s70, u0]" = torch.ops.aten.expand.default(slice_3, [1, 1, sym_size_int, sym_size_int_1]);  slice_3 = sym_size_int = sym_size_int_1 = None
    to_2: "f32[1, 1, s70, u0]" = torch.ops.aten.to.dtype(expand_2, torch.float32);  expand_2 = None
    _tensor_constant0: "f32[]" = self._tensor_constant0
    lift_fresh_copy: "f32[]" = torch.ops.aten.lift_fresh_copy.default(_tensor_constant0);  _tensor_constant0 = None
    detach_: "f32[]" = torch.ops.aten.detach_.default(lift_fresh_copy);  lift_fresh_copy = None
    sub: "f32[1, 1, s70, u0]" = torch.ops.aten.sub.Tensor(detach_, to_2);  detach_ = to_2 = None
    to_3: "b8[1, 1, s70, u0]" = torch.ops.aten.to.dtype(sub, torch.bool)
    masked_fill: "f32[1, 1, s70, u0]" = torch.ops.aten.masked_fill.Scalar(sub, to_3, -3.4028234663852886e+38);  sub = to_3 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\linear.py:134 in forward, code: return F.linear(input, self.weight, self.bias)
    linear: "f32[1, s70, 768]" = torch.ops.aten.linear.default(dropout, arg5_1, arg6_1);  dropout = arg5_1 = arg6_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\linear.py:134 in forward, code: return F.linear(input, self.weight, self.bias)
    linear_1: "f32[1, s70, 768]" = torch.ops.aten.linear.default(linear, arg9_1, arg10_1);  arg9_1 = arg10_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:379 in forward, code: .view(batch_size, -1, self.num_attention_heads, self.attention_head_size)
    view: "f32[1, s70, 12, 64]" = torch.ops.aten.view.default(linear_1, [1, -1, 12, 64]);  linear_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:380 in forward, code: .transpose(1, 2)
    transpose: "f32[1, 12, s70, 64]" = torch.ops.aten.transpose.int(view, 1, 2);  view = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\linear.py:134 in forward, code: return F.linear(input, self.weight, self.bias)
    linear_2: "f32[1, s70, 768]" = torch.ops.aten.linear.default(linear, arg11_1, arg12_1);  arg11_1 = arg12_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:384 in forward, code: .view(batch_size, -1, self.num_attention_heads, self.attention_head_size)
    view_1: "f32[1, s70, 12, 64]" = torch.ops.aten.view.default(linear_2, [1, -1, 12, 64]);  linear_2 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:385 in forward, code: .transpose(1, 2)
    transpose_1: "f32[1, 12, s70, 64]" = torch.ops.aten.transpose.int(view_1, 1, 2);  view_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\linear.py:134 in forward, code: return F.linear(input, self.weight, self.bias)
    linear_3: "f32[1, s70, 768]" = torch.ops.aten.linear.default(linear, arg13_1, arg14_1);  linear = arg13_1 = arg14_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:389 in forward, code: .view(batch_size, -1, self.num_attention_heads, self.attention_head_size)
    view_2: "f32[1, s70, 12, 64]" = torch.ops.aten.view.default(linear_3, [1, -1, 12, 64]);  linear_3 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:390 in forward, code: .transpose(1, 2)
    transpose_2: "f32[1, 12, s70, 64]" = torch.ops.aten.transpose.int(view_2, 1, 2);  view_2 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:393 in forward, code: attention_output = torch.nn.functional.scaled_dot_product_attention(
    scaled_dot_product_attention = torch.ops.aten.scaled_dot_product_attention.default(transpose, transpose_1, transpose_2, masked_fill);  transpose = transpose_1 = transpose_2 = masked_fill = scaled_dot_product_attention = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\kokoro\model.py:150 in forward, code: waveform, duration = self.kmodel.forward_with_tokens(input_ids, ref_s, speed)
    _set_grad_enabled_1 = torch._C._set_grad_enabled(True);  _set_grad_enabled_1 = None




def forward(self, arg0_1: "f32[178, 128]", arg1_1: "f32[512, 128]", arg2_1: "f32[2, 128]", arg3_1: "f32[128]", arg4_1: "f32[128]", arg5_1: "f32[768, 128]", arg6_1: "f32[768]", arg7_1: "f32[768]", arg8_1: "f32[768]", arg9_1: "f32[768, 768]", arg10_1: "f32[768]", arg11_1: "f32[768, 768]", arg12_1: "f32[768]", arg13_1: "f32[768, 768]", arg14_1: "f32[768]", arg15_1: "f32[768, 768]", arg16_1: "f32[768]", arg17_1: "f32[768]", arg18_1: "f32[768]", arg19_1: "f32[2048, 768]", arg20_1: "f32[2048]", arg21_1: "f32[768, 2048]", arg22_1: "f32[768]", arg23_1: "f32[768, 768]", arg24_1: "f32[768]", arg25_1: "f32[512, 768]", arg26_1: "f32[512]", arg27_1: "f32[1024, 640]", arg28_1: "f32[1024, 256]", arg29_1: "f32[1024]", arg30_1: "f32[1024]", arg31_1: "f32[1024, 640]", arg32_1: "f32[1024, 256]", arg33_1: "f32[1024]", arg34_1: "f32[1024]", arg35_1: "f32[1024, 128]", arg36_1: "f32[1024]", arg37_1: "f32[1024, 640]", arg38_1: "f32[1024, 256]", arg39_1: "f32[1024]", arg40_1: "f32[1024]", arg41_1: "f32[1024, 640]", arg42_1: "f32[1024, 256]", arg43_1: "f32[1024]", arg44_1: "f32[1024]", arg45_1: "f32[1024, 128]", arg46_1: "f32[1024]", arg47_1: "f32[1024, 640]", arg48_1: "f32[1024, 256]", arg49_1: "f32[1024]", arg50_1: "f32[1024]", arg51_1: "f32[1024, 640]", arg52_1: "f32[1024, 256]", arg53_1: "f32[1024]", arg54_1: "f32[1024]", arg55_1: "f32[1024, 128]", arg56_1: "f32[1024]", arg57_1: "f32[1024, 640]", arg58_1: "f32[1024, 256]", arg59_1: "f32[1024]", arg60_1: "f32[1024]", arg61_1: "f32[1024, 640]", arg62_1: "f32[1024, 256]", arg63_1: "f32[1024]", arg64_1: "f32[1024]", arg65_1: "f32[50, 512]", arg66_1: "f32[50]", arg67_1: "f32[1024, 640]", arg68_1: "f32[1024, 256]", arg69_1: "f32[1024]", arg70_1: "f32[1024]", arg71_1: "f32[1024, 640]", arg72_1: "f32[1024, 256]", arg73_1: "f32[1024]", arg74_1: "f32[1024]", arg75_1: "f32[512]", arg76_1: "f32[512, 1, 1]", arg77_1: "f32[512, 512, 3]", arg78_1: "f32[512]", arg79_1: "f32[512, 1, 1]", arg80_1: "f32[512, 512, 3]", arg81_1: "f32[512]", arg82_1: "f32[512]", arg83_1: "f32[1024, 128]", arg84_1: "f32[1024]", arg85_1: "f32[512]", arg86_1: "f32[512]", arg87_1: "f32[1024, 128]", arg88_1: "f32[1024]", arg89_1: "f32[256]", arg90_1: "f32[256, 1, 1]", arg91_1: "f32[256, 512, 3]", arg92_1: "f32[256]", arg93_1: "f32[256, 1, 1]", arg94_1: "f32[256, 256, 3]", arg95_1: "f32[512]", arg96_1: "f32[512]", arg97_1: "f32[1024, 128]", arg98_1: "f32[1024]", arg99_1: "f32[256]", arg100_1: "f32[256]", arg101_1: "f32[512, 128]", arg102_1: "f32[512]", arg103_1: "f32[256, 1, 1]", arg104_1: "f32[256, 512, 1]", arg105_1: "f32[512]", arg106_1: "f32[512, 1, 1]", arg107_1: "f32[512, 1, 3]", arg108_1: "f32[256]", arg109_1: "f32[256, 1, 1]", arg110_1: "f32[256, 256, 3]", arg111_1: "f32[256]", arg112_1: "f32[256, 1, 1]", arg113_1: "f32[256, 256, 3]", arg114_1: "f32[256]", arg115_1: "f32[256]", arg116_1: "f32[512, 128]", arg117_1: "f32[512]", arg118_1: "f32[256]", arg119_1: "f32[256]", arg120_1: "f32[512, 128]", arg121_1: "f32[512]", arg122_1: "f32[512]", arg123_1: "f32[512, 1, 1]", arg124_1: "f32[512, 512, 3]", arg125_1: "f32[512]", arg126_1: "f32[512, 1, 1]", arg127_1: "f32[512, 512, 3]", arg128_1: "f32[512]", arg129_1: "f32[512]", arg130_1: "f32[1024, 128]", arg131_1: "f32[1024]", arg132_1: "f32[512]", arg133_1: "f32[512]", arg134_1: "f32[1024, 128]", arg135_1: "f32[1024]", arg136_1: "f32[256]", arg137_1: "f32[256, 1, 1]", arg138_1: "f32[256, 512, 3]", arg139_1: "f32[256]", arg140_1: "f32[256, 1, 1]", arg141_1: "f32[256, 256, 3]", arg142_1: "f32[512]", arg143_1: "f32[512]", arg144_1: "f32[1024, 128]", arg145_1: "f32[1024]", arg146_1: "f32[256]", arg147_1: "f32[256]", arg148_1: "f32[512, 128]", arg149_1: "f32[512]", arg150_1: "f32[256, 1, 1]", arg151_1: "f32[256, 512, 1]", arg152_1: "f32[512]", arg153_1: "f32[512, 1, 1]", arg154_1: "f32[512, 1, 3]", arg155_1: "f32[256]", arg156_1: "f32[256, 1, 1]", arg157_1: "f32[256, 256, 3]", arg158_1: "f32[256]", arg159_1: "f32[256, 1, 1]", arg160_1: "f32[256, 256, 3]", arg161_1: "f32[256]", arg162_1: "f32[256]", arg163_1: "f32[512, 128]", arg164_1: "f32[512]", arg165_1: "f32[256]", arg166_1: "f32[256]", arg167_1: "f32[512, 128]", arg168_1: "f32[512]", arg169_1: "f32[1, 256, 1]", arg170_1: "f32[1]", arg171_1: "f32[1, 256, 1]", arg172_1: "f32[1]", arg173_1: "f32[178, 512]", arg174_1: "f32[512]", arg175_1: "f32[512, 1, 1]", arg176_1: "f32[512, 512, 5]", arg177_1: "f32[512]", arg178_1: "f32[512]", arg179_1: "f32[512]", arg180_1: "f32[512, 1, 1]", arg181_1: "f32[512, 512, 5]", arg182_1: "f32[512]", arg183_1: "f32[512]", arg184_1: "f32[512]", arg185_1: "f32[512, 1, 1]", arg186_1: "f32[512, 512, 5]", arg187_1: "f32[512]", arg188_1: "f32[512]", arg189_1: "f32[1024, 512]", arg190_1: "f32[1024, 256]", arg191_1: "f32[1024]", arg192_1: "f32[1024]", arg193_1: "f32[1024, 512]", arg194_1: "f32[1024, 256]", arg195_1: "f32[1024]", arg196_1: "f32[1024]", arg197_1: "f32[1024]", arg198_1: "f32[1024, 1, 1]", arg199_1: "f32[1024, 514, 3]", arg200_1: "f32[1024]", arg201_1: "f32[1024, 1, 1]", arg202_1: "f32[1024, 1024, 3]", arg203_1: "f32[514]", arg204_1: "f32[514]", arg205_1: "f32[1028, 128]", arg206_1: "f32[1028]", arg207_1: "f32[1024]", arg208_1: "f32[1024]", arg209_1: "f32[2048, 128]", arg210_1: "f32[2048]", arg211_1: "f32[1024, 1, 1]", arg212_1: "f32[1024, 514, 1]", arg213_1: "f32[1024]", arg214_1: "f32[1024, 1, 1]", arg215_1: "f32[1024, 1090, 3]", arg216_1: "f32[1024]", arg217_1: "f32[1024, 1, 1]", arg218_1: "f32[1024, 1024, 3]", arg219_1: "f32[1090]", arg220_1: "f32[1090]", arg221_1: "f32[2180, 128]", arg222_1: "f32[2180]", arg223_1: "f32[1024]", arg224_1: "f32[1024]", arg225_1: "f32[2048, 128]", arg226_1: "f32[2048]", arg227_1: "f32[1024, 1, 1]", arg228_1: "f32[1024, 1090, 1]", arg229_1: "f32[1024]", arg230_1: "f32[1024, 1, 1]", arg231_1: "f32[1024, 1090, 3]", arg232_1: "f32[1024]", arg233_1: "f32[1024, 1, 1]", arg234_1: "f32[1024, 1024, 3]", arg235_1: "f32[1090]", arg236_1: "f32[1090]", arg237_1: "f32[2180, 128]", arg238_1: "f32[2180]", arg239_1: "f32[1024]", arg240_1: "f32[1024]", arg241_1: "f32[2048, 128]", arg242_1: "f32[2048]", arg243_1: "f32[1024, 1, 1]", arg244_1: "f32[1024, 1090, 1]", arg245_1: "f32[1024]", arg246_1: "f32[1024, 1, 1]", arg247_1: "f32[1024, 1090, 3]", arg248_1: "f32[1024]", arg249_1: "f32[1024, 1, 1]", arg250_1: "f32[1024, 1024, 3]", arg251_1: "f32[1090]", arg252_1: "f32[1090]", arg253_1: "f32[2180, 128]", arg254_1: "f32[2180]", arg255_1: "f32[1024]", arg256_1: "f32[1024]", arg257_1: "f32[2048, 128]", arg258_1: "f32[2048]", arg259_1: "f32[1024, 1, 1]", arg260_1: "f32[1024, 1090, 1]", arg261_1: "f32[512]", arg262_1: "f32[512, 1, 1]", arg263_1: "f32[512, 1090, 3]", arg264_1: "f32[512]", arg265_1: "f32[512, 1, 1]", arg266_1: "f32[512, 512, 3]", arg267_1: "f32[1090]", arg268_1: "f32[1090]", arg269_1: "f32[2180, 128]", arg270_1: "f32[2180]", arg271_1: "f32[512]", arg272_1: "f32[512]", arg273_1: "f32[1024, 128]", arg274_1: "f32[1024]", arg275_1: "f32[512, 1, 1]", arg276_1: "f32[512, 1090, 1]", arg277_1: "f32[1090]", arg278_1: "f32[1090, 1, 1]", arg279_1: "f32[1090, 1, 3]", arg280_1: "f32[1]", arg281_1: "f32[1, 1, 1]", arg282_1: "f32[1, 1, 3]", arg283_1: "f32[1]", arg284_1: "f32[1, 1, 1]", arg285_1: "f32[1, 1, 3]", arg286_1: "f32[64]", arg287_1: "f32[64, 1, 1]", arg288_1: "f32[64, 512, 1]", arg289_1: "f32[1, 9]", arg290_1: "f32[1]", arg291_1: "f32[256, 22, 12]", arg292_1: "f32[256]", arg293_1: "f32[128, 22, 1]", arg294_1: "f32[128]", arg295_1: "f32[256]", arg296_1: "f32[256, 1, 1]", arg297_1: "f32[256, 256, 7]", arg298_1: "f32[256]", arg299_1: "f32[256, 1, 1]", arg300_1: "f32[256, 256, 7]", arg301_1: "f32[256]", arg302_1: "f32[256, 1, 1]", arg303_1: "f32[256, 256, 7]", arg304_1: "f32[256]", arg305_1: "f32[256, 1, 1]", arg306_1: "f32[256, 256, 7]", arg307_1: "f32[256]", arg308_1: "f32[256, 1, 1]", arg309_1: "f32[256, 256, 7]", arg310_1: "f32[256]", arg311_1: "f32[256, 1, 1]", arg312_1: "f32[256, 256, 7]", arg313_1: "f32[256]", arg314_1: "f32[256]", arg315_1: "f32[512, 128]", arg316_1: "f32[512]", arg317_1: "f32[256]", arg318_1: "f32[256]", arg319_1: "f32[512, 128]", arg320_1: "f32[512]", arg321_1: "f32[256]", arg322_1: "f32[256]", arg323_1: "f32[512, 128]", arg324_1: "f32[512]", arg325_1: "f32[256]", arg326_1: "f32[256]", arg327_1: "f32[512, 128]", arg328_1: "f32[512]", arg329_1: "f32[256]", arg330_1: "f32[256]", arg331_1: "f32[512, 128]", arg332_1: "f32[512]", arg333_1: "f32[256]", arg334_1: "f32[256]", arg335_1: "f32[512, 128]", arg336_1: "f32[512]", arg337_1: "f32[1, 256, 1]", arg338_1: "f32[1, 256, 1]", arg339_1: "f32[1, 256, 1]", arg340_1: "f32[1, 256, 1]", arg341_1: "f32[1, 256, 1]", arg342_1: "f32[1, 256, 1]", arg343_1: "f32[128]", arg344_1: "f32[128, 1, 1]", arg345_1: "f32[128, 128, 11]", arg346_1: "f32[128]", arg347_1: "f32[128, 1, 1]", arg348_1: "f32[128, 128, 11]", arg349_1: "f32[128]", arg350_1: "f32[128, 1, 1]", arg351_1: "f32[128, 128, 11]", arg352_1: "f32[128]", arg353_1: "f32[128, 1, 1]", arg354_1: "f32[128, 128, 11]", arg355_1: "f32[128]", arg356_1: "f32[128, 1, 1]", arg357_1: "f32[128, 128, 11]", arg358_1: "f32[128]", arg359_1: "f32[128, 1, 1]", arg360_1: "f32[128, 128, 11]", arg361_1: "f32[128]", arg362_1: "f32[128]", arg363_1: "f32[256, 128]", arg364_1: "f32[256]", arg365_1: "f32[128]", arg366_1: "f32[128]", arg367_1: "f32[256, 128]", arg368_1: "f32[256]", arg369_1: "f32[128]", arg370_1: "f32[128]", arg371_1: "f32[256, 128]", arg372_1: "f32[256]", arg373_1: "f32[128]", arg374_1: "f32[128]", arg375_1: "f32[256, 128]", arg376_1: "f32[256]", arg377_1: "f32[128]", arg378_1: "f32[128]", arg379_1: "f32[256, 128]", arg380_1: "f32[256]", arg381_1: "f32[128]", arg382_1: "f32[128]", arg383_1: "f32[256, 128]", arg384_1: "f32[256]", arg385_1: "f32[1, 128, 1]", arg386_1: "f32[1, 128, 1]", arg387_1: "f32[1, 128, 1]", arg388_1: "f32[1, 128, 1]", arg389_1: "f32[1, 128, 1]", arg390_1: "f32[1, 128, 1]", arg391_1: "f32[256]", arg392_1: "f32[512, 1, 1]", arg393_1: "f32[512, 256, 20]", arg394_1: "f32[128]", arg395_1: "f32[256, 1, 1]", arg396_1: "f32[256, 128, 12]", arg397_1: "f32[256]", arg398_1: "f32[256, 1, 1]", arg399_1: "f32[256, 256, 3]", arg400_1: "f32[256]", arg401_1: "f32[256, 1, 1]", arg402_1: "f32[256, 256, 3]", arg403_1: "f32[256]", arg404_1: "f32[256, 1, 1]", arg405_1: "f32[256, 256, 3]", arg406_1: "f32[256]", arg407_1: "f32[256, 1, 1]", arg408_1: "f32[256, 256, 3]", arg409_1: "f32[256]", arg410_1: "f32[256, 1, 1]", arg411_1: "f32[256, 256, 3]", arg412_1: "f32[256]", arg413_1: "f32[256, 1, 1]", arg414_1: "f32[256, 256, 3]", arg415_1: "f32[256]", arg416_1: "f32[256]", arg417_1: "f32[512, 128]", arg418_1: "f32[512]", arg419_1: "f32[256]", arg420_1: "f32[256]", arg421_1: "f32[512, 128]", arg422_1: "f32[512]", arg423_1: "f32[256]", arg424_1: "f32[256]", arg425_1: "f32[512, 128]", arg426_1: "f32[512]", arg427_1: "f32[256]", arg428_1: "f32[256]", arg429_1: "f32[512, 128]", arg430_1: "f32[512]", arg431_1: "f32[256]", arg432_1: "f32[256]", arg433_1: "f32[512, 128]", arg434_1: "f32[512]", arg435_1: "f32[256]", arg436_1: "f32[256]", arg437_1: "f32[512, 128]", arg438_1: "f32[512]", arg439_1: "f32[1, 256, 1]", arg440_1: "f32[1, 256, 1]", arg441_1: "f32[1, 256, 1]", arg442_1: "f32[1, 256, 1]", arg443_1: "f32[1, 256, 1]", arg444_1: "f32[1, 256, 1]", arg445_1: "f32[256]", arg446_1: "f32[256, 1, 1]", arg447_1: "f32[256, 256, 7]", arg448_1: "f32[256]", arg449_1: "f32[256, 1, 1]", arg450_1: "f32[256, 256, 7]", arg451_1: "f32[256]", arg452_1: "f32[256, 1, 1]", arg453_1: "f32[256, 256, 7]", arg454_1: "f32[256]", arg455_1: "f32[256, 1, 1]", arg456_1: "f32[256, 256, 7]", arg457_1: "f32[256]", arg458_1: "f32[256, 1, 1]", arg459_1: "f32[256, 256, 7]", arg460_1: "f32[256]", arg461_1: "f32[256, 1, 1]", arg462_1: "f32[256, 256, 7]", arg463_1: "f32[256]", arg464_1: "f32[256]", arg465_1: "f32[512, 128]", arg466_1: "f32[512]", arg467_1: "f32[256]", arg468_1: "f32[256]", arg469_1: "f32[512, 128]", arg470_1: "f32[512]", arg471_1: "f32[256]", arg472_1: "f32[256]", arg473_1: "f32[512, 128]", arg474_1: "f32[512]", arg475_1: "f32[256]", arg476_1: "f32[256]", arg477_1: "f32[512, 128]", arg478_1: "f32[512]", arg479_1: "f32[256]", arg480_1: "f32[256]", arg481_1: "f32[512, 128]", arg482_1: "f32[512]", arg483_1: "f32[256]", arg484_1: "f32[256]", arg485_1: "f32[512, 128]", arg486_1: "f32[512]", arg487_1: "f32[1, 256, 1]", arg488_1: "f32[1, 256, 1]", arg489_1: "f32[1, 256, 1]", arg490_1: "f32[1, 256, 1]", arg491_1: "f32[1, 256, 1]", arg492_1: "f32[1, 256, 1]", arg493_1: "f32[256]", arg494_1: "f32[256, 1, 1]", arg495_1: "f32[256, 256, 11]", arg496_1: "f32[256]", arg497_1: "f32[256, 1, 1]", arg498_1: "f32[256, 256, 11]", arg499_1: "f32[256]", arg500_1: "f32[256, 1, 1]", arg501_1: "f32[256, 256, 11]", arg502_1: "f32[256]", arg503_1: "f32[256, 1, 1]", arg504_1: "f32[256, 256, 11]", arg505_1: "f32[256]", arg506_1: "f32[256, 1, 1]", arg507_1: "f32[256, 256, 11]", arg508_1: "f32[256]", arg509_1: "f32[256, 1, 1]", arg510_1: "f32[256, 256, 11]", arg511_1: "f32[256]", arg512_1: "f32[256]", arg513_1: "f32[512, 128]", arg514_1: "f32[512]", arg515_1: "f32[256]", arg516_1: "f32[256]", arg517_1: "f32[512, 128]", arg518_1: "f32[512]", arg519_1: "f32[256]", arg520_1: "f32[256]", arg521_1: "f32[512, 128]", arg522_1: "f32[512]", arg523_1: "f32[256]", arg524_1: "f32[256]", arg525_1: "f32[512, 128]", arg526_1: "f32[512]", arg527_1: "f32[256]", arg528_1: "f32[256]", arg529_1: "f32[512, 128]", arg530_1: "f32[512]", arg531_1: "f32[256]", arg532_1: "f32[256]", arg533_1: "f32[512, 128]", arg534_1: "f32[512]", arg535_1: "f32[1, 256, 1]", arg536_1: "f32[1, 256, 1]", arg537_1: "f32[1, 256, 1]", arg538_1: "f32[1, 256, 1]", arg539_1: "f32[1, 256, 1]", arg540_1: "f32[1, 256, 1]", arg541_1: "f32[128]", arg542_1: "f32[128, 1, 1]", arg543_1: "f32[128, 128, 3]", arg544_1: "f32[128]", arg545_1: "f32[128, 1, 1]", arg546_1: "f32[128, 128, 3]", arg547_1: "f32[128]", arg548_1: "f32[128, 1, 1]", arg549_1: "f32[128, 128, 3]", arg550_1: "f32[128]", arg551_1: "f32[128, 1, 1]", arg552_1: "f32[128, 128, 3]", arg553_1: "f32[128]", arg554_1: "f32[128, 1, 1]", arg555_1: "f32[128, 128, 3]", arg556_1: "f32[128]", arg557_1: "f32[128, 1, 1]", arg558_1: "f32[128, 128, 3]", arg559_1: "f32[128]", arg560_1: "f32[128]", arg561_1: "f32[256, 128]", arg562_1: "f32[256]", arg563_1: "f32[128]", arg564_1: "f32[128]", arg565_1: "f32[256, 128]", arg566_1: "f32[256]", arg567_1: "f32[128]", arg568_1: "f32[128]", arg569_1: "f32[256, 128]", arg570_1: "f32[256]", arg571_1: "f32[128]", arg572_1: "f32[128]", arg573_1: "f32[256, 128]", arg574_1: "f32[256]", arg575_1: "f32[128]", arg576_1: "f32[128]", arg577_1: "f32[256, 128]", arg578_1: "f32[256]", arg579_1: "f32[128]", arg580_1: "f32[128]", arg581_1: "f32[256, 128]", arg582_1: "f32[256]", arg583_1: "f32[1, 128, 1]", arg584_1: "f32[1, 128, 1]", arg585_1: "f32[1, 128, 1]", arg586_1: "f32[1, 128, 1]", arg587_1: "f32[1, 128, 1]", arg588_1: "f32[1, 128, 1]", arg589_1: "f32[128]", arg590_1: "f32[128, 1, 1]", arg591_1: "f32[128, 128, 7]", arg592_1: "f32[128]", arg593_1: "f32[128, 1, 1]", arg594_1: "f32[128, 128, 7]", arg595_1: "f32[128]", arg596_1: "f32[128, 1, 1]", arg597_1: "f32[128, 128, 7]", arg598_1: "f32[128]", arg599_1: "f32[128, 1, 1]", arg600_1: "f32[128, 128, 7]", arg601_1: "f32[128]", arg602_1: "f32[128, 1, 1]", arg603_1: "f32[128, 128, 7]", arg604_1: "f32[128]", arg605_1: "f32[128, 1, 1]", arg606_1: "f32[128, 128, 7]", arg607_1: "f32[128]", arg608_1: "f32[128]", arg609_1: "f32[256, 128]", arg610_1: "f32[256]", arg611_1: "f32[128]", arg612_1: "f32[128]", arg613_1: "f32[256, 128]", arg614_1: "f32[256]", arg615_1: "f32[128]", arg616_1: "f32[128]", arg617_1: "f32[256, 128]", arg618_1: "f32[256]", arg619_1: "f32[128]", arg620_1: "f32[128]", arg621_1: "f32[256, 128]", arg622_1: "f32[256]", arg623_1: "f32[128]", arg624_1: "f32[128]", arg625_1: "f32[256, 128]", arg626_1: "f32[256]", arg627_1: "f32[128]", arg628_1: "f32[128]", arg629_1: "f32[256, 128]", arg630_1: "f32[256]", arg631_1: "f32[1, 128, 1]", arg632_1: "f32[1, 128, 1]", arg633_1: "f32[1, 128, 1]", arg634_1: "f32[1, 128, 1]", arg635_1: "f32[1, 128, 1]", arg636_1: "f32[1, 128, 1]", arg637_1: "f32[128]", arg638_1: "f32[128, 1, 1]", arg639_1: "f32[128, 128, 11]", arg640_1: "f32[128]", arg641_1: "f32[128, 1, 1]", arg642_1: "f32[128, 128, 11]", arg643_1: "f32[128]", arg644_1: "f32[128, 1, 1]", arg645_1: "f32[128, 128, 11]", arg646_1: "f32[128]", arg647_1: "f32[128, 1, 1]", arg648_1: "f32[128, 128, 11]", arg649_1: "f32[128]", arg650_1: "f32[128, 1, 1]", arg651_1: "f32[128, 128, 11]", arg652_1: "f32[128]", arg653_1: "f32[128, 1, 1]", arg654_1: "f32[128, 128, 11]", arg655_1: "f32[128]", arg656_1: "f32[128]", arg657_1: "f32[256, 128]", arg658_1: "f32[256]", arg659_1: "f32[128]", arg660_1: "f32[128]", arg661_1: "f32[256, 128]", arg662_1: "f32[256]", arg663_1: "f32[128]", arg664_1: "f32[128]", arg665_1: "f32[256, 128]", arg666_1: "f32[256]", arg667_1: "f32[128]", arg668_1: "f32[128]", arg669_1: "f32[256, 128]", arg670_1: "f32[256]", arg671_1: "f32[128]", arg672_1: "f32[128]", arg673_1: "f32[256, 128]", arg674_1: "f32[256]", arg675_1: "f32[128]", arg676_1: "f32[128]", arg677_1: "f32[256, 128]", arg678_1: "f32[256]", arg679_1: "f32[1, 128, 1]", arg680_1: "f32[1, 128, 1]", arg681_1: "f32[1, 128, 1]", arg682_1: "f32[1, 128, 1]", arg683_1: "f32[1, 128, 1]", arg684_1: "f32[1, 128, 1]", arg685_1: "f32[22]", arg686_1: "f32[22, 1, 1]", arg687_1: "f32[22, 128, 7]", arg688_1: "i64[1, 512]", arg689_1: "i64[1, 512]", arg690_1: "f32[20]", arg691_1: "f32[11, 1, 20]", arg692_1: "f32[11, 1, 20]", arg693_1: "f32[11, 1, 20]", arg694_1: "f32[11, 1, 20]", arg695_1: "i64[1, s70]", arg696_1: "f32[1, 128]", arg697_1: "i32[1]"):
     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\kokoro\model.py:150 in forward, code: waveform, duration = self.kmodel.forward_with_tokens(input_ids, ref_s, speed)
    _set_grad_enabled = torch._C._set_grad_enabled(False);  _set_grad_enabled = None
    sym_size_int: "Sym(s70)" = torch.ops.aten.sym_size.int(arg695_1, 1)
    full: "i64[1]" = torch.ops.aten.full.default([1], sym_size_int, dtype = torch.int64, device = device(type='cpu'), pin_memory = False)
    max_1: "i64[]" = torch.ops.aten.max.default(full)
    item: "Sym(u0)" = torch.ops.aten.item.default(max_1);  max_1 = None
    arange: "i64[u0]" = torch.ops.aten.arange.default(item, device = device(type='cpu'), pin_memory = False);  item = None
    unsqueeze: "i64[1, u0]" = torch.ops.aten.unsqueeze.default(arange, 0)
    expand: "i64[1, u0]" = torch.ops.aten.expand.default(unsqueeze, [1, -1]);  unsqueeze = None
    type_as: "i64[1, u0]" = torch.ops.aten.type_as.default(expand, full);  expand = None
    add: "i64[1, u0]" = torch.ops.aten.add.Tensor(type_as, 1);  type_as = None
    unsqueeze_1: "i64[1, 1]" = torch.ops.aten.unsqueeze.default(full, 1);  full = None
    gt: "b8[1, u0]" = torch.ops.aten.gt.Tensor(add, unsqueeze_1);  add = unsqueeze_1 = None
    to: "b8[1, u0]" = torch.ops.aten.to.dtype_layout(gt, dtype = torch.bool, layout = torch.strided, device = device(type='cpu'));  gt = None
    bitwise_not: "b8[1, u0]" = torch.ops.aten.bitwise_not.default(to);  to = None
    to_1: "i32[1, u0]" = torch.ops.aten.to.dtype(bitwise_not, torch.int32);  bitwise_not = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:684 in forward, code: buffered_token_type_ids = self.embeddings.token_type_ids[:, :seq_length]
    slice_1: "i64[1, s70]" = torch.ops.aten.slice.Tensor(arg689_1, 1, None, sym_size_int);  arg689_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:685 in forward, code: buffered_token_type_ids_expanded = buffered_token_type_ids.expand(batch_size, seq_length)
    expand_1: "i64[1, s70]" = torch.ops.aten.expand.default(slice_1, [1, sym_size_int]);  slice_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:218 in forward, code: position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length]
    add_1: "Sym(s70)" = sym_size_int + 0
    slice_2: "i64[1, s70]" = torch.ops.aten.slice.Tensor(arg688_1, 1, 0, add_1);  arg688_1 = add_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\sparse.py:192 in forward, code: return F.embedding(
    embedding: "f32[1, s70, 128]" = torch.ops.aten.embedding.default(arg0_1, arg695_1, 0);  arg0_1 = arg695_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\sparse.py:192 in forward, code: return F.embedding(
    embedding_1: "f32[1, s70, 128]" = torch.ops.aten.embedding.default(arg2_1, expand_1);  arg2_1 = expand_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:235 in forward, code: embeddings = inputs_embeds + token_type_embeddings
    add_2: "f32[1, s70, 128]" = torch.ops.aten.add.Tensor(embedding, embedding_1);  embedding = embedding_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\sparse.py:192 in forward, code: return F.embedding(
    embedding_2: "f32[1, s70, 128]" = torch.ops.aten.embedding.default(arg1_1, slice_2);  arg1_1 = slice_2 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:238 in forward, code: embeddings += position_embeddings
    add_: "f32[1, s70, 128]" = torch.ops.aten.add_.Tensor(add_2, embedding_2);  add_2 = embedding_2 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\normalization.py:229 in forward, code: return F.layer_norm(
    layer_norm: "f32[1, s70, 128]" = torch.ops.aten.layer_norm.default(add_, [128], arg3_1, arg4_1, 1e-12);  add_ = arg3_1 = arg4_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\dropout.py:73 in forward, code: return F.dropout(input, self.p, self.training, self.inplace)
    dropout: "f32[1, s70, 128]" = torch.ops.aten.dropout.default(layer_norm, 0.0, False);  layer_norm = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:702 in forward, code: extended_attention_mask = _prepare_4d_attention_mask_for_sdpa(
    unsqueeze_2: "i32[1, 1, u0]" = torch.ops.aten.unsqueeze.default(to_1, 1);  to_1 = None
    unsqueeze_3: "i32[1, 1, 1, u0]" = torch.ops.aten.unsqueeze.default(unsqueeze_2, 2);  unsqueeze_2 = None
    sym_size_int_1: "Sym(u0)" = torch.ops.aten.sym_size.int(arange, 0);  arange = None
    le: "Sym(u0 <= 9223372036854775807)" = sym_size_int_1 <= 9223372036854775807;  le = None
    slice_3: "i32[1, 1, 1, u0]" = torch.ops.aten.slice.Tensor(unsqueeze_3, 3, 0, 9223372036854775807);  unsqueeze_3 = None
    expand_2: "i32[1, 1, s70, u0]" = torch.ops.aten.expand.default(slice_3, [1, 1, sym_size_int, sym_size_int_1]);  slice_3 = sym_size_int = sym_size_int_1 = None
    to_2: "f32[1, 1, s70, u0]" = torch.ops.aten.to.dtype(expand_2, torch.float32);  expand_2 = None
    _tensor_constant0: "f32[]" = self._tensor_constant0
    lift_fresh_copy: "f32[]" = torch.ops.aten.lift_fresh_copy.default(_tensor_constant0);  _tensor_constant0 = None
    detach_: "f32[]" = torch.ops.aten.detach_.default(lift_fresh_copy);  lift_fresh_copy = None
    sub: "f32[1, 1, s70, u0]" = torch.ops.aten.sub.Tensor(detach_, to_2);  detach_ = to_2 = None
    to_3: "b8[1, 1, s70, u0]" = torch.ops.aten.to.dtype(sub, torch.bool)
    masked_fill: "f32[1, 1, s70, u0]" = torch.ops.aten.masked_fill.Scalar(sub, to_3, -3.4028234663852886e+38);  sub = to_3 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\linear.py:134 in forward, code: return F.linear(input, self.weight, self.bias)
    linear: "f32[1, s70, 768]" = torch.ops.aten.linear.default(dropout, arg5_1, arg6_1);  dropout = arg5_1 = arg6_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\linear.py:134 in forward, code: return F.linear(input, self.weight, self.bias)
    linear_1: "f32[1, s70, 768]" = torch.ops.aten.linear.default(linear, arg9_1, arg10_1);  arg9_1 = arg10_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:379 in forward, code: .view(batch_size, -1, self.num_attention_heads, self.attention_head_size)
    view: "f32[1, s70, 12, 64]" = torch.ops.aten.view.default(linear_1, [1, -1, 12, 64]);  linear_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:380 in forward, code: .transpose(1, 2)
    transpose: "f32[1, 12, s70, 64]" = torch.ops.aten.transpose.int(view, 1, 2);  view = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\linear.py:134 in forward, code: return F.linear(input, self.weight, self.bias)
    linear_2: "f32[1, s70, 768]" = torch.ops.aten.linear.default(linear, arg11_1, arg12_1);  arg11_1 = arg12_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:384 in forward, code: .view(batch_size, -1, self.num_attention_heads, self.attention_head_size)
    view_1: "f32[1, s70, 12, 64]" = torch.ops.aten.view.default(linear_2, [1, -1, 12, 64]);  linear_2 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:385 in forward, code: .transpose(1, 2)
    transpose_1: "f32[1, 12, s70, 64]" = torch.ops.aten.transpose.int(view_1, 1, 2);  view_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\linear.py:134 in forward, code: return F.linear(input, self.weight, self.bias)
    linear_3: "f32[1, s70, 768]" = torch.ops.aten.linear.default(linear, arg13_1, arg14_1);  linear = arg13_1 = arg14_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:389 in forward, code: .view(batch_size, -1, self.num_attention_heads, self.attention_head_size)
    view_2: "f32[1, s70, 12, 64]" = torch.ops.aten.view.default(linear_3, [1, -1, 12, 64]);  linear_3 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:390 in forward, code: .transpose(1, 2)
    transpose_2: "f32[1, 12, s70, 64]" = torch.ops.aten.transpose.int(view_2, 1, 2);  view_2 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:393 in forward, code: attention_output = torch.nn.functional.scaled_dot_product_attention(
    scaled_dot_product_attention = torch.ops.aten.scaled_dot_product_attention.default(transpose, transpose_1, transpose_2, masked_fill);  transpose = transpose_1 = transpose_2 = masked_fill = scaled_dot_product_attention = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\kokoro\model.py:150 in forward, code: waveform, duration = self.kmodel.forward_with_tokens(input_ids, ref_s, speed)
    _set_grad_enabled_1 = torch._C._set_grad_enabled(True);  _set_grad_enabled_1 = None

ERROR:__main__:❌ FP32 ONNX 导出失败: Failed to export the model with torch.export. This is step 1/3 of exporting the model to ONNX. Next steps:
- Modify the model code for `torch.export.export` to succeed. Refer to https://pytorch.org/docs/stable/generated/exportdb/index.html for more information.
- Debug `torch.export.export` and submit a PR to PyTorch.
- Create an issue in the PyTorch GitHub repository against the *torch.export* component and attach the full error stack as well as reproduction scripts.

## Exception summary

<class 'torch.fx.experimental.symbolic_shapes.GuardOnDataDependentSymNode'>: Could not guard on data-dependent expression Ne(u0, 50) (unhinted: Ne(u0, s70)).  (Size-like symbols: u0)

consider using data-dependent friendly APIs such as guard_or_false, guard_or_true and statically_known_trueCaused by: (_ops.py:841 in __call__)
For more information, run with TORCH_LOGS="dynamic"
For extended logs when we create symbols, also add TORCHDYNAMO_EXTENDED_DEBUG_CREATE_SYMBOL="u0"
If you suspect the guard was triggered from C++, add TORCHDYNAMO_EXTENDED_DEBUG_CPP=1
For more debugging help, see https://docs.google.com/document/d/1HSuTTVvYH1pTew89Rtpeu84Ht3nQEFTYhAX3Ypa_xJs/edit?usp=sharing

For C++ stack trace, run with TORCHDYNAMO_EXTENDED_DEBUG_CPP=1

The following call raised this error:
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py", line 393, in forward
    attention_output = torch.nn.functional.scaled_dot_product_attention(

To fix the error, insert one of the following checks before this call:
  1. torch._check(attention_mask.shape[3] != 50)
  2. torch._check(attention_mask.shape[3] == 50)

(These suggested fixes were derived by replacing `u0` with attention_mask.shape[3] in Ne(u0, 50) and its negation.)

The error above occurred when calling torch.export.export. If you would like to view some more information about this error, and get a list of all other errors that may occur in your export call, you can replace your `export()` call with `draft_export()`.

(Refer to the full stack trace above for more information.)
INFO:__main__:
==============================================
INFO:__main__:🎉 所有转换和量化流程完成！请检查输出目录。
INFO:__main__:==============================================

(kokoro-82m-v1-1-zh) C:\Users\admin\code\other\Kokoro-82M-v1.1-zh>

I also updated the https://github.com/thewh1teagle/kokoro-onnx/blob/main/scripts/export.py to do the convertion work, but it failed with the same errors.

Details

(kokoro-82m-v1-1-zh) C:\Users\admin\code\other\Kokoro-82M-v1.1-zh>python export.py
WARNING: Defaulting repo_id to hexgrad/Kokoro-82M. Pass repo_id='hexgrad/Kokoro-82M' to suppress this warning.
C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\rnn.py:123: UserWarning: dropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=0.2 and num_layers=1
  warnings.warn(
C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\utils\weight_norm.py:144: FutureWarning: `torch.nn.utils.weight_norm` is deprecated in favor of `torch.nn.utils.parametrizations.weight_norm`.
  WeightNorm.apply(module, name, dim)
C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\export.py:21: UserWarning: # 'dynamic_axes' is not recommended when dynamo=True, and may lead to 'torch._dynamo.exc.UserError: Constraints violated.' Supply the 'dynamic_shapes' argument instead if export is unsuccessful.
  torch.onnx.export(
W1209 15:54:16.717000 25776 .venv\Lib\site-packages\torch\onnx\_internal\exporter\_compat.py:114] Setting ONNX exporter to use operator set version 18 because the requested opset_version 17 is a lower version than we have implementations for. Automatic version conversion will be performed, which may not be successful at converting to the requested version. If version conversion is unsuccessful, the opset version of the exported model will be kept at 18. Please consider setting opset_version >=18 to leverage latest ONNX features
W1209 15:54:17.118000 25776 .venv\Lib\site-packages\torch\onnx\_internal\exporter\_registration.py:107] torchvision is not installed. Skipping torchvision::nms
[torch.onnx] Obtain model graph for `KModelForONNX([...]` with `torch.export.export(..., strict=False)`...



def forward(self, arg0_1: "f32[178, 128]", arg1_1: "f32[512, 128]", arg2_1: "f32[2, 128]", arg3_1: "f32[128]", arg4_1: "f32[128]", arg5_1: "f32[768, 128]", arg6_1: "f32[768]", arg7_1: "f32[768]", arg8_1: "f32[768]", arg9_1: "f32[768, 768]", arg10_1: "f32[768]", arg11_1: "f32[768, 768]", arg12_1: "f32[768]", arg13_1: "f32[768, 768]", arg14_1: "f32[768]", arg15_1: "f32[768, 768]", arg16_1: "f32[768]", arg17_1: "f32[768]", arg18_1: "f32[768]", arg19_1: "f32[2048, 768]", arg20_1: "f32[2048]", arg21_1: "f32[768, 2048]", arg22_1: "f32[768]", arg23_1: "f32[768, 768]", arg24_1: "f32[768]", arg25_1: "f32[512, 768]", arg26_1: "f32[512]", arg27_1: "f32[1024, 640]", arg28_1: "f32[1024, 256]", arg29_1: "f32[1024]", arg30_1: "f32[1024]", arg31_1: "f32[1024, 640]", arg32_1: "f32[1024, 256]", arg33_1: "f32[1024]", arg34_1: "f32[1024]", arg35_1: "f32[1024, 128]", arg36_1: "f32[1024]", arg37_1: "f32[1024, 640]", arg38_1: "f32[1024, 256]", arg39_1: "f32[1024]", arg40_1: "f32[1024]", arg41_1: "f32[1024, 640]", arg42_1: "f32[1024, 256]", arg43_1: "f32[1024]", arg44_1: "f32[1024]", arg45_1: "f32[1024, 128]", arg46_1: "f32[1024]", arg47_1: "f32[1024, 640]", arg48_1: "f32[1024, 256]", arg49_1: "f32[1024]", arg50_1: "f32[1024]", arg51_1: "f32[1024, 640]", arg52_1: "f32[1024, 256]", arg53_1: "f32[1024]", arg54_1: "f32[1024]", arg55_1: "f32[1024, 128]", arg56_1: "f32[1024]", arg57_1: "f32[1024, 640]", arg58_1: "f32[1024, 256]", arg59_1: "f32[1024]", arg60_1: "f32[1024]", arg61_1: "f32[1024, 640]", arg62_1: "f32[1024, 256]", arg63_1: "f32[1024]", arg64_1: "f32[1024]", arg65_1: "f32[50, 512]", arg66_1: "f32[50]", arg67_1: "f32[1024, 640]", arg68_1: "f32[1024, 256]", arg69_1: "f32[1024]", arg70_1: "f32[1024]", arg71_1: "f32[1024, 640]", arg72_1: "f32[1024, 256]", arg73_1: "f32[1024]", arg74_1: "f32[1024]", arg75_1: "f32[512]", arg76_1: "f32[512, 1, 1]", arg77_1: "f32[512, 512, 3]", arg78_1: "f32[512]", arg79_1: "f32[512, 1, 1]", arg80_1: "f32[512, 512, 3]", arg81_1: "f32[512]", arg82_1: "f32[512]", arg83_1: "f32[1024, 128]", arg84_1: "f32[1024]", arg85_1: "f32[512]", arg86_1: "f32[512]", arg87_1: "f32[1024, 128]", arg88_1: "f32[1024]", arg89_1: "f32[256]", arg90_1: "f32[256, 1, 1]", arg91_1: "f32[256, 512, 3]", arg92_1: "f32[256]", arg93_1: "f32[256, 1, 1]", arg94_1: "f32[256, 256, 3]", arg95_1: "f32[512]", arg96_1: "f32[512]", arg97_1: "f32[1024, 128]", arg98_1: "f32[1024]", arg99_1: "f32[256]", arg100_1: "f32[256]", arg101_1: "f32[512, 128]", arg102_1: "f32[512]", arg103_1: "f32[256, 1, 1]", arg104_1: "f32[256, 512, 1]", arg105_1: "f32[512]", arg106_1: "f32[512, 1, 1]", arg107_1: "f32[512, 1, 3]", arg108_1: "f32[256]", arg109_1: "f32[256, 1, 1]", arg110_1: "f32[256, 256, 3]", arg111_1: "f32[256]", arg112_1: "f32[256, 1, 1]", arg113_1: "f32[256, 256, 3]", arg114_1: "f32[256]", arg115_1: "f32[256]", arg116_1: "f32[512, 128]", arg117_1: "f32[512]", arg118_1: "f32[256]", arg119_1: "f32[256]", arg120_1: "f32[512, 128]", arg121_1: "f32[512]", arg122_1: "f32[512]", arg123_1: "f32[512, 1, 1]", arg124_1: "f32[512, 512, 3]", arg125_1: "f32[512]", arg126_1: "f32[512, 1, 1]", arg127_1: "f32[512, 512, 3]", arg128_1: "f32[512]", arg129_1: "f32[512]", arg130_1: "f32[1024, 128]", arg131_1: "f32[1024]", arg132_1: "f32[512]", arg133_1: "f32[512]", arg134_1: "f32[1024, 128]", arg135_1: "f32[1024]", arg136_1: "f32[256]", arg137_1: "f32[256, 1, 1]", arg138_1: "f32[256, 512, 3]", arg139_1: "f32[256]", arg140_1: "f32[256, 1, 1]", arg141_1: "f32[256, 256, 3]", arg142_1: "f32[512]", arg143_1: "f32[512]", arg144_1: "f32[1024, 128]", arg145_1: "f32[1024]", arg146_1: "f32[256]", arg147_1: "f32[256]", arg148_1: "f32[512, 128]", arg149_1: "f32[512]", arg150_1: "f32[256, 1, 1]", arg151_1: "f32[256, 512, 1]", arg152_1: "f32[512]", arg153_1: "f32[512, 1, 1]", arg154_1: "f32[512, 1, 3]", arg155_1: "f32[256]", arg156_1: "f32[256, 1, 1]", arg157_1: "f32[256, 256, 3]", arg158_1: "f32[256]", arg159_1: "f32[256, 1, 1]", arg160_1: "f32[256, 256, 3]", arg161_1: "f32[256]", arg162_1: "f32[256]", arg163_1: "f32[512, 128]", arg164_1: "f32[512]", arg165_1: "f32[256]", arg166_1: "f32[256]", arg167_1: "f32[512, 128]", arg168_1: "f32[512]", arg169_1: "f32[1, 256, 1]", arg170_1: "f32[1]", arg171_1: "f32[1, 256, 1]", arg172_1: "f32[1]", arg173_1: "f32[178, 512]", arg174_1: "f32[512]", arg175_1: "f32[512, 1, 1]", arg176_1: "f32[512, 512, 5]", arg177_1: "f32[512]", arg178_1: "f32[512]", arg179_1: "f32[512]", arg180_1: "f32[512, 1, 1]", arg181_1: "f32[512, 512, 5]", arg182_1: "f32[512]", arg183_1: "f32[512]", arg184_1: "f32[512]", arg185_1: "f32[512, 1, 1]", arg186_1: "f32[512, 512, 5]", arg187_1: "f32[512]", arg188_1: "f32[512]", arg189_1: "f32[1024, 512]", arg190_1: "f32[1024, 256]", arg191_1: "f32[1024]", arg192_1: "f32[1024]", arg193_1: "f32[1024, 512]", arg194_1: "f32[1024, 256]", arg195_1: "f32[1024]", arg196_1: "f32[1024]", arg197_1: "f32[1024]", arg198_1: "f32[1024, 1, 1]", arg199_1: "f32[1024, 514, 3]", arg200_1: "f32[1024]", arg201_1: "f32[1024, 1, 1]", arg202_1: "f32[1024, 1024, 3]", arg203_1: "f32[514]", arg204_1: "f32[514]", arg205_1: "f32[1028, 128]", arg206_1: "f32[1028]", arg207_1: "f32[1024]", arg208_1: "f32[1024]", arg209_1: "f32[2048, 128]", arg210_1: "f32[2048]", arg211_1: "f32[1024, 1, 1]", arg212_1: "f32[1024, 514, 1]", arg213_1: "f32[1024]", arg214_1: "f32[1024, 1, 1]", arg215_1: "f32[1024, 1090, 3]", arg216_1: "f32[1024]", arg217_1: "f32[1024, 1, 1]", arg218_1: "f32[1024, 1024, 3]", arg219_1: "f32[1090]", arg220_1: "f32[1090]", arg221_1: "f32[2180, 128]", arg222_1: "f32[2180]", arg223_1: "f32[1024]", arg224_1: "f32[1024]", arg225_1: "f32[2048, 128]", arg226_1: "f32[2048]", arg227_1: "f32[1024, 1, 1]", arg228_1: "f32[1024, 1090, 1]", arg229_1: "f32[1024]", arg230_1: "f32[1024, 1, 1]", arg231_1: "f32[1024, 1090, 3]", arg232_1: "f32[1024]", arg233_1: "f32[1024, 1, 1]", arg234_1: "f32[1024, 1024, 3]", arg235_1: "f32[1090]", arg236_1: "f32[1090]", arg237_1: "f32[2180, 128]", arg238_1: "f32[2180]", arg239_1: "f32[1024]", arg240_1: "f32[1024]", arg241_1: "f32[2048, 128]", arg242_1: "f32[2048]", arg243_1: "f32[1024, 1, 1]", arg244_1: "f32[1024, 1090, 1]", arg245_1: "f32[1024]", arg246_1: "f32[1024, 1, 1]", arg247_1: "f32[1024, 1090, 3]", arg248_1: "f32[1024]", arg249_1: "f32[1024, 1, 1]", arg250_1: "f32[1024, 1024, 3]", arg251_1: "f32[1090]", arg252_1: "f32[1090]", arg253_1: "f32[2180, 128]", arg254_1: "f32[2180]", arg255_1: "f32[1024]", arg256_1: "f32[1024]", arg257_1: "f32[2048, 128]", arg258_1: "f32[2048]", arg259_1: "f32[1024, 1, 1]", arg260_1: "f32[1024, 1090, 1]", arg261_1: "f32[512]", arg262_1: "f32[512, 1, 1]", arg263_1: "f32[512, 1090, 3]", arg264_1: "f32[512]", arg265_1: "f32[512, 1, 1]", arg266_1: "f32[512, 512, 3]", arg267_1: "f32[1090]", arg268_1: "f32[1090]", arg269_1: "f32[2180, 128]", arg270_1: "f32[2180]", arg271_1: "f32[512]", arg272_1: "f32[512]", arg273_1: "f32[1024, 128]", arg274_1: "f32[1024]", arg275_1: "f32[512, 1, 1]", arg276_1: "f32[512, 1090, 1]", arg277_1: "f32[1090]", arg278_1: "f32[1090, 1, 1]", arg279_1: "f32[1090, 1, 3]", arg280_1: "f32[1]", arg281_1: "f32[1, 1, 1]", arg282_1: "f32[1, 1, 3]", arg283_1: "f32[1]", arg284_1: "f32[1, 1, 1]", arg285_1: "f32[1, 1, 3]", arg286_1: "f32[64]", arg287_1: "f32[64, 1, 1]", arg288_1: "f32[64, 512, 1]", arg289_1: "f32[1, 9]", arg290_1: "f32[1]", arg291_1: "f32[256, 22, 12]", arg292_1: "f32[256]", arg293_1: "f32[128, 22, 1]", arg294_1: "f32[128]", arg295_1: "f32[256]", arg296_1: "f32[256, 1, 1]", arg297_1: "f32[256, 256, 7]", arg298_1: "f32[256]", arg299_1: "f32[256, 1, 1]", arg300_1: "f32[256, 256, 7]", arg301_1: "f32[256]", arg302_1: "f32[256, 1, 1]", arg303_1: "f32[256, 256, 7]", arg304_1: "f32[256]", arg305_1: "f32[256, 1, 1]", arg306_1: "f32[256, 256, 7]", arg307_1: "f32[256]", arg308_1: "f32[256, 1, 1]", arg309_1: "f32[256, 256, 7]", arg310_1: "f32[256]", arg311_1: "f32[256, 1, 1]", arg312_1: "f32[256, 256, 7]", arg313_1: "f32[256]", arg314_1: "f32[256]", arg315_1: "f32[512, 128]", arg316_1: "f32[512]", arg317_1: "f32[256]", arg318_1: "f32[256]", arg319_1: "f32[512, 128]", arg320_1: "f32[512]", arg321_1: "f32[256]", arg322_1: "f32[256]", arg323_1: "f32[512, 128]", arg324_1: "f32[512]", arg325_1: "f32[256]", arg326_1: "f32[256]", arg327_1: "f32[512, 128]", arg328_1: "f32[512]", arg329_1: "f32[256]", arg330_1: "f32[256]", arg331_1: "f32[512, 128]", arg332_1: "f32[512]", arg333_1: "f32[256]", arg334_1: "f32[256]", arg335_1: "f32[512, 128]", arg336_1: "f32[512]", arg337_1: "f32[1, 256, 1]", arg338_1: "f32[1, 256, 1]", arg339_1: "f32[1, 256, 1]", arg340_1: "f32[1, 256, 1]", arg341_1: "f32[1, 256, 1]", arg342_1: "f32[1, 256, 1]", arg343_1: "f32[128]", arg344_1: "f32[128, 1, 1]", arg345_1: "f32[128, 128, 11]", arg346_1: "f32[128]", arg347_1: "f32[128, 1, 1]", arg348_1: "f32[128, 128, 11]", arg349_1: "f32[128]", arg350_1: "f32[128, 1, 1]", arg351_1: "f32[128, 128, 11]", arg352_1: "f32[128]", arg353_1: "f32[128, 1, 1]", arg354_1: "f32[128, 128, 11]", arg355_1: "f32[128]", arg356_1: "f32[128, 1, 1]", arg357_1: "f32[128, 128, 11]", arg358_1: "f32[128]", arg359_1: "f32[128, 1, 1]", arg360_1: "f32[128, 128, 11]", arg361_1: "f32[128]", arg362_1: "f32[128]", arg363_1: "f32[256, 128]", arg364_1: "f32[256]", arg365_1: "f32[128]", arg366_1: "f32[128]", arg367_1: "f32[256, 128]", arg368_1: "f32[256]", arg369_1: "f32[128]", arg370_1: "f32[128]", arg371_1: "f32[256, 128]", arg372_1: "f32[256]", arg373_1: "f32[128]", arg374_1: "f32[128]", arg375_1: "f32[256, 128]", arg376_1: "f32[256]", arg377_1: "f32[128]", arg378_1: "f32[128]", arg379_1: "f32[256, 128]", arg380_1: "f32[256]", arg381_1: "f32[128]", arg382_1: "f32[128]", arg383_1: "f32[256, 128]", arg384_1: "f32[256]", arg385_1: "f32[1, 128, 1]", arg386_1: "f32[1, 128, 1]", arg387_1: "f32[1, 128, 1]", arg388_1: "f32[1, 128, 1]", arg389_1: "f32[1, 128, 1]", arg390_1: "f32[1, 128, 1]", arg391_1: "f32[256]", arg392_1: "f32[512, 1, 1]", arg393_1: "f32[512, 256, 20]", arg394_1: "f32[128]", arg395_1: "f32[256, 1, 1]", arg396_1: "f32[256, 128, 12]", arg397_1: "f32[256]", arg398_1: "f32[256, 1, 1]", arg399_1: "f32[256, 256, 3]", arg400_1: "f32[256]", arg401_1: "f32[256, 1, 1]", arg402_1: "f32[256, 256, 3]", arg403_1: "f32[256]", arg404_1: "f32[256, 1, 1]", arg405_1: "f32[256, 256, 3]", arg406_1: "f32[256]", arg407_1: "f32[256, 1, 1]", arg408_1: "f32[256, 256, 3]", arg409_1: "f32[256]", arg410_1: "f32[256, 1, 1]", arg411_1: "f32[256, 256, 3]", arg412_1: "f32[256]", arg413_1: "f32[256, 1, 1]", arg414_1: "f32[256, 256, 3]", arg415_1: "f32[256]", arg416_1: "f32[256]", arg417_1: "f32[512, 128]", arg418_1: "f32[512]", arg419_1: "f32[256]", arg420_1: "f32[256]", arg421_1: "f32[512, 128]", arg422_1: "f32[512]", arg423_1: "f32[256]", arg424_1: "f32[256]", arg425_1: "f32[512, 128]", arg426_1: "f32[512]", arg427_1: "f32[256]", arg428_1: "f32[256]", arg429_1: "f32[512, 128]", arg430_1: "f32[512]", arg431_1: "f32[256]", arg432_1: "f32[256]", arg433_1: "f32[512, 128]", arg434_1: "f32[512]", arg435_1: "f32[256]", arg436_1: "f32[256]", arg437_1: "f32[512, 128]", arg438_1: "f32[512]", arg439_1: "f32[1, 256, 1]", arg440_1: "f32[1, 256, 1]", arg441_1: "f32[1, 256, 1]", arg442_1: "f32[1, 256, 1]", arg443_1: "f32[1, 256, 1]", arg444_1: "f32[1, 256, 1]", arg445_1: "f32[256]", arg446_1: "f32[256, 1, 1]", arg447_1: "f32[256, 256, 7]", arg448_1: "f32[256]", arg449_1: "f32[256, 1, 1]", arg450_1: "f32[256, 256, 7]", arg451_1: "f32[256]", arg452_1: "f32[256, 1, 1]", arg453_1: "f32[256, 256, 7]", arg454_1: "f32[256]", arg455_1: "f32[256, 1, 1]", arg456_1: "f32[256, 256, 7]", arg457_1: "f32[256]", arg458_1: "f32[256, 1, 1]", arg459_1: "f32[256, 256, 7]", arg460_1: "f32[256]", arg461_1: "f32[256, 1, 1]", arg462_1: "f32[256, 256, 7]", arg463_1: "f32[256]", arg464_1: "f32[256]", arg465_1: "f32[512, 128]", arg466_1: "f32[512]", arg467_1: "f32[256]", arg468_1: "f32[256]", arg469_1: "f32[512, 128]", arg470_1: "f32[512]", arg471_1: "f32[256]", arg472_1: "f32[256]", arg473_1: "f32[512, 128]", arg474_1: "f32[512]", arg475_1: "f32[256]", arg476_1: "f32[256]", arg477_1: "f32[512, 128]", arg478_1: "f32[512]", arg479_1: "f32[256]", arg480_1: "f32[256]", arg481_1: "f32[512, 128]", arg482_1: "f32[512]", arg483_1: "f32[256]", arg484_1: "f32[256]", arg485_1: "f32[512, 128]", arg486_1: "f32[512]", arg487_1: "f32[1, 256, 1]", arg488_1: "f32[1, 256, 1]", arg489_1: "f32[1, 256, 1]", arg490_1: "f32[1, 256, 1]", arg491_1: "f32[1, 256, 1]", arg492_1: "f32[1, 256, 1]", arg493_1: "f32[256]", arg494_1: "f32[256, 1, 1]", arg495_1: "f32[256, 256, 11]", arg496_1: "f32[256]", arg497_1: "f32[256, 1, 1]", arg498_1: "f32[256, 256, 11]", arg499_1: "f32[256]", arg500_1: "f32[256, 1, 1]", arg501_1: "f32[256, 256, 11]", arg502_1: "f32[256]", arg503_1: "f32[256, 1, 1]", arg504_1: "f32[256, 256, 11]", arg505_1: "f32[256]", arg506_1: "f32[256, 1, 1]", arg507_1: "f32[256, 256, 11]", arg508_1: "f32[256]", arg509_1: "f32[256, 1, 1]", arg510_1: "f32[256, 256, 11]", arg511_1: "f32[256]", arg512_1: "f32[256]", arg513_1: "f32[512, 128]", arg514_1: "f32[512]", arg515_1: "f32[256]", arg516_1: "f32[256]", arg517_1: "f32[512, 128]", arg518_1: "f32[512]", arg519_1: "f32[256]", arg520_1: "f32[256]", arg521_1: "f32[512, 128]", arg522_1: "f32[512]", arg523_1: "f32[256]", arg524_1: "f32[256]", arg525_1: "f32[512, 128]", arg526_1: "f32[512]", arg527_1: "f32[256]", arg528_1: "f32[256]", arg529_1: "f32[512, 128]", arg530_1: "f32[512]", arg531_1: "f32[256]", arg532_1: "f32[256]", arg533_1: "f32[512, 128]", arg534_1: "f32[512]", arg535_1: "f32[1, 256, 1]", arg536_1: "f32[1, 256, 1]", arg537_1: "f32[1, 256, 1]", arg538_1: "f32[1, 256, 1]", arg539_1: "f32[1, 256, 1]", arg540_1: "f32[1, 256, 1]", arg541_1: "f32[128]", arg542_1: "f32[128, 1, 1]", arg543_1: "f32[128, 128, 3]", arg544_1: "f32[128]", arg545_1: "f32[128, 1, 1]", arg546_1: "f32[128, 128, 3]", arg547_1: "f32[128]", arg548_1: "f32[128, 1, 1]", arg549_1: "f32[128, 128, 3]", arg550_1: "f32[128]", arg551_1: "f32[128, 1, 1]", arg552_1: "f32[128, 128, 3]", arg553_1: "f32[128]", arg554_1: "f32[128, 1, 1]", arg555_1: "f32[128, 128, 3]", arg556_1: "f32[128]", arg557_1: "f32[128, 1, 1]", arg558_1: "f32[128, 128, 3]", arg559_1: "f32[128]", arg560_1: "f32[128]", arg561_1: "f32[256, 128]", arg562_1: "f32[256]", arg563_1: "f32[128]", arg564_1: "f32[128]", arg565_1: "f32[256, 128]", arg566_1: "f32[256]", arg567_1: "f32[128]", arg568_1: "f32[128]", arg569_1: "f32[256, 128]", arg570_1: "f32[256]", arg571_1: "f32[128]", arg572_1: "f32[128]", arg573_1: "f32[256, 128]", arg574_1: "f32[256]", arg575_1: "f32[128]", arg576_1: "f32[128]", arg577_1: "f32[256, 128]", arg578_1: "f32[256]", arg579_1: "f32[128]", arg580_1: "f32[128]", arg581_1: "f32[256, 128]", arg582_1: "f32[256]", arg583_1: "f32[1, 128, 1]", arg584_1: "f32[1, 128, 1]", arg585_1: "f32[1, 128, 1]", arg586_1: "f32[1, 128, 1]", arg587_1: "f32[1, 128, 1]", arg588_1: "f32[1, 128, 1]", arg589_1: "f32[128]", arg590_1: "f32[128, 1, 1]", arg591_1: "f32[128, 128, 7]", arg592_1: "f32[128]", arg593_1: "f32[128, 1, 1]", arg594_1: "f32[128, 128, 7]", arg595_1: "f32[128]", arg596_1: "f32[128, 1, 1]", arg597_1: "f32[128, 128, 7]", arg598_1: "f32[128]", arg599_1: "f32[128, 1, 1]", arg600_1: "f32[128, 128, 7]", arg601_1: "f32[128]", arg602_1: "f32[128, 1, 1]", arg603_1: "f32[128, 128, 7]", arg604_1: "f32[128]", arg605_1: "f32[128, 1, 1]", arg606_1: "f32[128, 128, 7]", arg607_1: "f32[128]", arg608_1: "f32[128]", arg609_1: "f32[256, 128]", arg610_1: "f32[256]", arg611_1: "f32[128]", arg612_1: "f32[128]", arg613_1: "f32[256, 128]", arg614_1: "f32[256]", arg615_1: "f32[128]", arg616_1: "f32[128]", arg617_1: "f32[256, 128]", arg618_1: "f32[256]", arg619_1: "f32[128]", arg620_1: "f32[128]", arg621_1: "f32[256, 128]", arg622_1: "f32[256]", arg623_1: "f32[128]", arg624_1: "f32[128]", arg625_1: "f32[256, 128]", arg626_1: "f32[256]", arg627_1: "f32[128]", arg628_1: "f32[128]", arg629_1: "f32[256, 128]", arg630_1: "f32[256]", arg631_1: "f32[1, 128, 1]", arg632_1: "f32[1, 128, 1]", arg633_1: "f32[1, 128, 1]", arg634_1: "f32[1, 128, 1]", arg635_1: "f32[1, 128, 1]", arg636_1: "f32[1, 128, 1]", arg637_1: "f32[128]", arg638_1: "f32[128, 1, 1]", arg639_1: "f32[128, 128, 11]", arg640_1: "f32[128]", arg641_1: "f32[128, 1, 1]", arg642_1: "f32[128, 128, 11]", arg643_1: "f32[128]", arg644_1: "f32[128, 1, 1]", arg645_1: "f32[128, 128, 11]", arg646_1: "f32[128]", arg647_1: "f32[128, 1, 1]", arg648_1: "f32[128, 128, 11]", arg649_1: "f32[128]", arg650_1: "f32[128, 1, 1]", arg651_1: "f32[128, 128, 11]", arg652_1: "f32[128]", arg653_1: "f32[128, 1, 1]", arg654_1: "f32[128, 128, 11]", arg655_1: "f32[128]", arg656_1: "f32[128]", arg657_1: "f32[256, 128]", arg658_1: "f32[256]", arg659_1: "f32[128]", arg660_1: "f32[128]", arg661_1: "f32[256, 128]", arg662_1: "f32[256]", arg663_1: "f32[128]", arg664_1: "f32[128]", arg665_1: "f32[256, 128]", arg666_1: "f32[256]", arg667_1: "f32[128]", arg668_1: "f32[128]", arg669_1: "f32[256, 128]", arg670_1: "f32[256]", arg671_1: "f32[128]", arg672_1: "f32[128]", arg673_1: "f32[256, 128]", arg674_1: "f32[256]", arg675_1: "f32[128]", arg676_1: "f32[128]", arg677_1: "f32[256, 128]", arg678_1: "f32[256]", arg679_1: "f32[1, 128, 1]", arg680_1: "f32[1, 128, 1]", arg681_1: "f32[1, 128, 1]", arg682_1: "f32[1, 128, 1]", arg683_1: "f32[1, 128, 1]", arg684_1: "f32[1, 128, 1]", arg685_1: "f32[22]", arg686_1: "f32[22, 1, 1]", arg687_1: "f32[22, 128, 7]", arg688_1: "i64[1, 512]", arg689_1: "i64[1, 512]", arg690_1: "f32[20]", arg691_1: "f32[11, 1, 20]", arg692_1: "f32[11, 1, 20]", arg693_1: "f32[11, 1, 20]", arg694_1: "f32[11, 1, 20]", arg695_1: "i64[1, s70]", arg696_1: "f32[1, 256]", arg697_1: "i32[1]"):
     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\kokoro\model.py:150 in forward, code: waveform, duration = self.kmodel.forward_with_tokens(input_ids, ref_s, speed)
    _set_grad_enabled = torch._C._set_grad_enabled(False);  _set_grad_enabled = None
    sym_size_int: "Sym(s70)" = torch.ops.aten.sym_size.int(arg695_1, 1)
    full: "i64[1]" = torch.ops.aten.full.default([1], sym_size_int, dtype = torch.int64, device = device(type='cpu'), pin_memory = False)
    max_1: "i64[]" = torch.ops.aten.max.default(full)
    item: "Sym(u0)" = torch.ops.aten.item.default(max_1);  max_1 = None
    arange: "i64[u0]" = torch.ops.aten.arange.default(item, device = device(type='cpu'), pin_memory = False);  item = None
    unsqueeze: "i64[1, u0]" = torch.ops.aten.unsqueeze.default(arange, 0)
    expand: "i64[1, u0]" = torch.ops.aten.expand.default(unsqueeze, [1, -1]);  unsqueeze = None
    type_as: "i64[1, u0]" = torch.ops.aten.type_as.default(expand, full);  expand = None
    add: "i64[1, u0]" = torch.ops.aten.add.Tensor(type_as, 1);  type_as = None
    unsqueeze_1: "i64[1, 1]" = torch.ops.aten.unsqueeze.default(full, 1);  full = None
    gt: "b8[1, u0]" = torch.ops.aten.gt.Tensor(add, unsqueeze_1);  add = unsqueeze_1 = None
    to: "b8[1, u0]" = torch.ops.aten.to.dtype_layout(gt, dtype = torch.bool, layout = torch.strided, device = device(type='cpu'));  gt = None
    bitwise_not: "b8[1, u0]" = torch.ops.aten.bitwise_not.default(to);  to = None
    to_1: "i32[1, u0]" = torch.ops.aten.to.dtype(bitwise_not, torch.int32);  bitwise_not = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:684 in forward, code: buffered_token_type_ids = self.embeddings.token_type_ids[:, :seq_length]
    slice_1: "i64[1, s70]" = torch.ops.aten.slice.Tensor(arg689_1, 1, None, sym_size_int);  arg689_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:685 in forward, code: buffered_token_type_ids_expanded = buffered_token_type_ids.expand(batch_size, seq_length)
    expand_1: "i64[1, s70]" = torch.ops.aten.expand.default(slice_1, [1, sym_size_int]);  slice_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:218 in forward, code: position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length]
    add_1: "Sym(s70)" = sym_size_int + 0
    slice_2: "i64[1, s70]" = torch.ops.aten.slice.Tensor(arg688_1, 1, 0, add_1);  arg688_1 = add_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\sparse.py:192 in forward, code: return F.embedding(
    embedding: "f32[1, s70, 128]" = torch.ops.aten.embedding.default(arg0_1, arg695_1, 0);  arg0_1 = arg695_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\sparse.py:192 in forward, code: return F.embedding(
    embedding_1: "f32[1, s70, 128]" = torch.ops.aten.embedding.default(arg2_1, expand_1);  arg2_1 = expand_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:235 in forward, code: embeddings = inputs_embeds + token_type_embeddings
    add_2: "f32[1, s70, 128]" = torch.ops.aten.add.Tensor(embedding, embedding_1);  embedding = embedding_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\sparse.py:192 in forward, code: return F.embedding(
    embedding_2: "f32[1, s70, 128]" = torch.ops.aten.embedding.default(arg1_1, slice_2);  arg1_1 = slice_2 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:238 in forward, code: embeddings += position_embeddings
    add_: "f32[1, s70, 128]" = torch.ops.aten.add_.Tensor(add_2, embedding_2);  add_2 = embedding_2 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\normalization.py:229 in forward, code: return F.layer_norm(
    layer_norm: "f32[1, s70, 128]" = torch.ops.aten.layer_norm.default(add_, [128], arg3_1, arg4_1, 1e-12);  add_ = arg3_1 = arg4_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\dropout.py:73 in forward, code: return F.dropout(input, self.p, self.training, self.inplace)
    dropout: "f32[1, s70, 128]" = torch.ops.aten.dropout.default(layer_norm, 0.0, False);  layer_norm = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:702 in forward, code: extended_attention_mask = _prepare_4d_attention_mask_for_sdpa(
    unsqueeze_2: "i32[1, 1, u0]" = torch.ops.aten.unsqueeze.default(to_1, 1);  to_1 = None
    unsqueeze_3: "i32[1, 1, 1, u0]" = torch.ops.aten.unsqueeze.default(unsqueeze_2, 2);  unsqueeze_2 = None
    sym_size_int_1: "Sym(u0)" = torch.ops.aten.sym_size.int(arange, 0);  arange = None
    le: "Sym(u0 <= 9223372036854775807)" = sym_size_int_1 <= 9223372036854775807;  le = None
    slice_3: "i32[1, 1, 1, u0]" = torch.ops.aten.slice.Tensor(unsqueeze_3, 3, 0, 9223372036854775807);  unsqueeze_3 = None
    expand_2: "i32[1, 1, s70, u0]" = torch.ops.aten.expand.default(slice_3, [1, 1, sym_size_int, sym_size_int_1]);  slice_3 = sym_size_int = sym_size_int_1 = None
    to_2: "f32[1, 1, s70, u0]" = torch.ops.aten.to.dtype(expand_2, torch.float32);  expand_2 = None
    _tensor_constant0: "f32[]" = self._tensor_constant0
    lift_fresh_copy: "f32[]" = torch.ops.aten.lift_fresh_copy.default(_tensor_constant0);  _tensor_constant0 = None
    detach_: "f32[]" = torch.ops.aten.detach_.default(lift_fresh_copy);  lift_fresh_copy = None
    sub: "f32[1, 1, s70, u0]" = torch.ops.aten.sub.Tensor(detach_, to_2);  detach_ = to_2 = None
    to_3: "b8[1, 1, s70, u0]" = torch.ops.aten.to.dtype(sub, torch.bool)
    masked_fill: "f32[1, 1, s70, u0]" = torch.ops.aten.masked_fill.Scalar(sub, to_3, -3.4028234663852886e+38);  sub = to_3 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\linear.py:134 in forward, code: return F.linear(input, self.weight, self.bias)
    linear: "f32[1, s70, 768]" = torch.ops.aten.linear.default(dropout, arg5_1, arg6_1);  dropout = arg5_1 = arg6_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\linear.py:134 in forward, code: return F.linear(input, self.weight, self.bias)
    linear_1: "f32[1, s70, 768]" = torch.ops.aten.linear.default(linear, arg9_1, arg10_1);  arg9_1 = arg10_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:379 in forward, code: .view(batch_size, -1, self.num_attention_heads, self.attention_head_size)
    view: "f32[1, s70, 12, 64]" = torch.ops.aten.view.default(linear_1, [1, -1, 12, 64]);  linear_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:380 in forward, code: .transpose(1, 2)
    transpose: "f32[1, 12, s70, 64]" = torch.ops.aten.transpose.int(view, 1, 2);  view = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\linear.py:134 in forward, code: return F.linear(input, self.weight, self.bias)
    linear_2: "f32[1, s70, 768]" = torch.ops.aten.linear.default(linear, arg11_1, arg12_1);  arg11_1 = arg12_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:384 in forward, code: .view(batch_size, -1, self.num_attention_heads, self.attention_head_size)
    view_1: "f32[1, s70, 12, 64]" = torch.ops.aten.view.default(linear_2, [1, -1, 12, 64]);  linear_2 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:385 in forward, code: .transpose(1, 2)
    transpose_1: "f32[1, 12, s70, 64]" = torch.ops.aten.transpose.int(view_1, 1, 2);  view_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\linear.py:134 in forward, code: return F.linear(input, self.weight, self.bias)
    linear_3: "f32[1, s70, 768]" = torch.ops.aten.linear.default(linear, arg13_1, arg14_1);  linear = arg13_1 = arg14_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:389 in forward, code: .view(batch_size, -1, self.num_attention_heads, self.attention_head_size)
    view_2: "f32[1, s70, 12, 64]" = torch.ops.aten.view.default(linear_3, [1, -1, 12, 64]);  linear_3 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:390 in forward, code: .transpose(1, 2)
    transpose_2: "f32[1, 12, s70, 64]" = torch.ops.aten.transpose.int(view_2, 1, 2);  view_2 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:393 in forward, code: attention_output = torch.nn.functional.scaled_dot_product_attention(
    scaled_dot_product_attention = torch.ops.aten.scaled_dot_product_attention.default(transpose, transpose_1, transpose_2, masked_fill);  transpose = transpose_1 = transpose_2 = masked_fill = scaled_dot_product_attention = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\kokoro\model.py:150 in forward, code: waveform, duration = self.kmodel.forward_with_tokens(input_ids, ref_s, speed)
    _set_grad_enabled_1 = torch._C._set_grad_enabled(True);  _set_grad_enabled_1 = None




def forward(self, arg0_1: "f32[178, 128]", arg1_1: "f32[512, 128]", arg2_1: "f32[2, 128]", arg3_1: "f32[128]", arg4_1: "f32[128]", arg5_1: "f32[768, 128]", arg6_1: "f32[768]", arg7_1: "f32[768]", arg8_1: "f32[768]", arg9_1: "f32[768, 768]", arg10_1: "f32[768]", arg11_1: "f32[768, 768]", arg12_1: "f32[768]", arg13_1: "f32[768, 768]", arg14_1: "f32[768]", arg15_1: "f32[768, 768]", arg16_1: "f32[768]", arg17_1: "f32[768]", arg18_1: "f32[768]", arg19_1: "f32[2048, 768]", arg20_1: "f32[2048]", arg21_1: "f32[768, 2048]", arg22_1: "f32[768]", arg23_1: "f32[768, 768]", arg24_1: "f32[768]", arg25_1: "f32[512, 768]", arg26_1: "f32[512]", arg27_1: "f32[1024, 640]", arg28_1: "f32[1024, 256]", arg29_1: "f32[1024]", arg30_1: "f32[1024]", arg31_1: "f32[1024, 640]", arg32_1: "f32[1024, 256]", arg33_1: "f32[1024]", arg34_1: "f32[1024]", arg35_1: "f32[1024, 128]", arg36_1: "f32[1024]", arg37_1: "f32[1024, 640]", arg38_1: "f32[1024, 256]", arg39_1: "f32[1024]", arg40_1: "f32[1024]", arg41_1: "f32[1024, 640]", arg42_1: "f32[1024, 256]", arg43_1: "f32[1024]", arg44_1: "f32[1024]", arg45_1: "f32[1024, 128]", arg46_1: "f32[1024]", arg47_1: "f32[1024, 640]", arg48_1: "f32[1024, 256]", arg49_1: "f32[1024]", arg50_1: "f32[1024]", arg51_1: "f32[1024, 640]", arg52_1: "f32[1024, 256]", arg53_1: "f32[1024]", arg54_1: "f32[1024]", arg55_1: "f32[1024, 128]", arg56_1: "f32[1024]", arg57_1: "f32[1024, 640]", arg58_1: "f32[1024, 256]", arg59_1: "f32[1024]", arg60_1: "f32[1024]", arg61_1: "f32[1024, 640]", arg62_1: "f32[1024, 256]", arg63_1: "f32[1024]", arg64_1: "f32[1024]", arg65_1: "f32[50, 512]", arg66_1: "f32[50]", arg67_1: "f32[1024, 640]", arg68_1: "f32[1024, 256]", arg69_1: "f32[1024]", arg70_1: "f32[1024]", arg71_1: "f32[1024, 640]", arg72_1: "f32[1024, 256]", arg73_1: "f32[1024]", arg74_1: "f32[1024]", arg75_1: "f32[512]", arg76_1: "f32[512, 1, 1]", arg77_1: "f32[512, 512, 3]", arg78_1: "f32[512]", arg79_1: "f32[512, 1, 1]", arg80_1: "f32[512, 512, 3]", arg81_1: "f32[512]", arg82_1: "f32[512]", arg83_1: "f32[1024, 128]", arg84_1: "f32[1024]", arg85_1: "f32[512]", arg86_1: "f32[512]", arg87_1: "f32[1024, 128]", arg88_1: "f32[1024]", arg89_1: "f32[256]", arg90_1: "f32[256, 1, 1]", arg91_1: "f32[256, 512, 3]", arg92_1: "f32[256]", arg93_1: "f32[256, 1, 1]", arg94_1: "f32[256, 256, 3]", arg95_1: "f32[512]", arg96_1: "f32[512]", arg97_1: "f32[1024, 128]", arg98_1: "f32[1024]", arg99_1: "f32[256]", arg100_1: "f32[256]", arg101_1: "f32[512, 128]", arg102_1: "f32[512]", arg103_1: "f32[256, 1, 1]", arg104_1: "f32[256, 512, 1]", arg105_1: "f32[512]", arg106_1: "f32[512, 1, 1]", arg107_1: "f32[512, 1, 3]", arg108_1: "f32[256]", arg109_1: "f32[256, 1, 1]", arg110_1: "f32[256, 256, 3]", arg111_1: "f32[256]", arg112_1: "f32[256, 1, 1]", arg113_1: "f32[256, 256, 3]", arg114_1: "f32[256]", arg115_1: "f32[256]", arg116_1: "f32[512, 128]", arg117_1: "f32[512]", arg118_1: "f32[256]", arg119_1: "f32[256]", arg120_1: "f32[512, 128]", arg121_1: "f32[512]", arg122_1: "f32[512]", arg123_1: "f32[512, 1, 1]", arg124_1: "f32[512, 512, 3]", arg125_1: "f32[512]", arg126_1: "f32[512, 1, 1]", arg127_1: "f32[512, 512, 3]", arg128_1: "f32[512]", arg129_1: "f32[512]", arg130_1: "f32[1024, 128]", arg131_1: "f32[1024]", arg132_1: "f32[512]", arg133_1: "f32[512]", arg134_1: "f32[1024, 128]", arg135_1: "f32[1024]", arg136_1: "f32[256]", arg137_1: "f32[256, 1, 1]", arg138_1: "f32[256, 512, 3]", arg139_1: "f32[256]", arg140_1: "f32[256, 1, 1]", arg141_1: "f32[256, 256, 3]", arg142_1: "f32[512]", arg143_1: "f32[512]", arg144_1: "f32[1024, 128]", arg145_1: "f32[1024]", arg146_1: "f32[256]", arg147_1: "f32[256]", arg148_1: "f32[512, 128]", arg149_1: "f32[512]", arg150_1: "f32[256, 1, 1]", arg151_1: "f32[256, 512, 1]", arg152_1: "f32[512]", arg153_1: "f32[512, 1, 1]", arg154_1: "f32[512, 1, 3]", arg155_1: "f32[256]", arg156_1: "f32[256, 1, 1]", arg157_1: "f32[256, 256, 3]", arg158_1: "f32[256]", arg159_1: "f32[256, 1, 1]", arg160_1: "f32[256, 256, 3]", arg161_1: "f32[256]", arg162_1: "f32[256]", arg163_1: "f32[512, 128]", arg164_1: "f32[512]", arg165_1: "f32[256]", arg166_1: "f32[256]", arg167_1: "f32[512, 128]", arg168_1: "f32[512]", arg169_1: "f32[1, 256, 1]", arg170_1: "f32[1]", arg171_1: "f32[1, 256, 1]", arg172_1: "f32[1]", arg173_1: "f32[178, 512]", arg174_1: "f32[512]", arg175_1: "f32[512, 1, 1]", arg176_1: "f32[512, 512, 5]", arg177_1: "f32[512]", arg178_1: "f32[512]", arg179_1: "f32[512]", arg180_1: "f32[512, 1, 1]", arg181_1: "f32[512, 512, 5]", arg182_1: "f32[512]", arg183_1: "f32[512]", arg184_1: "f32[512]", arg185_1: "f32[512, 1, 1]", arg186_1: "f32[512, 512, 5]", arg187_1: "f32[512]", arg188_1: "f32[512]", arg189_1: "f32[1024, 512]", arg190_1: "f32[1024, 256]", arg191_1: "f32[1024]", arg192_1: "f32[1024]", arg193_1: "f32[1024, 512]", arg194_1: "f32[1024, 256]", arg195_1: "f32[1024]", arg196_1: "f32[1024]", arg197_1: "f32[1024]", arg198_1: "f32[1024, 1, 1]", arg199_1: "f32[1024, 514, 3]", arg200_1: "f32[1024]", arg201_1: "f32[1024, 1, 1]", arg202_1: "f32[1024, 1024, 3]", arg203_1: "f32[514]", arg204_1: "f32[514]", arg205_1: "f32[1028, 128]", arg206_1: "f32[1028]", arg207_1: "f32[1024]", arg208_1: "f32[1024]", arg209_1: "f32[2048, 128]", arg210_1: "f32[2048]", arg211_1: "f32[1024, 1, 1]", arg212_1: "f32[1024, 514, 1]", arg213_1: "f32[1024]", arg214_1: "f32[1024, 1, 1]", arg215_1: "f32[1024, 1090, 3]", arg216_1: "f32[1024]", arg217_1: "f32[1024, 1, 1]", arg218_1: "f32[1024, 1024, 3]", arg219_1: "f32[1090]", arg220_1: "f32[1090]", arg221_1: "f32[2180, 128]", arg222_1: "f32[2180]", arg223_1: "f32[1024]", arg224_1: "f32[1024]", arg225_1: "f32[2048, 128]", arg226_1: "f32[2048]", arg227_1: "f32[1024, 1, 1]", arg228_1: "f32[1024, 1090, 1]", arg229_1: "f32[1024]", arg230_1: "f32[1024, 1, 1]", arg231_1: "f32[1024, 1090, 3]", arg232_1: "f32[1024]", arg233_1: "f32[1024, 1, 1]", arg234_1: "f32[1024, 1024, 3]", arg235_1: "f32[1090]", arg236_1: "f32[1090]", arg237_1: "f32[2180, 128]", arg238_1: "f32[2180]", arg239_1: "f32[1024]", arg240_1: "f32[1024]", arg241_1: "f32[2048, 128]", arg242_1: "f32[2048]", arg243_1: "f32[1024, 1, 1]", arg244_1: "f32[1024, 1090, 1]", arg245_1: "f32[1024]", arg246_1: "f32[1024, 1, 1]", arg247_1: "f32[1024, 1090, 3]", arg248_1: "f32[1024]", arg249_1: "f32[1024, 1, 1]", arg250_1: "f32[1024, 1024, 3]", arg251_1: "f32[1090]", arg252_1: "f32[1090]", arg253_1: "f32[2180, 128]", arg254_1: "f32[2180]", arg255_1: "f32[1024]", arg256_1: "f32[1024]", arg257_1: "f32[2048, 128]", arg258_1: "f32[2048]", arg259_1: "f32[1024, 1, 1]", arg260_1: "f32[1024, 1090, 1]", arg261_1: "f32[512]", arg262_1: "f32[512, 1, 1]", arg263_1: "f32[512, 1090, 3]", arg264_1: "f32[512]", arg265_1: "f32[512, 1, 1]", arg266_1: "f32[512, 512, 3]", arg267_1: "f32[1090]", arg268_1: "f32[1090]", arg269_1: "f32[2180, 128]", arg270_1: "f32[2180]", arg271_1: "f32[512]", arg272_1: "f32[512]", arg273_1: "f32[1024, 128]", arg274_1: "f32[1024]", arg275_1: "f32[512, 1, 1]", arg276_1: "f32[512, 1090, 1]", arg277_1: "f32[1090]", arg278_1: "f32[1090, 1, 1]", arg279_1: "f32[1090, 1, 3]", arg280_1: "f32[1]", arg281_1: "f32[1, 1, 1]", arg282_1: "f32[1, 1, 3]", arg283_1: "f32[1]", arg284_1: "f32[1, 1, 1]", arg285_1: "f32[1, 1, 3]", arg286_1: "f32[64]", arg287_1: "f32[64, 1, 1]", arg288_1: "f32[64, 512, 1]", arg289_1: "f32[1, 9]", arg290_1: "f32[1]", arg291_1: "f32[256, 22, 12]", arg292_1: "f32[256]", arg293_1: "f32[128, 22, 1]", arg294_1: "f32[128]", arg295_1: "f32[256]", arg296_1: "f32[256, 1, 1]", arg297_1: "f32[256, 256, 7]", arg298_1: "f32[256]", arg299_1: "f32[256, 1, 1]", arg300_1: "f32[256, 256, 7]", arg301_1: "f32[256]", arg302_1: "f32[256, 1, 1]", arg303_1: "f32[256, 256, 7]", arg304_1: "f32[256]", arg305_1: "f32[256, 1, 1]", arg306_1: "f32[256, 256, 7]", arg307_1: "f32[256]", arg308_1: "f32[256, 1, 1]", arg309_1: "f32[256, 256, 7]", arg310_1: "f32[256]", arg311_1: "f32[256, 1, 1]", arg312_1: "f32[256, 256, 7]", arg313_1: "f32[256]", arg314_1: "f32[256]", arg315_1: "f32[512, 128]", arg316_1: "f32[512]", arg317_1: "f32[256]", arg318_1: "f32[256]", arg319_1: "f32[512, 128]", arg320_1: "f32[512]", arg321_1: "f32[256]", arg322_1: "f32[256]", arg323_1: "f32[512, 128]", arg324_1: "f32[512]", arg325_1: "f32[256]", arg326_1: "f32[256]", arg327_1: "f32[512, 128]", arg328_1: "f32[512]", arg329_1: "f32[256]", arg330_1: "f32[256]", arg331_1: "f32[512, 128]", arg332_1: "f32[512]", arg333_1: "f32[256]", arg334_1: "f32[256]", arg335_1: "f32[512, 128]", arg336_1: "f32[512]", arg337_1: "f32[1, 256, 1]", arg338_1: "f32[1, 256, 1]", arg339_1: "f32[1, 256, 1]", arg340_1: "f32[1, 256, 1]", arg341_1: "f32[1, 256, 1]", arg342_1: "f32[1, 256, 1]", arg343_1: "f32[128]", arg344_1: "f32[128, 1, 1]", arg345_1: "f32[128, 128, 11]", arg346_1: "f32[128]", arg347_1: "f32[128, 1, 1]", arg348_1: "f32[128, 128, 11]", arg349_1: "f32[128]", arg350_1: "f32[128, 1, 1]", arg351_1: "f32[128, 128, 11]", arg352_1: "f32[128]", arg353_1: "f32[128, 1, 1]", arg354_1: "f32[128, 128, 11]", arg355_1: "f32[128]", arg356_1: "f32[128, 1, 1]", arg357_1: "f32[128, 128, 11]", arg358_1: "f32[128]", arg359_1: "f32[128, 1, 1]", arg360_1: "f32[128, 128, 11]", arg361_1: "f32[128]", arg362_1: "f32[128]", arg363_1: "f32[256, 128]", arg364_1: "f32[256]", arg365_1: "f32[128]", arg366_1: "f32[128]", arg367_1: "f32[256, 128]", arg368_1: "f32[256]", arg369_1: "f32[128]", arg370_1: "f32[128]", arg371_1: "f32[256, 128]", arg372_1: "f32[256]", arg373_1: "f32[128]", arg374_1: "f32[128]", arg375_1: "f32[256, 128]", arg376_1: "f32[256]", arg377_1: "f32[128]", arg378_1: "f32[128]", arg379_1: "f32[256, 128]", arg380_1: "f32[256]", arg381_1: "f32[128]", arg382_1: "f32[128]", arg383_1: "f32[256, 128]", arg384_1: "f32[256]", arg385_1: "f32[1, 128, 1]", arg386_1: "f32[1, 128, 1]", arg387_1: "f32[1, 128, 1]", arg388_1: "f32[1, 128, 1]", arg389_1: "f32[1, 128, 1]", arg390_1: "f32[1, 128, 1]", arg391_1: "f32[256]", arg392_1: "f32[512, 1, 1]", arg393_1: "f32[512, 256, 20]", arg394_1: "f32[128]", arg395_1: "f32[256, 1, 1]", arg396_1: "f32[256, 128, 12]", arg397_1: "f32[256]", arg398_1: "f32[256, 1, 1]", arg399_1: "f32[256, 256, 3]", arg400_1: "f32[256]", arg401_1: "f32[256, 1, 1]", arg402_1: "f32[256, 256, 3]", arg403_1: "f32[256]", arg404_1: "f32[256, 1, 1]", arg405_1: "f32[256, 256, 3]", arg406_1: "f32[256]", arg407_1: "f32[256, 1, 1]", arg408_1: "f32[256, 256, 3]", arg409_1: "f32[256]", arg410_1: "f32[256, 1, 1]", arg411_1: "f32[256, 256, 3]", arg412_1: "f32[256]", arg413_1: "f32[256, 1, 1]", arg414_1: "f32[256, 256, 3]", arg415_1: "f32[256]", arg416_1: "f32[256]", arg417_1: "f32[512, 128]", arg418_1: "f32[512]", arg419_1: "f32[256]", arg420_1: "f32[256]", arg421_1: "f32[512, 128]", arg422_1: "f32[512]", arg423_1: "f32[256]", arg424_1: "f32[256]", arg425_1: "f32[512, 128]", arg426_1: "f32[512]", arg427_1: "f32[256]", arg428_1: "f32[256]", arg429_1: "f32[512, 128]", arg430_1: "f32[512]", arg431_1: "f32[256]", arg432_1: "f32[256]", arg433_1: "f32[512, 128]", arg434_1: "f32[512]", arg435_1: "f32[256]", arg436_1: "f32[256]", arg437_1: "f32[512, 128]", arg438_1: "f32[512]", arg439_1: "f32[1, 256, 1]", arg440_1: "f32[1, 256, 1]", arg441_1: "f32[1, 256, 1]", arg442_1: "f32[1, 256, 1]", arg443_1: "f32[1, 256, 1]", arg444_1: "f32[1, 256, 1]", arg445_1: "f32[256]", arg446_1: "f32[256, 1, 1]", arg447_1: "f32[256, 256, 7]", arg448_1: "f32[256]", arg449_1: "f32[256, 1, 1]", arg450_1: "f32[256, 256, 7]", arg451_1: "f32[256]", arg452_1: "f32[256, 1, 1]", arg453_1: "f32[256, 256, 7]", arg454_1: "f32[256]", arg455_1: "f32[256, 1, 1]", arg456_1: "f32[256, 256, 7]", arg457_1: "f32[256]", arg458_1: "f32[256, 1, 1]", arg459_1: "f32[256, 256, 7]", arg460_1: "f32[256]", arg461_1: "f32[256, 1, 1]", arg462_1: "f32[256, 256, 7]", arg463_1: "f32[256]", arg464_1: "f32[256]", arg465_1: "f32[512, 128]", arg466_1: "f32[512]", arg467_1: "f32[256]", arg468_1: "f32[256]", arg469_1: "f32[512, 128]", arg470_1: "f32[512]", arg471_1: "f32[256]", arg472_1: "f32[256]", arg473_1: "f32[512, 128]", arg474_1: "f32[512]", arg475_1: "f32[256]", arg476_1: "f32[256]", arg477_1: "f32[512, 128]", arg478_1: "f32[512]", arg479_1: "f32[256]", arg480_1: "f32[256]", arg481_1: "f32[512, 128]", arg482_1: "f32[512]", arg483_1: "f32[256]", arg484_1: "f32[256]", arg485_1: "f32[512, 128]", arg486_1: "f32[512]", arg487_1: "f32[1, 256, 1]", arg488_1: "f32[1, 256, 1]", arg489_1: "f32[1, 256, 1]", arg490_1: "f32[1, 256, 1]", arg491_1: "f32[1, 256, 1]", arg492_1: "f32[1, 256, 1]", arg493_1: "f32[256]", arg494_1: "f32[256, 1, 1]", arg495_1: "f32[256, 256, 11]", arg496_1: "f32[256]", arg497_1: "f32[256, 1, 1]", arg498_1: "f32[256, 256, 11]", arg499_1: "f32[256]", arg500_1: "f32[256, 1, 1]", arg501_1: "f32[256, 256, 11]", arg502_1: "f32[256]", arg503_1: "f32[256, 1, 1]", arg504_1: "f32[256, 256, 11]", arg505_1: "f32[256]", arg506_1: "f32[256, 1, 1]", arg507_1: "f32[256, 256, 11]", arg508_1: "f32[256]", arg509_1: "f32[256, 1, 1]", arg510_1: "f32[256, 256, 11]", arg511_1: "f32[256]", arg512_1: "f32[256]", arg513_1: "f32[512, 128]", arg514_1: "f32[512]", arg515_1: "f32[256]", arg516_1: "f32[256]", arg517_1: "f32[512, 128]", arg518_1: "f32[512]", arg519_1: "f32[256]", arg520_1: "f32[256]", arg521_1: "f32[512, 128]", arg522_1: "f32[512]", arg523_1: "f32[256]", arg524_1: "f32[256]", arg525_1: "f32[512, 128]", arg526_1: "f32[512]", arg527_1: "f32[256]", arg528_1: "f32[256]", arg529_1: "f32[512, 128]", arg530_1: "f32[512]", arg531_1: "f32[256]", arg532_1: "f32[256]", arg533_1: "f32[512, 128]", arg534_1: "f32[512]", arg535_1: "f32[1, 256, 1]", arg536_1: "f32[1, 256, 1]", arg537_1: "f32[1, 256, 1]", arg538_1: "f32[1, 256, 1]", arg539_1: "f32[1, 256, 1]", arg540_1: "f32[1, 256, 1]", arg541_1: "f32[128]", arg542_1: "f32[128, 1, 1]", arg543_1: "f32[128, 128, 3]", arg544_1: "f32[128]", arg545_1: "f32[128, 1, 1]", arg546_1: "f32[128, 128, 3]", arg547_1: "f32[128]", arg548_1: "f32[128, 1, 1]", arg549_1: "f32[128, 128, 3]", arg550_1: "f32[128]", arg551_1: "f32[128, 1, 1]", arg552_1: "f32[128, 128, 3]", arg553_1: "f32[128]", arg554_1: "f32[128, 1, 1]", arg555_1: "f32[128, 128, 3]", arg556_1: "f32[128]", arg557_1: "f32[128, 1, 1]", arg558_1: "f32[128, 128, 3]", arg559_1: "f32[128]", arg560_1: "f32[128]", arg561_1: "f32[256, 128]", arg562_1: "f32[256]", arg563_1: "f32[128]", arg564_1: "f32[128]", arg565_1: "f32[256, 128]", arg566_1: "f32[256]", arg567_1: "f32[128]", arg568_1: "f32[128]", arg569_1: "f32[256, 128]", arg570_1: "f32[256]", arg571_1: "f32[128]", arg572_1: "f32[128]", arg573_1: "f32[256, 128]", arg574_1: "f32[256]", arg575_1: "f32[128]", arg576_1: "f32[128]", arg577_1: "f32[256, 128]", arg578_1: "f32[256]", arg579_1: "f32[128]", arg580_1: "f32[128]", arg581_1: "f32[256, 128]", arg582_1: "f32[256]", arg583_1: "f32[1, 128, 1]", arg584_1: "f32[1, 128, 1]", arg585_1: "f32[1, 128, 1]", arg586_1: "f32[1, 128, 1]", arg587_1: "f32[1, 128, 1]", arg588_1: "f32[1, 128, 1]", arg589_1: "f32[128]", arg590_1: "f32[128, 1, 1]", arg591_1: "f32[128, 128, 7]", arg592_1: "f32[128]", arg593_1: "f32[128, 1, 1]", arg594_1: "f32[128, 128, 7]", arg595_1: "f32[128]", arg596_1: "f32[128, 1, 1]", arg597_1: "f32[128, 128, 7]", arg598_1: "f32[128]", arg599_1: "f32[128, 1, 1]", arg600_1: "f32[128, 128, 7]", arg601_1: "f32[128]", arg602_1: "f32[128, 1, 1]", arg603_1: "f32[128, 128, 7]", arg604_1: "f32[128]", arg605_1: "f32[128, 1, 1]", arg606_1: "f32[128, 128, 7]", arg607_1: "f32[128]", arg608_1: "f32[128]", arg609_1: "f32[256, 128]", arg610_1: "f32[256]", arg611_1: "f32[128]", arg612_1: "f32[128]", arg613_1: "f32[256, 128]", arg614_1: "f32[256]", arg615_1: "f32[128]", arg616_1: "f32[128]", arg617_1: "f32[256, 128]", arg618_1: "f32[256]", arg619_1: "f32[128]", arg620_1: "f32[128]", arg621_1: "f32[256, 128]", arg622_1: "f32[256]", arg623_1: "f32[128]", arg624_1: "f32[128]", arg625_1: "f32[256, 128]", arg626_1: "f32[256]", arg627_1: "f32[128]", arg628_1: "f32[128]", arg629_1: "f32[256, 128]", arg630_1: "f32[256]", arg631_1: "f32[1, 128, 1]", arg632_1: "f32[1, 128, 1]", arg633_1: "f32[1, 128, 1]", arg634_1: "f32[1, 128, 1]", arg635_1: "f32[1, 128, 1]", arg636_1: "f32[1, 128, 1]", arg637_1: "f32[128]", arg638_1: "f32[128, 1, 1]", arg639_1: "f32[128, 128, 11]", arg640_1: "f32[128]", arg641_1: "f32[128, 1, 1]", arg642_1: "f32[128, 128, 11]", arg643_1: "f32[128]", arg644_1: "f32[128, 1, 1]", arg645_1: "f32[128, 128, 11]", arg646_1: "f32[128]", arg647_1: "f32[128, 1, 1]", arg648_1: "f32[128, 128, 11]", arg649_1: "f32[128]", arg650_1: "f32[128, 1, 1]", arg651_1: "f32[128, 128, 11]", arg652_1: "f32[128]", arg653_1: "f32[128, 1, 1]", arg654_1: "f32[128, 128, 11]", arg655_1: "f32[128]", arg656_1: "f32[128]", arg657_1: "f32[256, 128]", arg658_1: "f32[256]", arg659_1: "f32[128]", arg660_1: "f32[128]", arg661_1: "f32[256, 128]", arg662_1: "f32[256]", arg663_1: "f32[128]", arg664_1: "f32[128]", arg665_1: "f32[256, 128]", arg666_1: "f32[256]", arg667_1: "f32[128]", arg668_1: "f32[128]", arg669_1: "f32[256, 128]", arg670_1: "f32[256]", arg671_1: "f32[128]", arg672_1: "f32[128]", arg673_1: "f32[256, 128]", arg674_1: "f32[256]", arg675_1: "f32[128]", arg676_1: "f32[128]", arg677_1: "f32[256, 128]", arg678_1: "f32[256]", arg679_1: "f32[1, 128, 1]", arg680_1: "f32[1, 128, 1]", arg681_1: "f32[1, 128, 1]", arg682_1: "f32[1, 128, 1]", arg683_1: "f32[1, 128, 1]", arg684_1: "f32[1, 128, 1]", arg685_1: "f32[22]", arg686_1: "f32[22, 1, 1]", arg687_1: "f32[22, 128, 7]", arg688_1: "i64[1, 512]", arg689_1: "i64[1, 512]", arg690_1: "f32[20]", arg691_1: "f32[11, 1, 20]", arg692_1: "f32[11, 1, 20]", arg693_1: "f32[11, 1, 20]", arg694_1: "f32[11, 1, 20]", arg695_1: "i64[1, s70]", arg696_1: "f32[1, 256]", arg697_1: "i32[1]"):
     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\kokoro\model.py:150 in forward, code: waveform, duration = self.kmodel.forward_with_tokens(input_ids, ref_s, speed)
    _set_grad_enabled = torch._C._set_grad_enabled(False);  _set_grad_enabled = None
    sym_size_int: "Sym(s70)" = torch.ops.aten.sym_size.int(arg695_1, 1)
    full: "i64[1]" = torch.ops.aten.full.default([1], sym_size_int, dtype = torch.int64, device = device(type='cpu'), pin_memory = False)
    max_1: "i64[]" = torch.ops.aten.max.default(full)
    item: "Sym(u0)" = torch.ops.aten.item.default(max_1);  max_1 = None
    arange: "i64[u0]" = torch.ops.aten.arange.default(item, device = device(type='cpu'), pin_memory = False);  item = None
    unsqueeze: "i64[1, u0]" = torch.ops.aten.unsqueeze.default(arange, 0)
    expand: "i64[1, u0]" = torch.ops.aten.expand.default(unsqueeze, [1, -1]);  unsqueeze = None
    type_as: "i64[1, u0]" = torch.ops.aten.type_as.default(expand, full);  expand = None
    add: "i64[1, u0]" = torch.ops.aten.add.Tensor(type_as, 1);  type_as = None
    unsqueeze_1: "i64[1, 1]" = torch.ops.aten.unsqueeze.default(full, 1);  full = None
    gt: "b8[1, u0]" = torch.ops.aten.gt.Tensor(add, unsqueeze_1);  add = unsqueeze_1 = None
    to: "b8[1, u0]" = torch.ops.aten.to.dtype_layout(gt, dtype = torch.bool, layout = torch.strided, device = device(type='cpu'));  gt = None
    bitwise_not: "b8[1, u0]" = torch.ops.aten.bitwise_not.default(to);  to = None
    to_1: "i32[1, u0]" = torch.ops.aten.to.dtype(bitwise_not, torch.int32);  bitwise_not = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:684 in forward, code: buffered_token_type_ids = self.embeddings.token_type_ids[:, :seq_length]
    slice_1: "i64[1, s70]" = torch.ops.aten.slice.Tensor(arg689_1, 1, None, sym_size_int);  arg689_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:685 in forward, code: buffered_token_type_ids_expanded = buffered_token_type_ids.expand(batch_size, seq_length)
    expand_1: "i64[1, s70]" = torch.ops.aten.expand.default(slice_1, [1, sym_size_int]);  slice_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:218 in forward, code: position_ids = self.position_ids[:, past_key_values_length : seq_length + past_key_values_length]
    add_1: "Sym(s70)" = sym_size_int + 0
    slice_2: "i64[1, s70]" = torch.ops.aten.slice.Tensor(arg688_1, 1, 0, add_1);  arg688_1 = add_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\sparse.py:192 in forward, code: return F.embedding(
    embedding: "f32[1, s70, 128]" = torch.ops.aten.embedding.default(arg0_1, arg695_1, 0);  arg0_1 = arg695_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\sparse.py:192 in forward, code: return F.embedding(
    embedding_1: "f32[1, s70, 128]" = torch.ops.aten.embedding.default(arg2_1, expand_1);  arg2_1 = expand_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:235 in forward, code: embeddings = inputs_embeds + token_type_embeddings
    add_2: "f32[1, s70, 128]" = torch.ops.aten.add.Tensor(embedding, embedding_1);  embedding = embedding_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\sparse.py:192 in forward, code: return F.embedding(
    embedding_2: "f32[1, s70, 128]" = torch.ops.aten.embedding.default(arg1_1, slice_2);  arg1_1 = slice_2 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:238 in forward, code: embeddings += position_embeddings
    add_: "f32[1, s70, 128]" = torch.ops.aten.add_.Tensor(add_2, embedding_2);  add_2 = embedding_2 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\normalization.py:229 in forward, code: return F.layer_norm(
    layer_norm: "f32[1, s70, 128]" = torch.ops.aten.layer_norm.default(add_, [128], arg3_1, arg4_1, 1e-12);  add_ = arg3_1 = arg4_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\dropout.py:73 in forward, code: return F.dropout(input, self.p, self.training, self.inplace)
    dropout: "f32[1, s70, 128]" = torch.ops.aten.dropout.default(layer_norm, 0.0, False);  layer_norm = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:702 in forward, code: extended_attention_mask = _prepare_4d_attention_mask_for_sdpa(
    unsqueeze_2: "i32[1, 1, u0]" = torch.ops.aten.unsqueeze.default(to_1, 1);  to_1 = None
    unsqueeze_3: "i32[1, 1, 1, u0]" = torch.ops.aten.unsqueeze.default(unsqueeze_2, 2);  unsqueeze_2 = None
    sym_size_int_1: "Sym(u0)" = torch.ops.aten.sym_size.int(arange, 0);  arange = None
    le: "Sym(u0 <= 9223372036854775807)" = sym_size_int_1 <= 9223372036854775807;  le = None
    slice_3: "i32[1, 1, 1, u0]" = torch.ops.aten.slice.Tensor(unsqueeze_3, 3, 0, 9223372036854775807);  unsqueeze_3 = None
    expand_2: "i32[1, 1, s70, u0]" = torch.ops.aten.expand.default(slice_3, [1, 1, sym_size_int, sym_size_int_1]);  slice_3 = sym_size_int = sym_size_int_1 = None
    to_2: "f32[1, 1, s70, u0]" = torch.ops.aten.to.dtype(expand_2, torch.float32);  expand_2 = None
    _tensor_constant0: "f32[]" = self._tensor_constant0
    lift_fresh_copy: "f32[]" = torch.ops.aten.lift_fresh_copy.default(_tensor_constant0);  _tensor_constant0 = None
    detach_: "f32[]" = torch.ops.aten.detach_.default(lift_fresh_copy);  lift_fresh_copy = None
    sub: "f32[1, 1, s70, u0]" = torch.ops.aten.sub.Tensor(detach_, to_2);  detach_ = to_2 = None
    to_3: "b8[1, 1, s70, u0]" = torch.ops.aten.to.dtype(sub, torch.bool)
    masked_fill: "f32[1, 1, s70, u0]" = torch.ops.aten.masked_fill.Scalar(sub, to_3, -3.4028234663852886e+38);  sub = to_3 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\linear.py:134 in forward, code: return F.linear(input, self.weight, self.bias)
    linear: "f32[1, s70, 768]" = torch.ops.aten.linear.default(dropout, arg5_1, arg6_1);  dropout = arg5_1 = arg6_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\linear.py:134 in forward, code: return F.linear(input, self.weight, self.bias)
    linear_1: "f32[1, s70, 768]" = torch.ops.aten.linear.default(linear, arg9_1, arg10_1);  arg9_1 = arg10_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:379 in forward, code: .view(batch_size, -1, self.num_attention_heads, self.attention_head_size)
    view: "f32[1, s70, 12, 64]" = torch.ops.aten.view.default(linear_1, [1, -1, 12, 64]);  linear_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:380 in forward, code: .transpose(1, 2)
    transpose: "f32[1, 12, s70, 64]" = torch.ops.aten.transpose.int(view, 1, 2);  view = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\linear.py:134 in forward, code: return F.linear(input, self.weight, self.bias)
    linear_2: "f32[1, s70, 768]" = torch.ops.aten.linear.default(linear, arg11_1, arg12_1);  arg11_1 = arg12_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:384 in forward, code: .view(batch_size, -1, self.num_attention_heads, self.attention_head_size)
    view_1: "f32[1, s70, 12, 64]" = torch.ops.aten.view.default(linear_2, [1, -1, 12, 64]);  linear_2 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:385 in forward, code: .transpose(1, 2)
    transpose_1: "f32[1, 12, s70, 64]" = torch.ops.aten.transpose.int(view_1, 1, 2);  view_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\linear.py:134 in forward, code: return F.linear(input, self.weight, self.bias)
    linear_3: "f32[1, s70, 768]" = torch.ops.aten.linear.default(linear, arg13_1, arg14_1);  linear = arg13_1 = arg14_1 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:389 in forward, code: .view(batch_size, -1, self.num_attention_heads, self.attention_head_size)
    view_2: "f32[1, s70, 12, 64]" = torch.ops.aten.view.default(linear_3, [1, -1, 12, 64]);  linear_3 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:390 in forward, code: .transpose(1, 2)
    transpose_2: "f32[1, 12, s70, 64]" = torch.ops.aten.transpose.int(view_2, 1, 2);  view_2 = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py:393 in forward, code: attention_output = torch.nn.functional.scaled_dot_product_attention(
    scaled_dot_product_attention = torch.ops.aten.scaled_dot_product_attention.default(transpose, transpose_1, transpose_2, masked_fill);  transpose = transpose_1 = transpose_2 = masked_fill = scaled_dot_product_attention = None

     # File: C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\kokoro\model.py:150 in forward, code: waveform, duration = self.kmodel.forward_with_tokens(input_ids, ref_s, speed)
    _set_grad_enabled_1 = torch._C._set_grad_enabled(True);  _set_grad_enabled_1 = None

[torch.onnx] Obtain model graph for `KModelForONNX([...]` with `torch.export.export(..., strict=False)`... ❌
[torch.onnx] Obtain model graph for `KModelForONNX([...]` with `torch.export.export(..., strict=True)`...
[torch.onnx] Obtain model graph for `KModelForONNX([...]` with `torch.export.export(..., strict=True)`... ❌
Traceback (most recent call last):
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\onnx\_internal\exporter\_capture_strategies.py", line 118, in __call__
    exported_program = self._capture(model, args, kwargs, dynamic_shapes)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\onnx\_internal\exporter\_capture_strategies.py", line 210, in _capture
    return torch.export.export(
           ~~~~~~~~~~~~~~~~~~~^
        model,
        ^^^^^^
    ...<3 lines>...
        strict=False,
        ^^^^^^^^^^^^^
    )
    ^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\export\__init__.py", line 311, in export
    raise e
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\export\__init__.py", line 277, in export
    return _export(
        mod,
    ...<6 lines>...
        prefer_deferred_runtime_asserts_over_guards=prefer_deferred_runtime_asserts_over_guards,
    )
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\export\_trace.py", line 1163, in wrapper
    raise e
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\export\_trace.py", line 1129, in wrapper
    ep = fn(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\export\exported_program.py", line 124, in wrapper
    return fn(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\export\_trace.py", line 2255, in _export
    ep = _export_for_training(
        mod,
    ...<5 lines>...
        prefer_deferred_runtime_asserts_over_guards=prefer_deferred_runtime_asserts_over_guards,
    )
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\export\_trace.py", line 1163, in wrapper
    raise e
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\export\_trace.py", line 1129, in wrapper
    ep = fn(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\export\exported_program.py", line 124, in wrapper
    return fn(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\export\_trace.py", line 2071, in _export_for_training
    export_artifact = export_func(
        mod=mod,
    ...<6 lines>...
        _to_aten_func=_export_to_aten_ir_make_fx,
    )
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\export\_trace.py", line 2002, in _non_strict_export
    aten_export_artifact = _to_aten_func(  # type: ignore[operator]
        patched_mod,
    ...<5 lines>...
        transform=_tuplify_outputs,
    )
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\export\_trace.py", line 1793, in _export_to_aten_ir_make_fx
    gm, graph_signature = transform(_make_fx_helper)(
                          ~~~~~~~~~~~~~~~~~~~~~~~~~~^
        mod,
        ^^^^
    ...<2 lines>...
        kwargs=fake_kwargs,
        ^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\export\_trace.py", line 1922, in _aot_export_non_strict
    gm, sig = aot_export(wrapped_mod, args, kwargs=kwargs, **flags)
              ~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\export\_trace.py", line 1706, in _make_fx_helper
    gm = make_fx(
    ...<2 lines>...
        pre_dispatch=True,
    )(*flat_args)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 2429, in wrapped
    return make_fx_tracer.trace(f, *args)
           ~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 2356, in trace
    return self._trace_inner(f, *args)
           ~~~~~~~~~~~~~~~~~^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 2318, in _trace_inner
    t = dispatch_trace(
        wrap_key(func, args, self.fx_tracer, self.pre_dispatch),
        tracer=self.fx_tracer,
        concrete_args=tuple(phs),
    )
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\_compile.py", line 53, in inner
    return disable_fn(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\_dynamo\eval_frame.py", line 1044, in _fn
    return fn(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 1303, in dispatch_trace
    graph = tracer.trace(root, concrete_args)  # type: ignore[arg-type]
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 1908, in trace
    res = super().trace(root, concrete_args)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 868, in trace
    (self.create_arg(fn(*args)),),
                     ~~^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 1361, in wrapped
    out = f(*tensors)  # type:ignore[call-arg]
  File "<string>", line 1, in <lambda>
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\export\_trace.py", line 1593, in wrapped_fn
    return tuple(flat_fn(*args))
                 ~~~~~~~^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\_functorch\_aot_autograd\utils.py", line 187, in flat_fn
    tree_out = fn(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\_functorch\_aot_autograd\graph_capture_wrappers.py", line 1354, in functional_call
    out = mod(*args[params_len:], **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 843, in module_call_wrapper
    return self.call_module(mod, forward, args, kwargs)
           ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 1997, in call_module
    return Tracer.call_module(self, m, forward, args, kwargs)
           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 560, in call_module
    ret_val = forward(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 836, in forward
    return _orig_module_call(mod, *args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\module.py", line 1775, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\module.py", line 1786, in _call_impl
    return forward_call(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\export\_trace.py", line 1906, in forward
    tree_out = mod(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 843, in module_call_wrapper
    return self.call_module(mod, forward, args, kwargs)
           ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 1997, in call_module
    return Tracer.call_module(self, m, forward, args, kwargs)
           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 560, in call_module
    ret_val = forward(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 836, in forward
    return _orig_module_call(mod, *args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\module.py", line 1775, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\module.py", line 1786, in _call_impl
    return forward_call(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\kokoro\model.py", line 150, in forward
    waveform, duration = self.kmodel.forward_with_tokens(input_ids, ref_s, speed)
                         ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\utils\_contextlib.py", line 120, in decorate_context
    return func(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\kokoro\model.py", line 102, in forward_with_tokens
    bert_dur = self.bert(input_ids, attention_mask=(~text_mask).int())
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 843, in module_call_wrapper
    return self.call_module(mod, forward, args, kwargs)
           ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 1997, in call_module
    return Tracer.call_module(self, m, forward, args, kwargs)
           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 560, in call_module
    ret_val = forward(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 836, in forward
    return _orig_module_call(mod, *args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\module.py", line 1775, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\module.py", line 1786, in _call_impl
    return forward_call(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\kokoro\modules.py", line 182, in forward
    outputs = super().forward(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py", line 712, in forward
    encoder_outputs = self.encoder(
        embedding_output,
    ...<4 lines>...
        return_dict=return_dict,
    )
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 843, in module_call_wrapper
    return self.call_module(mod, forward, args, kwargs)
           ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 1997, in call_module
    return Tracer.call_module(self, m, forward, args, kwargs)
           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 560, in call_module
    ret_val = forward(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 836, in forward
    return _orig_module_call(mod, *args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\module.py", line 1775, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\module.py", line 1786, in _call_impl
    return forward_call(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py", line 524, in forward
    layer_group_output = self.albert_layer_groups[group_idx](
        hidden_states,
    ...<3 lines>...
        output_hidden_states,
    )
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 843, in module_call_wrapper
    return self.call_module(mod, forward, args, kwargs)
           ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 1997, in call_module
    return Tracer.call_module(self, m, forward, args, kwargs)
           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 560, in call_module
    ret_val = forward(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 836, in forward
    return _orig_module_call(mod, *args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\module.py", line 1775, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\module.py", line 1786, in _call_impl
    return forward_call(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py", line 476, in forward
    layer_output = albert_layer(hidden_states, attention_mask, head_mask[layer_index], output_attentions)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 843, in module_call_wrapper
    return self.call_module(mod, forward, args, kwargs)
           ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 1997, in call_module
    return Tracer.call_module(self, m, forward, args, kwargs)
           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 560, in call_module
    ret_val = forward(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 836, in forward
    return _orig_module_call(mod, *args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\module.py", line 1775, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\module.py", line 1786, in _call_impl
    return forward_call(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py", line 439, in forward
    attention_output = self.attention(hidden_states, attention_mask, head_mask, output_attentions)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 843, in module_call_wrapper
    return self.call_module(mod, forward, args, kwargs)
           ~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 1997, in call_module
    return Tracer.call_module(self, m, forward, args, kwargs)
           ~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 560, in call_module
    ret_val = forward(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\_symbolic_trace.py", line 836, in forward
    return _orig_module_call(mod, *args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\module.py", line 1775, in _wrapped_call_impl
    return self._call_impl(*args, **kwargs)
           ~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\nn\modules\module.py", line 1786, in _call_impl
    return forward_call(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py", line 393, in forward
    attention_output = torch.nn.functional.scaled_dot_product_attention(
        query=query_layer,
    ...<4 lines>...
        is_causal=False,
    )
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 1409, in __torch_function__
    return func(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 1479, in __torch_function__
    return func(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\_export\non_strict_utils.py", line 1066, in __torch_function__
    return func(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\_ops.py", line 962, in handler
    return torch._library.utils.handle_dispatch_mode(
           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^
        curr_mode, self, *args, **kwargs
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\_library\utils.py", line 286, in handle_dispatch_mode
    return curr_mode.__torch_dispatch__(op_overload, overload_types, args, kwargs)
           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\utils\_stats.py", line 28, in wrapper
    return fn(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 1534, in __torch_dispatch__
    return proxy_call(self, func, self.pre_dispatch, args, kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\proxy_tensor.py", line 994, in proxy_call
    out = func(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\_ops.py", line 841, in __call__
    return self._op(*args, **kwargs)
           ~~~~~~~~^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\sym_node.py", line 538, in guard_bool
    r = self.evaluate()
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\sym_node.py", line 512, in evaluate
    return self.shape_env.evaluate_sym_node(self, size_oblivious)
           ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\symbolic_shapes.py", line 7233, in evaluate_sym_node
    return self.evaluate_expr(
           ~~~~~~~~~~~~~~~~~~^
        sym_node.expr,
        ^^^^^^^^^^^^^^
    ...<3 lines>...
        fallback_value=fallback_value,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\symbolic_shapes.py", line 7333, in evaluate_expr
    return self._inner_evaluate_expr(
           ~~~~~~~~~~~~~~~~~~~~~~~~~^
        orig_expr,
        ^^^^^^^^^^
    ...<5 lines>...
        fallback_value,
        ^^^^^^^^^^^^^^^
    )
    ^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\recording.py", line 272, in wrapper
    return retlog(fn(*args, **kwargs))
                  ~~^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\symbolic_shapes.py", line 7356, in _inner_evaluate_expr
    return self._evaluate_expr(
           ~~~~~~~~~~~~~~~~~~~^
        orig_expr,
        ^^^^^^^^^^
    ...<4 lines>...
        forcing_spec=forcing_spec,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\fx\experimental\symbolic_shapes.py", line 7574, in _evaluate_expr
    raise self._make_data_dependent_error(
    ...<3 lines>...
    )
torch.fx.experimental.symbolic_shapes.GuardOnDataDependentSymNode: Could not guard on data-dependent expression Ne(u0, 50) (unhinted: Ne(u0, s70)).  (Size-like symbols: u0)

consider using data-dependent friendly APIs such as guard_or_false, guard_or_true and statically_known_trueCaused by: (_ops.py:841 in __call__)
For more information, run with TORCH_LOGS="dynamic"
For extended logs when we create symbols, also add TORCHDYNAMO_EXTENDED_DEBUG_CREATE_SYMBOL="u0"
If you suspect the guard was triggered from C++, add TORCHDYNAMO_EXTENDED_DEBUG_CPP=1
For more debugging help, see https://docs.google.com/document/d/1HSuTTVvYH1pTew89Rtpeu84Ht3nQEFTYhAX3Ypa_xJs/edit?usp=sharing

For C++ stack trace, run with TORCHDYNAMO_EXTENDED_DEBUG_CPP=1

The following call raised this error:
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py", line 393, in forward
    attention_output = torch.nn.functional.scaled_dot_product_attention(

To fix the error, insert one of the following checks before this call:
  1. torch._check(attention_mask.shape[3] != 50)
  2. torch._check(attention_mask.shape[3] == 50)

(These suggested fixes were derived by replacing `u0` with attention_mask.shape[3] in Ne(u0, 50) and its negation.)

The error above occurred when calling torch.export.export. If you would like to view some more information about this error, and get a list of all other errors that may occur in your export call, you can replace your `export()` call with `draft_export()`.

The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\export.py", line 171, in <module>
    export_onnx(model, output_dir)
    ~~~~~~~~~~~^^^^^^^^^^^^^^^^^^^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\export.py", line 21, in export_onnx
    torch.onnx.export(
    ~~~~~~~~~~~~~~~~~^
        model,
        ^^^^^^
    ...<11 lines>...
        do_constant_folding=True,
        ^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\onnx\__init__.py", line 296, in export
    return _compat.export_compat(
           ~~~~~~~~~~~~~~~~~~~~~^
        model,
        ^^^^^^
    ...<20 lines>...
        legacy_export_kwargs=legacy_export_kwargs,
        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
    )
    ^
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\onnx\_internal\exporter\_compat.py", line 143, in export_compat
    onnx_program = _core.export(
        model,
    ...<11 lines>...
        verbose=verbose,
    )
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\onnx\_internal\exporter\_flags.py", line 23, in wrapper
    return func(*args, **kwargs)
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\torch\onnx\_internal\exporter\_core.py", line 1385, in export
    raise _errors.TorchExportError(
    ...<7 lines>...
    ) from first_error
torch.onnx._internal.exporter._errors.TorchExportError: Failed to export the model with torch.export. This is step 1/3 of exporting the model to ONNX. Next steps:
- Modify the model code for `torch.export.export` to succeed. Refer to https://pytorch.org/docs/stable/generated/exportdb/index.html for more information.
- Debug `torch.export.export` and submit a PR to PyTorch.
- Create an issue in the PyTorch GitHub repository against the *torch.export* component and attach the full error stack as well as reproduction scripts.

## Exception summary

<class 'torch.fx.experimental.symbolic_shapes.GuardOnDataDependentSymNode'>: Could not guard on data-dependent expression Ne(u0, 50) (unhinted: Ne(u0, s70)).  (Size-like symbols: u0)

consider using data-dependent friendly APIs such as guard_or_false, guard_or_true and statically_known_trueCaused by: (_ops.py:841 in __call__)
For more information, run with TORCH_LOGS="dynamic"
For extended logs when we create symbols, also add TORCHDYNAMO_EXTENDED_DEBUG_CREATE_SYMBOL="u0"
If you suspect the guard was triggered from C++, add TORCHDYNAMO_EXTENDED_DEBUG_CPP=1
For more debugging help, see https://docs.google.com/document/d/1HSuTTVvYH1pTew89Rtpeu84Ht3nQEFTYhAX3Ypa_xJs/edit?usp=sharing

For C++ stack trace, run with TORCHDYNAMO_EXTENDED_DEBUG_CPP=1

The following call raised this error:
  File "C:\Users\admin\code\other\Kokoro-82M-v1.1-zh\.venv\Lib\site-packages\transformers\models\albert\modeling_albert.py", line 393, in forward
    attention_output = torch.nn.functional.scaled_dot_product_attention(

To fix the error, insert one of the following checks before this call:
  1. torch._check(attention_mask.shape[3] != 50)
  2. torch._check(attention_mask.shape[3] == 50)

(These suggested fixes were derived by replacing `u0` with attention_mask.shape[3] in Ne(u0, 50) and its negation.)

The error above occurred when calling torch.export.export. If you would like to view some more information about this error, and get a list of all other errors that may occur in your export call, you can replace your `export()` call with `draft_export()`.

(Refer to the full stack trace above for more information.)

(kokoro-82m-v1-1-zh) C:\Users\admin\code\other\Kokoro-82M-v1.1-zh>

And I also tried to convert the https://github.com/thewh1teagle/kokoro-onnx/releases/download/model-files-v1.1/kokoro-v1.1-zh.onnx to the fp16 and int8 version.

But due to the onnxsim install failure. I can only convert to int8 version.

import os
import logging
from pathlib import Path
from onnxruntime.quantization import quantize_dynamic, QuantType
import onnx # 用于 FP16 转换和模型加载
import json # 用于获取 style_dim 参数 (如果需要 FP16 转换)
# 尝试导入 onnxsim
try:
    import onnxsim
except ImportError:
    onnxsim = None
    logging.warning("onnxsim 库未安装，FP16 转换将被跳过。请运行 'pip install onnxsim'")


# --- 配置和路径设定 ---
# ❗ 请将此路径设置为您现有的 ONNX 文件
FP32_ONNX_PATH = "kokoro-v1.1-zh.onnx"
CONFIG_PATH = "config.json" # 用于获取 style_dim 等参数 (如果需要 onnxsim)

INT8_ONNX_PATH = "kokoro_int8_dynamic_quant.onnx"
FP16_ONNX_PATH = "kokoro_fp16_optimized.onnx"

# 设置日志级别
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)


# --- 1. INT8 动态量化 ---

def quantize_dynamic_onnx(fp32_path, int8_path):
    """
    使用 ONNX Runtime Tools 对 FP32 ONNX 模型进行动态 INT8 量化。
    """
    if not Path(fp32_path).exists():
        logger.error(f"❌ 找不到 FP32 ONNX 文件: {fp32_path}")
        return False
        
    logger.info(f"\n--- 1. 开始 INT8 动态量化到 {int8_path}...")
    try:
        # 动态量化主要针对权重，将它们转换为 Int8。
        quantize_dynamic(
            model_input=fp32_path,
            model_output=int8_path,
            # optimize_model=True,  # 已经删除或注释掉
            per_channel=False,    
            weight_type=QuantType.QInt8, 
        )
        logger.info(f"✅ INT8 动态量化模型生成成功: {int8_path}")
        return True
    except Exception as e:
        logger.error(f"❌ INT8 动态量化失败: {e}")
        return False

# --- 2. FP16 半精度转换 (依赖 onnxsim) ---

def convert_to_fp16_onnx(fp32_path, fp16_path, config_data=None):
    """
    将 ONNX 模型转换为 FP16 格式。
    """
    if onnxsim is None:
        logger.warning("跳过 FP16 转换：缺少 onnxsim 库。")
        return False

    logger.info(f"\n--- 2. 开始 FP16 半精度转换到 {fp16_path}...")
    
    try:
        STYLE_DIM = config_data.get('style_dim', 256) if config_data else 256
        
        # 构造输入数据字典 (用于 onnxsim 的形状推断)
        input_data = {
            # 假设 input_ids 是 LongTensor [1, L]
            "input_ids": torch.LongTensor([[0] * 50]).numpy(), 
            # style 是 FloatTensor [1, D_style]
            "style": torch.randn(1, STYLE_DIM).numpy(),
            # speed 是 IntTensor [1]
            "speed": torch.IntTensor([1]).numpy()
        }
        
        onnx_model = onnx.load(fp32_path)
        
        # 简化并转换为 FP16
        model_fp16, check_ok = onnxsim.simplify(
            onnx_model, 
            input_data=input_data, 
            dynamic_input_shape=True, 
            convert_to_fp16=True
        )

        if not check_ok:
            logger.error("FP16 转换简化未能通过一致性检查。")
            return False

        onnx.save(model_fp16, fp16_path)
        logger.info(f"✅ FP16 优化模型生成成功: {fp16_path}")
        return True

    except Exception as e:
        logger.error(f"❌ FP16 转换失败: {e}")
        return False

# --- 3. 执行主流程 ---

if __name__ == '__main__':
    # 尝试加载配置 (仅用于 FP16 转换中的形状推断)
    config_data = None
    if Path(CONFIG_PATH).exists():
        with open(CONFIG_PATH, 'r', encoding='utf-8') as f:
            config_data = json.load(f)

    # 1. 动态量化到 INT8 (CPU 优化)
    quantize_dynamic_onnx(FP32_ONNX_PATH, INT8_ONNX_PATH)

    # 2. 优化到 FP16 (GPU 优化)
    convert_to_fp16_onnx(FP32_ONNX_PATH, FP16_ONNX_PATH, config_data)
    
    logger.info("\n==============================================")
    logger.info("🎉 ONNX 模型量化流程完成！请检查输出文件。")
    logger.info(f"1. INT8 (动态量化): {INT8_ONNX_PATH}")
    logger.info(f"2. FP16 (半精度): {FP16_ONNX_PATH}")
    logger.info("==============================================")

Details

(kokoro-82m-v1-1-zh) C:\Users\admin\code\other\Kokoro-82M-v1.1-zh>python onnx_quantizer.py
WARNING:root:onnxsim 库未安装，FP16 转换将被跳过。请运行 'pip install onnxsim'
WARNING:root:Please consider to run pre-processing before quantization. Refer to example: https://github.com/microsoft/onnxruntime-inference-examples/blob/main/quantization/image_classification/cpu/ReadMe.md
WARNING:root:Inference failed or unsupported type to quantize for tensor '/text_encoder/Unsqueeze_2_output_0', type is tensor_type {
  elem_type: 9
  shape {
    dim {
      dim_param: "unk__5"
    }
    dim {
      dim_param: "unk__4"
    }
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/text_encoder/Unsqueeze_2_output_0', type is tensor_type {
  elem_type: 9
  shape {
    dim {
      dim_param: "unk__5"
    }
    dim {
      dim_param: "unk__4"
    }
    dim {
      dim_value: 1
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/decoder/generator/Slice_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 2
    }
    dim {
      dim_value: 2
    }
  }
}
.
WARNING:root:Inference failed or unsupported type to quantize for tensor '/decoder/generator/reflection_pad/Slice_output_0', type is tensor_type {
  elem_type: 7
  shape {
    dim {
      dim_value: 3
    }
    dim {
      dim_value: 2
    }
  }
}
.
WARNING:__main__:跳过 FP16 转换：缺少 onnxsim 库。

(kokoro-82m-v1-1-zh) C:\Users\admin\code\other\Kokoro-82M-v1.1-zh>

Uh oh!

Update Kokoro-82M-v1.1-zh #177

Description

Describe the feature

Activity

Metadata

Metadata

Assignees

Labels

Projects

Milestone

Relationships

Development

Issue actions