�[38;20m2026-05-19 13:58:35 INFO data_driven.py L673: caching done�[0m
0%| | 0/48 [00:00<?, ?it/s]
Quantizing model.layers.0: 0%| | 0/48 [00:00<?, ?it/s]Traceback (most recent call last):
File "/data/jenkins/B60/workspace/Intel-GPU-key-model-test/neural-compressor/examples/pytorch/nlp/huggingface_models/language-modeling/quantization/auto_round/qwen/quantize.py", line 212, in <module>
quant_model(args)
File "/data/jenkins/B60/workspace/Intel-GPU-key-model-test/neural-compressor/examples/pytorch/nlp/huggingface_models/language-modeling/quantization/auto_round/qwen/quantize.py", line 132, in quant_model
inc_model = convert(model)
^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/neural_compressor/common/utils/utility.py", line 321, in inner_wrapper
result = func(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/neural_compressor/torch/quantization/quantize.py", line 283, in convert
q_model = algo_func(
^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/neural_compressor/torch/quantization/algorithm_entry.py", line 632, in autoround_quantize_entry
model = quantizer.execute(model=model, mode=mode, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/neural_compressor/torch/algorithms/base_algorithm.py", line 113, in execute
model = self.convert(model, *args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/neural_compressor/torch/algorithms/autoround/autoround.py", line 227, in convert
_, quantized_model_path = rounder.quantize_and_save(
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/auto_round/compressors/base.py", line 1486, in quantize_and_save
self.quantize()
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/auto_round/compressors/data_driven.py", line 710, in quantize
self._quantize_blocks(
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/auto_round/compressors/data_driven.py", line 545, in _quantize_blocks
self.quantizer.quantize_block(
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/auto_round/algorithms/quantization/sign_round/quantizer.py", line 275, in quantize_block
self._scale_loss_and_backward(scaler, loss)
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/auto_round/algorithms/quantization/sign_round/quantizer.py", line 534, in _scale_loss_and_backward
scale_loss.backward()
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_tensor.py", line 630, in backward
torch.autograd.backward(
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/autograd/__init__.py", line 364, in backward
_engine_run_backward(
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/autograd/graph.py", line 865, in _engine_run_backward
return Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/autograd/function.py", line 317, in apply
return user_fn(self, *args)
^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2338, in backward
return impl_fn()
^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2324, in impl_fn
out = CompiledFunction._backward_impl(ctx, all_args)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/runtime_wrappers.py", line 2465, in _backward_impl
CompiledFunction.compiled_bw = aot_config.bw_compiler(
^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_functorch/_aot_autograd/schemas.py", line 1249, in __call__
return self.compiler_fn(gm, example_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_dynamo/backends/common.py", line 83, in _wrapped_bw_compiler
disable(
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_dynamo/eval_frame.py", line 1181, in _fn
return fn(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_utils_internal.py", line 97, in wrapper_function
return function(*args, **kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2716, in bw_compiler
return compile_fx_backward(
^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 2396, in compile_fx_backward
return inner_compile(
^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 806, in compile_fx_inner
return wrap_compiler_debug(_compile_fx_inner, compiler_name="inductor")(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_dynamo/repro/after_aot.py", line 146, in debug_wrapper
inner_compiled_fn = compiler_fn(gm, example_inputs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 1019, in _compile_fx_inner
raise InductorError(e, currentframe()).with_traceback(
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 1003, in _compile_fx_inner
mb_compiled_graph = fx_codegen_and_compile(
^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 1766, in fx_codegen_and_compile
return scheme.codegen_and_compile(gm, example_inputs, inputs_to_check, graph_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/compile_fx.py", line 1537, in codegen_and_compile
compiled_module = graph.compile_to_module()
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/graph.py", line 2416, in compile_to_module
return self._compile_to_module()
^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/graph.py", line 2426, in _compile_to_module
mod = self._compile_to_module_lines(wrapper_code)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/graph.py", line 2501, in _compile_to_module_lines
mod = PyCodeCache.load_by_key_path(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/codecache.py", line 3674, in load_by_key_path
mod = _reload_python_module(key, path, set_sys_modules=in_toplevel)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/runtime/compile_tasks.py", line 35, in _reload_python_module
exec(code, mod.__dict__, mod.__dict__)
File "/tmp/torchinductor_sdp/5e/c5ewx6knfjqbvl4g7d5kkz7qhiw4gfp5juajb5y7cmmfa5kzhqp6.py", line 2962, in <module>
async_compile.wait(globals())
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/async_compile.py", line 649, in wait
self._wait_futures(scope)
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/async_compile.py", line 669, in _wait_futures
kernel = result.result()
^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/codecache.py", line 4417, in result
return self.result_fn()
^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/async_compile.py", line 438, in get_result
raise e.with_name(kernel_name) from e
torch._inductor.exc.InductorError: SubprocException: An exception occurred in a subprocess:
Name=triton_per_fused__to_copy_abs_add_clamp_div_eq_floor_ge_le_log2_logical_and_mul_neg_ones_like_pow_scalar_tensor_scatter_sgn_sign_sub_sum_unsqueeze_view_where_2
Traceback (most recent call last):
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/compile_worker/subproc_pool.py", line 457, in do_job
result = job()
^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/runtime/compile_tasks.py", line 68, in _worker_compile_triton
kernel.precompile(warm_cache_only=True)
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 432, in precompile
self._precompile_worker()
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 463, in _precompile_worker
compile_results.append(self._precompile_config(c))
^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/torch/_inductor/runtime/triton_heuristics.py", line 808, in _precompile_config
binary = triton.compile(*compile_args, **compile_kwargs)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/triton/compiler/compiler.py", line 347, in compile
next_module = compile_ir(module, metadata)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/triton/backends/intel/compiler.py", line 482, in <lambda>
stages["ttgir"] = lambda src, metadata: self.make_ttgir(src, metadata, options, self.properties)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/home/sdp/miniforge3/envs/jenkins-key-model-test/lib/python3.12/site-packages/triton/backends/intel/compiler.py", line 304, in make_ttgir
pm.run(mod, 'make_ttgir')
RuntimeError: PassManager::run failed
Quantizing model.layers.0: 0%| | 0/48 [00:26<?, ?it/s]
Problem Description
Qwen3-30B-A3B-Instruct-2507 mxfp4 quant failed on XPU
Reproduction Steps
https://inteltf-jenk.sh.intel.com/job/Intel-GPU-key-model-test/582/ multi-cards, failed
https://inteltf-jenk.sh.intel.com/job/Intel-GPU-key-model-test/583/ single card, failed
Environment Information
https://inteltf-jenk.sh.intel.com/job/Intel-GPU-key-model-test/582/ multi-cards, failed
https://inteltf-jenk.sh.intel.com/job/Intel-GPU-key-model-test/583/ single card, failed
Error Logs
Additional Context
No response