[NPUW] gemma-2 patterns added to preserve tail constants matcher #32465
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -43,6 +43,23 @@ | |
|
|
||
| namespace opp = ov::pass::pattern; | ||
|
|
||
| // specific function that match subgraph appeared as result of lpt transformations | ||
| auto match_down_up_convert_subgraph_after_lpt = [](const ov::Output<ov::Node>& input) { | ||
| auto upconvert = opp::wrap_type<ov::op::v0::Convert>({input}, opp::type_matches(ov::element::f32)); | ||
|
|
||
| auto upscale = opp::wrap_type<ov::op::v0::Constant>(opp::rank_equals(0)); | ||
| auto upmul = opp::wrap_type<ov::op::v1::Multiply>({upconvert, upscale}); | ||
|
|
||
| auto downscale = opp::wrap_type<ov::op::v0::Constant>(opp::rank_equals(0)); | ||
| auto downmul = opp::wrap_type<ov::op::v1::Multiply>({upmul, downscale}); | ||
|
|
||
| auto downconvert = | ||
| opp::wrap_type<ov::op::v0::Convert>({downmul}, | ||
| opp::type_matches_any({ov::element::f8e4m3, ov::element::f8e5m2})); | ||
|
|
||
| return downconvert; | ||
| }; | ||
|
|
||
| class RemoveEmptyKVTensors : public ov::pass::MatcherPass { | ||
| public: | ||
| OPENVINO_MATCHER_PASS_RTTI("npuw::LLMCompiledModel::RemoveEmptyKVTensors"); | ||
|
|
@@ -54,7 +71,10 @@ class RemoveEmptyKVTensors : public ov::pass::MatcherPass { | |
|
|
||
| RemoveEmptyKVTensors(Context::Ref ctx) { | ||
| auto param = opp::wrap_type<ov::op::v0::Parameter>(); | ||
| auto param_or = | ||
| std::make_shared<opp::op::Or>(ov::OutputVector{param, match_down_up_convert_subgraph_after_lpt(param)}); | ||
|
|
||
| auto concat = opp::wrap_type<ov::op::v0::Concat>({param_or, opp::any_input()}); | ||
|
|
||
| auto callback = [=](opp::Matcher& m) { | ||
| auto& node_to_output = m.get_pattern_value_map(); | ||
|
|
@@ -63,15 +83,28 @@ class RemoveEmptyKVTensors : public ov::pass::MatcherPass { | |
|
|
||
| ctx.get().old_params.push_back(matched_param); | ||
|
|
||
| // Use concat's first input source node to find ShapeOf users. | ||
| // This works universally for both plain parameter and down_up_convert subgraph cases, | ||
| // because in the subgraph case matched_param->get_users() would return the Convert | ||
| // node (first node of the subgraph), not the ShapeOf. | ||
| auto concat_input0_node = matched_node_concat->input(0).get_source_output().get_node_shared_ptr(); | ||
| auto users = concat_input0_node->get_users(); | ||
|
|
||
| // In subgraph case the parameter itself may also have a ShapeOf user, | ||
| // so check both the concat input node and the parameter. | ||
| if (concat_input0_node != matched_param) { | ||
| auto param_users = matched_param->get_users(); | ||
| users.insert(users.end(), param_users.begin(), param_users.end()); | ||
| } | ||
|
|
||
| // Remove duplicates (concat itself will appear in users) | ||
|
||
| // Find and replace ShapeOf nodes with constants | ||
| for (auto& user : users) { | ||
| if (ov::is_type<ov::op::v3::ShapeOf>(user)) { | ||
| auto cst_node = | ||
| ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, matched_param->get_shape()); | ||
| ov::replace_node(user, cst_node); | ||
| } | ||
| } | ||
|
|
||
| // Redirect second concat input to every node which reads from concat | ||
|
|
@@ -323,22 +356,6 @@ class GroupQueryAttentionDecomposition : public ov::pass::MatcherPass { | |
| class RedirectNewKvToOutput : public ov::pass::MatcherPass { | ||
| public: | ||
| RedirectNewKvToOutput() { | ||
| // example of fp8 inputs to concat | ||
| // input0 : float8e4m3[1,32,1151,96] | ||
| // input1 : float8e4m3[1,32,1,96] | ||
|
|
@@ -348,13 +365,13 @@ class RedirectNewKvToOutput : public ov::pass::MatcherPass { | |
| // TODO: this matcher logic better to cover with unit-tests | ||
| auto input0 = opp::wrap_type<ov::op::v0::Parameter>(); | ||
| auto input0_or = | ||
| std::make_shared<opp::op::Or>(ov::OutputVector{input0, match_down_up_convert_subgraph_after_lpt(input0)}); | ||
|
|
||
| auto input1 = opp::any_input(); | ||
|
|
||
| auto kv_concat = opp::wrap_type<ov::op::v0::Concat>({input0_or, input1}); | ||
| auto result1 = opp::wrap_type<ov::op::v0::Result>(kv_concat); | ||
| auto result2 = opp::wrap_type<ov::op::v0::Result>(match_down_up_convert_subgraph_after_lpt(kv_concat)); | ||
|
|
||
| auto result_or = std::make_shared<opp::op::Or>(ov::OutputVector{result1, result2}); | ||
|
|
||
|
|
@@ -1162,6 +1179,7 @@ struct NPUDesc { | |
| std::string arch; | ||
| int64_t max_tiles = 0; | ||
| bool compiler_dq = false; | ||
| bool compiler_matmul_gate = false; | ||
| int64_t compiler_ver = 0; | ||
| bool support_flash_attention_tile = false; | ||
| }; | ||
|
|
@@ -1199,6 +1217,19 @@ std::optional<NPUDesc> extract_npu_descriptor(const std::shared_ptr<const ov::IP | |
| ov::AnyMap{{ov::intel_npu::compiler_type.name(), target_compiler_type}}) | ||
| .as<int64_t>(); | ||
| } | ||
| LOG_INFO("Compiler version: " << ONEAPI_VERSION_MAJOR(desc.compiler_ver) << "." | ||
| << ONEAPI_VERSION_MINOR(desc.compiler_ver)); | ||
|
|
||
| constexpr std::string_view compiler_gate_support_msg = | ||
| "Compiler: accurate gated matmul (MatMul -> Divide -> Tanh -> Multiply -> Result) : "; | ||
|
|
||
| if (desc.compiler_ver >= ONEAPI_MAKE_VERSION(7, 28)) { | ||
| // accuracy for gated matmul fixed at 7.28 | ||
| desc.compiler_matmul_gate = true; | ||
| LOG_INFO(compiler_gate_support_msg << "supported"); | ||
| } else { | ||
| LOG_WARN(compiler_gate_support_msg << "unsupported"); | ||
| } | ||
|
|
||
| if (desc.arch == "5010" && desc.compiler_ver >= ONEAPI_MAKE_VERSION(7, 29)) { | ||
| // Flash attention tile is supported starting from compiler version 7.29 on NPU5010 | ||
|
|
@@ -1247,6 +1278,13 @@ ov::AnyMap get_baseline_common_config(const std::optional<NPUDesc>& npudesc) { | |
| config.erase("NPUW_DCOFF_TYPE"); | ||
| config.erase("NPUW_DCOFF_SCALE"); | ||
| } | ||
|
|
||
| // default value is ON | ||
| // for compiler versions >= 7.28 value is ON | ||
| // for other compiler versions value is OFF | ||
| if (npudesc.has_value()) { | ||
| config.emplace("NPUW_MM_GATED", (npudesc->compiler_matmul_gate ? "YES" : "NO")); | ||
| } | ||
| return config; | ||
| } | ||
|
|
||
|
|
@@ -1877,7 +1915,6 @@ ov::npuw::LLMCompiledModel::LLMCompiledModel(const std::shared_ptr<ov::Model>& m | |
|
|
||
| if (!m_is_embedding) { | ||
| if (!m_use_chunk_prefill) { | ||
| NPUW_ASSERT(remove_empty_kv_inputs(prefill_model)); | ||
| } else { | ||
| LOG_DEBUG("Don't remove input key/values from prefill model."); | ||
|
|
||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Great catch!!