Support Gemma4 model (openvinotoolkit#3644)

<!-- Keep your pull requests (PRs) as atomic as possible. That increases
the likelihood that an individual PR won't be stuck because of adjacent
problems, merge conflicts, or code review.
Your merged PR is going to appear in the automatically generated release
notes on GitHub. So the clearer the title the better. -->
## Description
<!-- Please include a summary of the change. Also include relevant
motivation and context. -->

Depends on: huggingface/optimum-intel#1688
optimum-intel PR depends on transformers v5 (**update**: transformers v5
support merged to optimum-intel).

### WWB Accuracy:

genai vs optimum-intel: 0.9682357
genai vs transformers: 0.94821364
optimum-intel vs transformers: 0.9387633

Fixes: openvinotoolkit#3653

Current implementation support image text inputs only. Ticket for video
support implementation: 185850

## Checklist:
- [x] This PR follows [GenAI Contributing
guidelines](https://github.com/openvinotoolkit/openvino.genai?tab=contributing-ov-file#contributing).
<!-- Always follow them. If there are deviations, explain what and why.
-->
- [x] Tests have been updated or added to cover the new code. <!--
Specify exactly which tests were added or updated. If the change isn't
maintenance related, update the tests at
https://github.com/openvinotoolkit/openvino.genai/tree/master/tests or
explain in the description why the tests don't need an update. -->
- [x] This PR fully addresses the ticket. <!--- If not, explain clearly
what is covered and what is not. If follow-up pull requests are needed,
specify in the description. -->
- [x] I have made corresponding changes to the documentation. <!-- Run
github.com/\<username>/openvino.genai/actions/workflows/deploy_gh_pages.yml
on your fork with your branch as a parameter to deploy a test version
with the updated content. Replace this comment with the link to the
built docs. If the documentation is updated in a separate PR, clearly
specify it. -->

Author: as-suvorov

.github/workflows/linux.yml

@@ -652,6 +652,14 @@ jobs:
               python -m pytest -s -v tests/python_tests/test_vlm_pipeline.py --override-ini cache_dir=/mount/caches/pytest/ -k "qwen3-vl"
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).visual_language.test }}
             timeout: 60
+          - name: 'VLM (gemma4)'
+            cmd: |
+              python -m pip install --no-deps git+https://github.com/huggingface/optimum-intel.git@ff99d6e13774841bdd17ac0d4c8bd2d181cf7c27 # PR 1688
+              python -m pip install transformers==5.5.0
+              pip show transformers optimum-intel openvino_tokenizers openvino_genai
+              python -m pytest -s -v tests/python_tests/test_vlm_pipeline.py --override-ini cache_dir=/mount/caches/pytest/ -k "gemma4"
+            run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).visual_language.test }}
+            timeout: 60
     defaults:
       run:
         shell: bash

.github/workflows/manylinux_2_28.yml

@@ -575,6 +575,15 @@ jobs:
               python -m pytest -s -v tests/python_tests/test_vlm_pipeline.py --override-ini cache_dir=/mount/caches/pytest/ -k "qwen3-vl"
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).visual_language.test }}
             timeout: 60
+          - name: 'VLM (gemma4)'
+            cmd: |
+              python -m pip install --no-deps git+https://github.com/huggingface/optimum-intel.git@ff99d6e13774841bdd17ac0d4c8bd2d181cf7c27 # PR 1688
+              python -m pip install transformers==5.5.0
+              pip show transformers optimum-intel openvino_tokenizers openvino_genai
+              python -m pytest -s -v tests/python_tests/test_vlm_pipeline.py --override-ini cache_dir=/mount/caches/pytest/ -k "gemma4"
+            run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).visual_language.test }}
+            timeout: 60
+
     defaults:
       run:
         shell: bash

.github/workflows/windows.yml

@@ -740,6 +740,14 @@ jobs:
               python -m pytest -s -v tests/python_tests/test_vlm_pipeline.py --override-ini cache_dir=/mount/caches/pytest/ -k "qwen3-vl"
             run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).visual_language.test }}
             timeout: 60
+          - name: 'VLM (gemma4)'
+            cmd: |
+              python -m pip install --no-deps git+https://github.com/huggingface/optimum-intel.git@ff99d6e13774841bdd17ac0d4c8bd2d181cf7c27 # PR 1688
+              python -m pip install transformers==5.5.0
+              pip show transformers optimum-intel openvino_tokenizers openvino_genai
+              python -m pytest -s -v tests/python_tests/test_vlm_pipeline.py --override-ini cache_dir=/mount/caches/pytest/ -k "gemma4"
+            run_condition: ${{ fromJSON(needs.smart_ci.outputs.affected_components).visual_language.test }}
+            timeout: 60
     defaults:
       run:
         shell: pwsh

site/docs/supported-models/_components/vlm-models-table/models.ts

@@ -193,4 +193,16 @@ export const VLM_MODELS: VLMModelType[] = [
       },
     ],
   },
+  {
+    architecture: 'Gemma4ForConditionalGeneration',
+    models: [
+      {
+        name: 'gemma4',
+        links: [
+          'https://huggingface.co/google/gemma-4-E2B-it',
+          'https://huggingface.co/google/gemma-4-E4B-it',
+        ],
+      },
+    ],
+  },
 ];

site/docs/supported-models/index.mdx

@@ -87,6 +87,12 @@ Apply https://huggingface.co/microsoft/Phi-4-multimodal-instruct/discussions/78/
 2. Visual history is not preserved across rounds, so multi-turn interactions have limited visual context.
 3. If the number of input frames is not divisible by `mm_local_num_frames` (as defined in `config.json`), additional frames will be automatically padded by duplicating the last frame. For example, if there are 10 frames and `mm_local_num_frames = 4`, it will be padded to 12 frames.
 
+#### Gemma4 {#gemma4-notes}
+
+Gemma4 implementation supports text and image inputs only. Video input is not supported at the moment.
+
+The model requires `transformers==5.5.0` for the export with `optimum-cli`.
+
 #### Qwen3-VL {#qwen3_vl-notes}
 
 The model requires `transformers>=4.57` for the export with `optimum-cli`.

src/cpp/src/lm_encoding.cpp

@@ -86,7 +86,8 @@ ov::genai::utils::GenerationFinishInfo get_lm_encoded_results(
     std::optional<int64_t> rope_delta,
     const size_t max_kv_cache_size,
     const bool use_intermediate_remote_tensor,
-    const std::unordered_map<std::string, ov::Tensor>& lm_extra_inputs
+    const std::unordered_map<std::string, ov::Tensor>& lm_extra_inputs,
+    std::function<ov::Tensor(const ov::Tensor& new_input_ids)> per_layer_embeddings_callback
 ) {
     std::vector<GenerationHandle> generations;
     for (SequenceGroup::Ptr sequence_group : sequence_groups) {
@@ -261,6 +262,8 @@ ov::genai::utils::GenerationFinishInfo get_lm_encoded_results(
                     ov::Tensor new_visual_pos_masks{tensor.get_element_type(), {batch_size, 1}};
                     std::fill_n(new_visual_pos_masks.data<bool>(), new_visual_pos_masks.get_size(), false);
                     m_llm.set_tensor(name, new_visual_pos_masks);
+                } else if (name == "per_layer_inputs" && per_layer_embeddings_callback) {
+                    m_llm.set_tensor(name, per_layer_embeddings_callback(new_input_ids));
                 }
             }
         } else {

src/cpp/src/lm_encoding.hpp

@@ -3,15 +3,16 @@
 
 #pragma once
 
+#include <functional>
 #include <optional>
+
 #include "openvino/genai/llm_pipeline.hpp"
-#include "visual_language/embedding_model.hpp"
 #include "sampling/sampler.hpp"
+#include "visual_language/embedding_model.hpp"
 
 namespace ov {
 namespace genai {
 
-
 ov::genai::utils::GenerationFinishInfo get_lm_encoded_results(
     ov::InferRequest& m_llm,
     const ov::Tensor& input_ids,
@@ -26,13 +27,12 @@ ov::genai::utils::GenerationFinishInfo get_lm_encoded_results(
     std::optional<int64_t> rope_delta = std::nullopt,
     const size_t max_kv_cache_size = std::numeric_limits<size_t>::max(),
     const bool use_intermediate_remote_tensor = true,
-    const std::unordered_map<std::string, ov::Tensor>& lm_extra_inputs = {});
-
+    const std::unordered_map<std::string, ov::Tensor>& lm_extra_inputs = {},
+    std::function<ov::Tensor(const ov::Tensor& new_input_ids)> per_layer_embeddings_callback = nullptr);
 
 void align_cache_and_history(const ov::Tensor& new_chat_tokens, utils::CacheState& cache_state);
 
-
 TokenizedInputs get_chat_encoded_input(const ov::Tensor& new_chat_tokens, utils::CacheState& cache_state);
 
-}
-}
+}  // namespace genai
+}  // namespace ov

src/cpp/src/utils.cpp

@@ -239,6 +239,8 @@ ProcessorConfig from_any_map(
     read_anymap_param(config_map, "max_slice_nums", extracted_config.max_slice_nums);
     read_anymap_param(config_map, "norm_mean", extracted_config.norm_mean);
     read_anymap_param(config_map, "norm_std", extracted_config.norm_std);
+    read_anymap_param(config_map, "pooling_kernel_size", extracted_config.pooling_kernel_size);
+    read_anymap_param(config_map, "max_soft_tokens", extracted_config.max_soft_tokens);
     return extracted_config;
 }
 

src/cpp/src/visual_language/gemma3/classes.cpp

@@ -106,6 +106,8 @@ NormalizedPrompt InputsEmbedderGemma3::normalize_prompt(const std::string& promp
         }
         expanded_tag += end_of_image + "\n\n";
 
+        // fixme: there seems to be an issue with how image_token is replaced. unified_prompt.find needs search_offset.
+        // refer to gemma4 implementation.
         unified_prompt.replace(unified_prompt.find(start_of_image), start_of_image.length(), expanded_tag);
     }
     return {std::move(unified_prompt), std::move(images_sequence), {}};