Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 0 additions & 26 deletions samples/cpp/video_generation/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -74,29 +74,3 @@ install(TARGETS taylorseer_text2video
RUNTIME DESTINATION samples_bin/
COMPONENT samples_bin
EXCLUDE_FROM_ALL)

# create LoRA sample executable
add_executable(lora_text2video lora_text2video.cpp imwrite_video.cpp)

target_include_directories(lora_text2video PRIVATE ${CMAKE_BINARY_DIR} "${CMAKE_CURRENT_SOURCE_DIR}/../image_generation/" "${CMAKE_CURRENT_SOURCE_DIR}/../../../src/cpp/src/")
ov_genai_link_opencv(lora_text2video core imgproc videoio imgcodecs)
target_link_libraries(lora_text2video PRIVATE openvino::genai indicators::indicators)

if(UNIX AND NOT APPLE)
set_target_properties(lora_text2video PROPERTIES
INSTALL_RPATH "$ORIGIN/../lib"
)
elseif(APPLE)
set_target_properties(lora_text2video PROPERTIES
INSTALL_RPATH "@loader_path/../lib"
)
endif()

set_target_properties(lora_text2video PROPERTIES
# Ensure out of box LC_RPATH on macOS with SIP
INSTALL_RPATH_USE_LINK_PATH ON)

install(TARGETS lora_text2video
RUNTIME DESTINATION samples_bin/
COMPONENT samples_bin
EXCLUDE_FROM_ALL)
40 changes: 8 additions & 32 deletions samples/cpp/video_generation/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,18 +24,18 @@ pip install --upgrade-strategy eager -r ../../export-requirements.txt
Then, run the export with Optimum CLI:

```sh
optimum-cli export openvino --model Lightricks/LTX-Video --task text-to-video --weight-format fp32 ltx_video_ov/FP32
optimum-cli export openvino --model Lightricks/LTX-Video --task text-to-video --weight-format int8 ltx_video_ov/INT8
```

> **Note:** `--weight-format fp32` is required for LoRA adapter support. For basic video generation without LoRA, `--weight-format int8` produces a smaller model.

Alternatively, do it in Python code:
Alternatively, do it in Python code. If NNCF is installed, the model will be compressed to INT8 automatically.

```python
from optimum.intel.openvino import OVLTXPipeline

pipeline = OVLTXPipeline.from_pretrained("Lightricks/LTX-Video", export=True, compile=False)
pipeline.save_pretrained("ltx_video_ov/FP32")
output_dir = "ltx_video_ov/INT8"

pipeline = OVLTXPipeline.from_pretrained("Lightricks/LTX-Video", export=True, compile=False, load_in_8bit=True)
pipeline.save_pretrained(output_dir)
```

## Sample Descriptions
Expand Down Expand Up @@ -63,34 +63,10 @@ GPUs usually provide better performance compared to CPUs. Modify the source code

Example:
```bash
./text2video ltx_video_ov/FP32 "A woman with long brown hair and light skin smiles at another woman with long blonde hair"
```

### LoRA Text to Video Sample (`lora_text2video.cpp`)

- **Description:**
Video generation with LoRA adapters using a text-to-video model. This sample demonstrates how to generate videos from text prompts while applying multiple LoRA adapters.

Recommended models: Lightricks/LTX-Video

To download the LoRA adapter used in the example below:
```sh
huggingface-cli download svjack/ltx_video_pixel_early_lora ltx_pixel_pytorch_lora_weights.safetensors
```

- **Main Feature:** Apply LoRA adapters to a text-to-video pipeline for customized generation.

- **Run Command:**
```bash
./lora_text2video model_dir prompt [lora_adapter_path alpha] ...
```

Example:
```bash
./lora_text2video ltx_video_ov/FP32 "In the style of Pixel, the video shifts to a majestic castle under a starry sky." ltx_pixel_pytorch_lora_weights.safetensors 3.0
./text2video ltx_video_ov/INT8 "A woman with long brown hair and light skin smiles at another woman with long blonde hair"
```

The LoRA text-to-video sample will generate two video files, `lora_video.avi` and `baseline_video.avi`, in the current directory.
The sample will generate a video file `genai_video.avi` in the current directory.

Users can modify the source code to experiment with different generation parameters:
- Change width or height of generated video
Expand Down
85 changes: 0 additions & 85 deletions samples/cpp/video_generation/lora_text2video.cpp

This file was deleted.

40 changes: 8 additions & 32 deletions samples/python/video_generation/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -24,18 +24,18 @@ pip install --upgrade-strategy eager -r ../../export-requirements.txt
Then, run the export with Optimum CLI:

```sh
optimum-cli export openvino --model Lightricks/LTX-Video --task text-to-video --weight-format fp32 ltx_video_ov/FP32
optimum-cli export openvino --model Lightricks/LTX-Video --task text-to-video --weight-format int8 ltx_video_ov/INT8
```

> **Note:** For basic video generation without LoRA, `--weight-format int8` produces a smaller model.

Alternatively, do it in Python code:
Alternatively, do it in Python code. If NNCF is installed, the model will be compressed to INT8 automatically.

```python
from optimum.intel.openvino import OVLTXPipeline

pipeline = OVLTXPipeline.from_pretrained("Lightricks/LTX-Video", export=True, compile=False)
pipeline.save_pretrained("ltx_video_ov/FP32")
output_dir = "ltx_video_ov/INT8"

pipeline = OVLTXPipeline.from_pretrained("Lightricks/LTX-Video", export=True, compile=False, load_in_8bit=True)
pipeline.save_pretrained(output_dir)
```

## Sample Descriptions
Expand Down Expand Up @@ -68,34 +68,10 @@ pip install --upgrade-strategy eager -r ../../deployment-requirements.txt

Example:
```bash
python text2video.py ./ltx_video_ov/FP32 "A woman with long brown hair and light skin smiles at another woman with long blonde hair"
```

### LoRA Text to Video Sample (`lora_text2video.py`)

- **Description:**
Video generation with LoRA adapters using a text-to-video model. This sample demonstrates how to generate videos from text prompts while applying a LoRA adapter.

Recommended models: Lightricks/LTX-Video

To download the LoRA adapter used in the example below:
```sh
huggingface-cli download svjack/ltx_video_pixel_early_lora ltx_pixel_pytorch_lora_weights.safetensors
```

- **Main Feature:** Apply a LoRA adapter to a text-to-video pipeline for customized generation.

- **Run Command:**
```bash
python lora_text2video.py model_dir prompt [lora_adapter_path alpha] ...
```

Example:
```bash
python lora_text2video.py ./ltx_video_ov/FP32 "In the style of Pixel, the video shifts to a majestic castle under a starry sky." ltx_pixel_pytorch_lora_weights.safetensors 3.0
python text2video.py ./ltx_video_ov/INT8 "A woman with long brown hair and light skin smiles at another woman with long blonde hair"
```

The sample will generate two video files, `lora_video.avi` and `baseline_video.avi`, in the current directory.
The sample will generate a video file `genai_video.avi` in the current directory.

Users can modify the source code to experiment with different generation parameters:
- Change width or height of generated video
Expand Down
75 changes: 0 additions & 75 deletions samples/python/video_generation/lora_text2video.py

This file was deleted.

2 changes: 1 addition & 1 deletion site/docs/guides/lora-adapters.mdx
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ LoRA adapters enable customization of model outputs for specific tasks, styles,
For more details about LoRA, see [Low-Rank Adaptation (LoRA)](/docs/concepts/lora).
:::

OpenVINO GenAI provides built-in support for LoRA adapters in [text generation](/docs/use-cases/text-generation/), [image generation](/docs/use-cases/image-generation/), [image processing (VLM)](/docs/use-cases/image-processing), and [video generation](/docs/use-cases/video-generation/) pipelines.
OpenVINO GenAI provides built-in support for LoRA adapters in [text generation](/docs/use-cases/text-generation/), [image generation](/docs/use-cases/image-generation/), and [image processing (VLM)](/docs/use-cases/image-processing) pipelines.
This capability allows you to dynamically switch between or combine multiple adapters without recompiling the model.

:::info
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -66,12 +66,6 @@ You can adjust several parameters to control the video generation process, inclu
</TabItemCpp>
</LanguageTabs>

### Working with LoRA Adapters

For video generation models like LTX-Video, LoRA adapters can modify the generation process to produce videos with specific artistic styles, content types, or quality enhancements.

Refer to the [LoRA Adapters](/docs/guides/lora-adapters) for more details on working with LoRA adapters.

:::info Understanding Video Generation Parameters

- `negative_prompt`: Negative prompt for video(s) generation.
Expand Down
2 changes: 0 additions & 2 deletions src/cpp/include/openvino/genai/lora_adapter.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ namespace genai {
class OPENVINO_GENAI_EXPORTS AdapterController;
struct AdapterControllerImpl;
class AdapterImpl;
struct AdapterConfig;

// Immutable LoRA Adapter that carries the adaptation matrices and serves as unique adapter identifier
class OPENVINO_GENAI_EXPORTS Adapter {
Expand All @@ -34,7 +33,6 @@ class OPENVINO_GENAI_EXPORTS Adapter {

friend Adapter flux_adapter_normalization(const Adapter& adapter);
friend Adapter diffusers_adapter_normalization(const Adapter& adapter);
friend std::string detect_lora_prefix(const AdapterConfig& adapters);

Adapter(const std::shared_ptr<AdapterImpl>& pimpl);
public:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
#include <optional>

#include "openvino/genai/image_generation/generation_config.hpp"
#include "openvino/genai/lora_adapter.hpp"


namespace ov::genai {
/**
Expand Down Expand Up @@ -63,9 +63,6 @@ struct VideoGenerationConfig {
* and predicts outputs using Taylor series approximation.
*/
std::optional<TaylorSeerCacheConfig> taylorseer_config;

/// LoRA adapters applied during generation.
std::optional<AdapterConfig> adapters = std::nullopt;
};

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
#include "openvino/runtime/infer_request.hpp"
#include "openvino/runtime/properties.hpp"
#include "openvino/runtime/tensor.hpp"
#include "openvino/genai/lora_adapter.hpp"
#include "openvino/genai/visibility.hpp"

namespace ov::genai {
Expand Down Expand Up @@ -47,8 +46,6 @@ class OPENVINO_GENAI_EXPORTS LTXVideoTransformer3DModel {

void set_hidden_states(const std::string& tensor_name, const ov::Tensor& encoder_hidden_states);

void set_adapters(const std::optional<AdapterConfig>& adapters);

ov::Tensor infer(const ov::Tensor& latent, const ov::Tensor& timestep);

LTXVideoTransformer3DModel& reshape(int64_t batch_size, int64_t num_frames, int64_t height, int64_t width, int64_t tokenizer_model_max_length);
Expand All @@ -61,8 +58,6 @@ class OPENVINO_GENAI_EXPORTS LTXVideoTransformer3DModel {
std::shared_ptr<Inference> m_impl;

Config m_config;
AdapterController m_adapter_controller;
std::string m_lora_prefix;
ov::InferRequest m_request;
std::shared_ptr<ov::Model> m_model;
size_t m_expected_batch_size = 0;
Expand Down
Loading
Loading