From a3f9705b35ccb053a2dbf7884aa8a1255f5c0581 Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Tue, 25 Aug 2026 14:28:28 -0700 Subject: [PATCH 01/18] Transplant LingBot-VA model integration Signed-off-by: Jonathan McCaffrey --- integrations/lingbot_va/README.md | 90 ++++ .../lingbot_va/lingbot_va/__init__.py | 18 + .../lingbot_va/lingbot_va/_loaders.py | 183 +++++++ integrations/lingbot_va/lingbot_va/action.py | 110 ++++ integrations/lingbot_va/lingbot_va/config.py | 110 ++++ .../lingbot_va/lingbot_va/constants.py | 99 ++++ .../lingbot_va/lingbot_va/pipeline.py | 254 +++++++++ integrations/lingbot_va/lingbot_va/runner.py | 503 ++++++++++++++++++ .../lingbot_va/lingbot_va/scheduler.py | 130 +++++ .../lingbot_va/transformer/__init__.py | 280 ++++++++++ .../lingbot_va/transformer/checkpoint.py | 94 ++++ .../lingbot_va/transformer/impl/__init__.py | 15 + .../lingbot_va/transformer/impl/kvcache.py | 157 ++++++ .../lingbot_va/transformer/impl/modules.py | 219 ++++++++ .../lingbot_va/transformer/impl/network.py | 422 +++++++++++++++ integrations/lingbot_va/lingbot_va/utils.py | 105 ++++ integrations/lingbot_va/pyproject.toml | 50 ++ 17 files changed, 2839 insertions(+) create mode 100644 integrations/lingbot_va/README.md create mode 100644 integrations/lingbot_va/lingbot_va/__init__.py create mode 100644 integrations/lingbot_va/lingbot_va/_loaders.py create mode 100644 integrations/lingbot_va/lingbot_va/action.py create mode 100644 integrations/lingbot_va/lingbot_va/config.py create mode 100644 integrations/lingbot_va/lingbot_va/constants.py create mode 100644 integrations/lingbot_va/lingbot_va/pipeline.py create mode 100644 integrations/lingbot_va/lingbot_va/runner.py create mode 100644 integrations/lingbot_va/lingbot_va/scheduler.py create mode 100644 integrations/lingbot_va/lingbot_va/transformer/__init__.py create mode 100644 integrations/lingbot_va/lingbot_va/transformer/checkpoint.py create mode 100644 integrations/lingbot_va/lingbot_va/transformer/impl/__init__.py create mode 100644 integrations/lingbot_va/lingbot_va/transformer/impl/kvcache.py create mode 100644 integrations/lingbot_va/lingbot_va/transformer/impl/modules.py create mode 100644 integrations/lingbot_va/lingbot_va/transformer/impl/network.py create mode 100644 integrations/lingbot_va/lingbot_va/utils.py create mode 100644 integrations/lingbot_va/pyproject.toml diff --git a/integrations/lingbot_va/README.md b/integrations/lingbot_va/README.md new file mode 100644 index 000000000..d99e57805 --- /dev/null +++ b/integrations/lingbot_va/README.md @@ -0,0 +1,90 @@ + + +# flashdreams-lingbot-va + +LingBot-VA Image-to-Action-Video (I2AV) integration, packaged as +a [`flashdreams`](../..) plugin. + +This plugin adapts [LingBot-VA](https://github.com/robbyant/lingbot-va) into the +standard flashdreams runner/pipeline interface, achieving **2.3× speedup** over +the original repository implementation. Note that the upstream repo wraps models +with FSDP; compared to the original implementation with FSDP removed, the +speedup is **1.48×**. + + +## Install + +This plugin is a workspace member in the repo-root `pyproject.toml`, which +is included automatically when you set up the flashdreams environment: + +```bash +uv sync +``` + +No separate install step is needed. + +## Run + +```bash +uv run flashdreams-run lingbot-va-robotwin-i2av \ + --input-image-dir assets/example_data/lingbot-va/robotwin \ + --output-dir outputs/lingbot_va/robotwin_i2av \ + --checkpoint-root /path/to/lingbot-va-posttrain-robotwin \ + --num-chunks 10 \ + --benchmark True +``` + +### CLI arguments + +| flag | type | default | description | +| --- | --- | --- | --- | +| `--checkpoint-root` | str | `robbyant/lingbot-va-posttrain-robotwin` | Local path or HuggingFace repo ID for model weights. Must contain `transformer/`, `vae/`, `text_encoder/`, `tokenizer/` subdirs. | +| `--input-image-dir` | path | `assets/example_data/lingbot-va/robotwin` | Directory containing three observation camera PNGs (see below). | +| `--output-dir` | path | `outputs/lingbot_va/robotwin_i2av` | Where to write `demo.mp4`, `actions.npy`, `latents.pt`, and timing JSON. | +| `--prompt` | str | `"Grab the medium-sized white mug, rotate it, place it on the table, and hook it onto the smooth dark gray rack."` | Text prompt describing the manipulation task. Can also be a path to a `.txt` file. | +| `--num-chunks` | int | `10` | Number of autoregressive chunks to generate. Each chunk produces `frame_chunk_size` (2) video frames and `action_per_frame × frame_chunk_size` (32) action steps. | +| `--seed` | int | `42` | Random seed for diffusion sampling. | +| `--benchmark` | bool | `False` | Print per-chunk and total pipeline timing, and save `timing_flashdreams.json`. | +| `--compile-network` | bool | `True` | Apply `torch.compile` to the DiT for faster inference. Set `False` for debugging. | +| `--enable-offload` | bool | `False` | Offload VAE/text-encoder to CPU after use to reduce VRAM (slower). | +| `--save-video` | bool | `True` | Decode latents and save `demo.mp4`. | +| `--save-actions` | bool | `True` | Save predicted actions to `actions.npy`. | +| `--num-inference-steps` | int | `25` | Diffusion steps for video denoising. | +| `--action-num-inference-steps` | int | `50` | Diffusion steps for action denoising. | +| `--guidance-scale` | float | `5.0` | Classifier-free guidance scale for video. | +| `--action-guidance-scale` | float | `1.0` | Classifier-free guidance scale for actions. | +| `--snr-shift` | float | `5.0` | Flow-match sigma shift for video scheduler. | +| `--action-snr-shift` | float | `1.0` | Flow-match sigma shift for action scheduler. | + +### Input images + +The runner expects these files under `--input-image-dir`: + +- `observation.images.cam_high.png` +- `observation.images.cam_left_wrist.png` +- `observation.images.cam_right_wrist.png` + +### Outputs + +| file | description | +| --- | --- | +| `demo.mp4` | Decoded video (all chunks concatenated). | +| `actions.npy` | Predicted actions array, shape `(num_chunks × action_per_frame × frame_chunk_size, action_dim)`. | +| `latents.pt` | Raw latent tensors before VAE decode. | +| `timing_flashdreams.json` | Per-chunk and total timing (only when `--benchmark True`). | diff --git a/integrations/lingbot_va/lingbot_va/__init__.py b/integrations/lingbot_va/lingbot_va/__init__.py new file mode 100644 index 000000000..fc627fbf6 --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/__init__.py @@ -0,0 +1,18 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""LingBot-VA Robotwin I2AV integration for FlashDreams.""" + +__all__: list[str] = [] diff --git a/integrations/lingbot_va/lingbot_va/_loaders.py b/integrations/lingbot_va/lingbot_va/_loaders.py new file mode 100644 index 000000000..d0fa8cef9 --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/_loaders.py @@ -0,0 +1,183 @@ +# SPDX-FileCopyrightText: Copyright 2024-2025 The Robbyant Team Authors +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Model loading utilities adapted from upstream LingBot-VA (wan_va/modules/utils.py +and wan_va/utils/scheduler.py).""" + +from __future__ import annotations + +import math + +import torch +from diffusers import AutoencoderKLWan +from transformers import T5TokenizerFast, UMT5EncoderModel + + +def load_vae(vae_path: str, torch_dtype: torch.dtype, torch_device): + vae = AutoencoderKLWan.from_pretrained(vae_path, torch_dtype=torch_dtype) + return vae.to(torch_device) + + +def load_text_encoder(text_encoder_path: str, torch_dtype: torch.dtype, torch_device): + text_encoder = UMT5EncoderModel.from_pretrained( + text_encoder_path, torch_dtype=torch_dtype + ) + return text_encoder.to(torch_device) + + +def load_tokenizer(tokenizer_path: str): + return T5TokenizerFast.from_pretrained(tokenizer_path) + + +def patchify(x: torch.Tensor, patch_size: int | None) -> torch.Tensor: + if patch_size is None or patch_size == 1: + return x + batch_size, channels, frames, height, width = x.shape + x = x.view( + batch_size, channels, frames, + height // patch_size, patch_size, + width // patch_size, patch_size, + ) + x = x.permute(0, 1, 6, 4, 2, 3, 5).contiguous() + x = x.view( + batch_size, channels * patch_size * patch_size, frames, + height // patch_size, width // patch_size, + ) + return x + + +class WanVAEStreamingWrapper: + + def __init__(self, vae_model): + self.vae = vae_model + self.encoder = vae_model.encoder + self.quant_conv = vae_model.quant_conv + + if hasattr(self.vae, "_cached_conv_counts"): + self.enc_conv_num = self.vae._cached_conv_counts["encoder"] + else: + count = 0 + for m in self.encoder.modules(): + if m.__class__.__name__ == "WanCausalConv3d": + count += 1 + self.enc_conv_num = count + + self.clear_cache() + + def clear_cache(self): + self.feat_cache = [None] * self.enc_conv_num + + def encode_chunk(self, x_chunk: torch.Tensor) -> torch.Tensor: + if ( + hasattr(self.vae.config, "patch_size") + and self.vae.config.patch_size is not None + ): + x_chunk = patchify(x_chunk, self.vae.config.patch_size) + feat_idx = [0] + out = self.encoder(x_chunk, feat_cache=self.feat_cache, feat_idx=feat_idx) + enc = self.quant_conv(out) + return enc + + +# --------------------------------------------------------------------------- +# Upstream FlowMatchScheduler (wan_va/utils/scheduler.py) +# --------------------------------------------------------------------------- + + +class FlowMatchScheduler: + + def __init__( + self, + num_inference_steps=100, + num_train_timesteps=1000, + shift=3.0, + sigma_max=1.0, + sigma_min=0.003 / 1.002, + inverse_timesteps=False, + extra_one_step=False, + reverse_sigmas=False, + exponential_shift=False, + exponential_shift_mu=None, + shift_terminal=None, + ): + self.num_train_timesteps = num_train_timesteps + self.shift = shift + self.sigma_max = sigma_max + self.sigma_min = sigma_min + self.inverse_timesteps = inverse_timesteps + self.extra_one_step = extra_one_step + self.reverse_sigmas = reverse_sigmas + self.exponential_shift = exponential_shift + self.exponential_shift_mu = exponential_shift_mu + self.shift_terminal = shift_terminal + self.set_timesteps(num_inference_steps) + + def set_timesteps(self, num_inference_steps=100, denoising_strength=1.0, + training=False, shift=None, dynamic_shift_len=None): + if shift is not None: + self.shift = shift + sigma_start = self.sigma_min + (self.sigma_max - self.sigma_min) * denoising_strength + if self.extra_one_step: + self.sigmas = torch.linspace(sigma_start, self.sigma_min, + num_inference_steps + 1)[:-1] + else: + self.sigmas = torch.linspace(sigma_start, self.sigma_min, + num_inference_steps) + if self.inverse_timesteps: + self.sigmas = torch.flip(self.sigmas, dims=[0]) + if self.exponential_shift: + mu = (self.calculate_shift(dynamic_shift_len) + if dynamic_shift_len is not None else self.exponential_shift_mu) + self.sigmas = math.exp(mu) / (math.exp(mu) + (1 / self.sigmas - 1)) + else: + self.sigmas = self.shift * self.sigmas / (1 + (self.shift - 1) * self.sigmas) + if self.shift_terminal is not None: + one_minus_z = 1 - self.sigmas + scale_factor = one_minus_z[-1] / (1 - self.shift_terminal) + self.sigmas = 1 - (one_minus_z / scale_factor) + if self.reverse_sigmas: + self.sigmas = 1 - self.sigmas + self.timesteps = self.sigmas * self.num_train_timesteps + + def step(self, model_output, timestep, sample, to_final=False, **kwargs): + if isinstance(timestep, torch.Tensor): + timestep = timestep.cpu() + timestep_id = torch.argmin((self.timesteps - timestep).abs()) + sigma = self.sigmas[timestep_id] + if to_final or timestep_id + 1 >= len(self.timesteps): + sigma_ = 1 if (self.inverse_timesteps or self.reverse_sigmas) else 0 + else: + sigma_ = self.sigmas[timestep_id + 1] + prev_sample = sample + model_output * (sigma_ - sigma) + return prev_sample + + def add_noise(self, original_samples, noise, timestep, t_dim=2): + if isinstance(timestep, torch.Tensor): + timestep = timestep.cpu() + timestep = timestep[None] + timestep_id = torch.argmin((self.timesteps[:, None] - timestep).abs(), dim=0) + shape = [1] * noise.ndim + shape[t_dim] = timestep_id.shape[0] + sigma = self.sigmas[timestep_id].to(original_samples).view(shape) + sample = (1 - sigma) * original_samples + sigma * noise + return sample + + def calculate_shift(self, image_seq_len, base_seq_len=256, + max_seq_len=8192, base_shift=0.5, max_shift=0.9): + m = (max_shift - base_shift) / (max_seq_len - base_seq_len) + b = base_shift - m * base_seq_len + mu = image_seq_len * m + b + return mu diff --git a/integrations/lingbot_va/lingbot_va/action.py b/integrations/lingbot_va/lingbot_va/action.py new file mode 100644 index 000000000..a2782cf40 --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/action.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Robotwin action normalization and channel selection for LingBot-VA.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import torch +from torch import Tensor + +from lingbot_va.constants import ( + ROBOTWIN_ACTION_DIM, + ROBOTWIN_ACTION_Q01, + ROBOTWIN_ACTION_Q99, + ROBOTWIN_USED_ACTION_CHANNEL_IDS, +) + + +@dataclass(frozen=True) +class LingbotVAActionProcessorConfig: + """Pure-Python config for Robotwin action preprocessing/postprocessing.""" + + action_dim: int = ROBOTWIN_ACTION_DIM + used_action_channel_ids: tuple[int, ...] = ROBOTWIN_USED_ACTION_CHANNEL_IDS + q01: tuple[float, ...] = ROBOTWIN_ACTION_Q01 + q99: tuple[float, ...] = ROBOTWIN_ACTION_Q99 + norm_method: str = "quantiles" + + +@dataclass +class LingbotVAActionProcessor: + """Normalize, mask, and denormalize Robotwin actions exactly like upstream.""" + + config: LingbotVAActionProcessorConfig = field( + default_factory=LingbotVAActionProcessorConfig + ) + + @property + def inverse_used_action_channel_ids(self) -> tuple[int, ...]: + inverse = [len(self.config.used_action_channel_ids)] * self.config.action_dim + for i, channel in enumerate(self.config.used_action_channel_ids): + inverse[channel] = i + return tuple(inverse) + + def action_mask(self, *, device: torch.device | None = None) -> Tensor: + mask = torch.zeros([self.config.action_dim], dtype=torch.bool, device=device) + mask[list(self.config.used_action_channel_ids)] = True + return mask + + def q01_tensor(self, *, device: torch.device | None = None) -> Tensor: + return torch.tensor(self.config.q01, dtype=torch.float32, device=device).reshape(-1, 1, 1) + + def q99_tensor(self, *, device: torch.device | None = None) -> Tensor: + return torch.tensor(self.config.q99, dtype=torch.float32, device=device).reshape(-1, 1, 1) + + def preprocess(self, action: Tensor) -> Tensor: + """Normalize a raw action tensor of shape ``[C_used_or_full, F, H]``. + + Mirrors upstream: pad one action channel, expand selected channels + to the model's full 30-channel order via ``inverse_used_action_channel_ids``, + quantile-normalize to roughly ``[-1, 1]``, then return + ``[1, action_dim, F, H, 1]``. + """ + assert action.ndim == 3, f"expected [C, F, H], got {tuple(action.shape)}" + padded = torch.nn.functional.pad(action, [0, 0, 0, 0, 0, 1], mode="constant", value=0) + expanded = padded[list(self.inverse_used_action_channel_ids)] + if self.config.norm_method != "quantiles": + raise NotImplementedError(self.config.norm_method) + q01 = self.q01_tensor(device=expanded.device) + q99 = self.q99_tensor(device=expanded.device) + expanded = (expanded - q01) / (q99 - q01 + 1e-6) * 2.0 - 1.0 + return expanded.unsqueeze(0).unsqueeze(-1) + + def zero_unused_channels(self, action: Tensor) -> Tensor: + """Return a copy with all non-Robotwin-used channels set to zero.""" + masked = action.clone() + masked[:, ~self.action_mask(device=masked.device)] = 0 + return masked + + def postprocess(self, action: Tensor) -> Tensor: + """Denormalize model action output and return selected Robotwin channels. + + Args: + action: Tensor with shape ``[B, 30, F, H, 1]``. + + Returns: + Tensor with shape ``[len(used_action_channel_ids), F, H]``. + """ + assert action.ndim == 5, f"expected [B, C, F, H, W], got {tuple(action.shape)}" + action_cpu = action[0, ..., 0].detach().cpu() + if self.config.norm_method != "quantiles": + raise NotImplementedError(self.config.norm_method) + q01 = self.q01_tensor() + q99 = self.q99_tensor() + denorm = (action_cpu + 1.0) / 2.0 * (q99 - q01 + 1e-6) + q01 + return denorm[list(self.config.used_action_channel_ids)] diff --git a/integrations/lingbot_va/lingbot_va/config.py b/integrations/lingbot_va/lingbot_va/config.py new file mode 100644 index 000000000..f3324162d --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/config.py @@ -0,0 +1,110 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Static configs for the LingBot-VA Robotwin I2AV integration.""" + +from __future__ import annotations + +from flashdreams.infra.diffusion.model import DiffusionModelConfig +from flashdreams.infra.runner import RunnerConfig +from lingbot_va.constants import ( + DEFAULT_CHECKPOINT_ROOT, + DEFAULT_OUTPUT_DIR, + ROBOTWIN_ACTION_INFERENCE_STEPS, + ROBOTWIN_ACTION_PER_FRAME, + ROBOTWIN_ACTION_SNR_SHIFT, + ROBOTWIN_ACTION_TOKEN_PER_CHUNK, + ROBOTWIN_ATTENTION_WINDOW, + ROBOTWIN_FRAME_CHUNK_SIZE, + ROBOTWIN_HEIGHT, + ROBOTWIN_LATENT_CHANNELS, + ROBOTWIN_LATENT_HEIGHT, + ROBOTWIN_LATENT_TOKEN_PER_CHUNK, + ROBOTWIN_LATENT_WIDTH, + ROBOTWIN_SNR_SHIFT, + ROBOTWIN_VIDEO_INFERENCE_STEPS, + ROBOTWIN_WIDTH, + RUNNER_NAME_ROBOTWIN_I2AV, +) +from lingbot_va.pipeline import LingbotVAInferencePipelineConfig +from lingbot_va.runner import LingbotVARobotwinRunnerConfig +from lingbot_va.scheduler import LingbotVAFlowMatchSchedulerConfig +from lingbot_va.transformer import LingbotVATransformerConfig + +PIPELINE_LINGBOT_VA_ROBOTWIN_I2AV = LingbotVAInferencePipelineConfig( + name=RUNNER_NAME_ROBOTWIN_I2AV, + enable_sync_and_profile=True, + encoder=None, + decoder=None, + checkpoint_root=DEFAULT_CHECKPOINT_ROOT, + robotwin_height=ROBOTWIN_HEIGHT, + robotwin_width=ROBOTWIN_WIDTH, + frame_chunk_size=ROBOTWIN_FRAME_CHUNK_SIZE, + action_per_frame=ROBOTWIN_ACTION_PER_FRAME, + attn_window=ROBOTWIN_ATTENTION_WINDOW, + latent_height=ROBOTWIN_LATENT_HEIGHT, + latent_width=ROBOTWIN_LATENT_WIDTH, + latent_channels=ROBOTWIN_LATENT_CHANNELS, + latent_token_per_chunk=ROBOTWIN_LATENT_TOKEN_PER_CHUNK, + action_token_per_chunk=ROBOTWIN_ACTION_TOKEN_PER_CHUNK, + diffusion_model=DiffusionModelConfig( + seed=42, + transformer=LingbotVATransformerConfig( + checkpoint_root=DEFAULT_CHECKPOINT_ROOT, + guidance_scale=5.0, + action_guidance_scale=1.0, + latent_height=ROBOTWIN_LATENT_HEIGHT, + latent_width=ROBOTWIN_LATENT_WIDTH, + frame_chunk_size=ROBOTWIN_FRAME_CHUNK_SIZE, + action_per_frame=ROBOTWIN_ACTION_PER_FRAME, + attn_window=ROBOTWIN_ATTENTION_WINDOW, + ), + scheduler=LingbotVAFlowMatchSchedulerConfig( + num_inference_steps=ROBOTWIN_VIDEO_INFERENCE_STEPS, + shift=ROBOTWIN_SNR_SHIFT, + ), + ), + action_scheduler=LingbotVAFlowMatchSchedulerConfig( + num_inference_steps=ROBOTWIN_ACTION_INFERENCE_STEPS, + shift=ROBOTWIN_ACTION_SNR_SHIFT, + ), +) +"""Robotwin I2AV pipeline config shell. + +This config is intentionally no-instantiate safe; GPU inference remains gated +until the native LingBot-VA DiT/VAE path is ported. +""" + +RUNNER_LINGBOT_VA_ROBOTWIN_I2AV = LingbotVARobotwinRunnerConfig( + runner_name=PIPELINE_LINGBOT_VA_ROBOTWIN_I2AV.name, + description=( + "LingBot-VA Robotwin I2AV inference scaffold " + "(three-camera Robotwin config; native DiT port pending)." + ), + pipeline=PIPELINE_LINGBOT_VA_ROBOTWIN_I2AV, + output_dir=DEFAULT_OUTPUT_DIR, + checkpoint_root=DEFAULT_CHECKPOINT_ROOT, + num_inference_steps=ROBOTWIN_VIDEO_INFERENCE_STEPS, + action_num_inference_steps=ROBOTWIN_ACTION_INFERENCE_STEPS, + snr_shift=ROBOTWIN_SNR_SHIFT, + action_snr_shift=ROBOTWIN_ACTION_SNR_SHIFT, +) + +PIPELINE_CONFIGS: dict[str, LingbotVAInferencePipelineConfig] = { + PIPELINE_LINGBOT_VA_ROBOTWIN_I2AV.name: PIPELINE_LINGBOT_VA_ROBOTWIN_I2AV, +} +RUNNER_CONFIGS: dict[str, RunnerConfig] = { + RUNNER_LINGBOT_VA_ROBOTWIN_I2AV.runner_name: RUNNER_LINGBOT_VA_ROBOTWIN_I2AV, +} diff --git a/integrations/lingbot_va/lingbot_va/constants.py b/integrations/lingbot_va/lingbot_va/constants.py new file mode 100644 index 000000000..2b40f082c --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/constants.py @@ -0,0 +1,99 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Robotwin constants for the LingBot-VA FlashDreams integration.""" + +from __future__ import annotations + +from pathlib import Path + +RUNNER_NAME_ROBOTWIN_I2AV = "lingbot-va-robotwin-i2av" +DEFAULT_CHECKPOINT_ROOT = "robbyant/lingbot-va-posttrain-robotwin" +DEFAULT_INPUT_IMAGE_DIR = Path("assets/example_data/lingbot-va/robotwin") +DEFAULT_OUTPUT_DIR = Path("outputs/lingbot_va/robotwin_i2av") +DEFAULT_PROMPT = ( + "Grab the medium-sized white mug, rotate it, place it on the table, " + "and hook it onto the smooth dark gray rack." +) + +ROBOTWIN_HEIGHT = 256 +ROBOTWIN_WIDTH = 320 +ROBOTWIN_ATTENTION_WINDOW = 64 +ROBOTWIN_FRAME_CHUNK_SIZE = 2 +ROBOTWIN_ACTION_DIM = 30 +ROBOTWIN_ACTION_PER_FRAME = 16 +ROBOTWIN_GUIDANCE_SCALE = 5.0 +ROBOTWIN_ACTION_GUIDANCE_SCALE = 1.0 +ROBOTWIN_VIDEO_INFERENCE_STEPS = 25 +ROBOTWIN_ACTION_INFERENCE_STEPS = 50 +ROBOTWIN_SNR_SHIFT = 5.0 +ROBOTWIN_ACTION_SNR_SHIFT = 1.0 +ROBOTWIN_PATCH_SIZE = (1, 2, 2) +ROBOTWIN_ENV_TYPE = "robotwin_tshape" +ROBOTWIN_OBS_CAM_KEYS = ( + "observation.images.cam_high", + "observation.images.cam_left_wrist", + "observation.images.cam_right_wrist", +) +ROBOTWIN_USED_ACTION_CHANNEL_IDS = tuple( + list(range(0, 7)) + [28] + list(range(7, 14)) + [29] +) +ROBOTWIN_ACTION_Q01 = ( + -0.06172713458538055, + -3.6716461181640625e-05, + -0.08783501386642456, + -1.0, + -1.0, + -1.0, + -1.0, + -0.3547105032205582, + -1.3113021850585938e-06, + -0.11975435614585876, + -1.0, + -1.0, + -1.0, + -1.0, + *([0.0] * 16), +) +ROBOTWIN_ACTION_Q99 = ( + 0.3462600058317184, + 0.39966784834861746, + 0.14745532035827624, + 1.0, + 1.0, + 1.0, + 1.0, + 0.034201726913452024, + 0.39142737388610793, + 0.1792279863357542, + 1.0, + 1.0, + 1.0, + 1.0, + *([0.0] * 14), + 1.0, + 1.0, +) + +ROBOTWIN_LATENT_HEIGHT = (ROBOTWIN_HEIGHT // 16 * 3) // 2 +ROBOTWIN_LATENT_WIDTH = ROBOTWIN_WIDTH // 16 +ROBOTWIN_LATENT_CHANNELS = 48 +ROBOTWIN_LATENT_TOKEN_PER_CHUNK = ( + ROBOTWIN_FRAME_CHUNK_SIZE + * ROBOTWIN_LATENT_HEIGHT + * ROBOTWIN_LATENT_WIDTH + // (ROBOTWIN_PATCH_SIZE[0] * ROBOTWIN_PATCH_SIZE[1] * ROBOTWIN_PATCH_SIZE[2]) +) +ROBOTWIN_ACTION_TOKEN_PER_CHUNK = ROBOTWIN_FRAME_CHUNK_SIZE * ROBOTWIN_ACTION_PER_FRAME diff --git a/integrations/lingbot_va/lingbot_va/pipeline.py b/integrations/lingbot_va/lingbot_va/pipeline.py new file mode 100644 index 000000000..925beb808 --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/pipeline.py @@ -0,0 +1,254 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""LingBot-VA inference pipeline with dual-denoise generate() override.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, NamedTuple + +import torch +import torch.nn.functional as F +from einops import rearrange +from torch import Tensor +from tqdm import tqdm + +from flashdreams.infra.pipeline import StreamInferencePipeline, StreamInferencePipelineConfig +from flashdreams.infra.pipeline.base import StreamInferencePipelineCache + +from lingbot_va.scheduler import LingbotVAFlowMatchScheduler, LingbotVAFlowMatchSchedulerConfig +from lingbot_va.transformer import LingbotVATransformer, LingbotVATransformerCache +from lingbot_va.utils import get_mesh_id + + +class LingbotVAOutput(NamedTuple): + """Output from one AR step of the LingBot-VA pipeline.""" + latent: Tensor + action: Tensor + + +@dataclass(kw_only=True) +class LingbotVAInferencePipelineConfig(StreamInferencePipelineConfig): + """Pipeline config for LingBot-VA Robotwin I2AV. + + Holds the video scheduler via ``diffusion_model.scheduler`` and the + action scheduler separately. + """ + + _target: type["LingbotVAInferencePipeline"] = field( + default_factory=lambda: LingbotVAInferencePipeline + ) + + checkpoint_root: str + robotwin_height: int = 256 + robotwin_width: int = 320 + frame_chunk_size: int = 2 + action_dim: int = 30 + action_per_frame: int = 16 + attn_window: int = 64 + latent_height: int = 24 + latent_width: int = 20 + latent_channels: int = 48 + latent_token_per_chunk: int = 240 + action_token_per_chunk: int = 32 + + action_scheduler: LingbotVAFlowMatchSchedulerConfig = field( + default_factory=lambda: LingbotVAFlowMatchSchedulerConfig( + num_inference_steps=10, + shift=3.0, + ) + ) + + +class LingbotVAInferencePipeline(StreamInferencePipeline): + """LingBot-VA pipeline with dual video+action denoising. + + Overrides ``generate()`` to run the video denoise loop followed by the + action denoise loop, both writing to the shared KV cache. The + ``DiffusionModel`` built by the parent provides the transformer and + video scheduler; the action scheduler is held separately. + """ + + config: LingbotVAInferencePipelineConfig + + def __init__(self, config: LingbotVAInferencePipelineConfig) -> None: + super().__init__(config) + self.config = config + self.action_scheduler = config.action_scheduler.setup() + + @property + def transformer(self) -> LingbotVATransformer: + return self.diffusion_model.transformer # type: ignore[return-value] + + @property + def video_scheduler(self) -> LingbotVAFlowMatchScheduler: + return self.diffusion_model.scheduler # type: ignore[return-value] + + def initialize_cache( # type: ignore[override] + self, + *, + text_embeddings: Tensor, + negative_text_embeddings: Tensor | None = None, + batch_size: int = 1, + ) -> StreamInferencePipelineCache: + """Build the per-rollout cache.""" + transformer_cache = self.transformer.initialize_autoregressive_cache( + text_embeddings=text_embeddings, + negative_text_embeddings=negative_text_embeddings, + batch_size=batch_size, + ) + return StreamInferencePipelineCache( + transformer_cache=transformer_cache, + encoder_cache=None, + decoder_cache=None, + ) + + @torch.no_grad() + def generate( # type: ignore[override] + self, + autoregressive_index: int, + cache: StreamInferencePipelineCache, + input: dict[str, Any] | None = None, + ) -> LingbotVAOutput: + """Run video + action denoising for one AR chunk. + + Args: + autoregressive_index: Chunk index (0-based). + cache: Per-rollout cache from ``initialize_cache``. + input: Dict with keys: + - ``init_latent``: Encoded observation ``[B, C, 1, H, W]``. + - ``action_mask``: Bool mask ``[action_dim]``. + - ``device``: Target device. + - ``dtype``: Target dtype. + + Returns: + ``LingbotVAOutput(latent, action)`` for this chunk. + """ + assert input is not None + cfg = self.config + transformer_cache: LingbotVATransformerCache = cache.transformer_cache # type: ignore[assignment] + + init_latent = input["init_latent"] + action_mask = input["action_mask"] + device = input["device"] + dtype = input["dtype"] + + fcs = cfg.frame_chunk_size + ps = self.transformer.config.network.patch_size + lh = cfg.latent_height + lw = cfg.latent_width + frame_st_id = autoregressive_index * fcs + + # Initial noise + latents = torch.randn(1, cfg.latent_channels, fcs, lh, lw, device=device, dtype=dtype) + actions = torch.randn(1, cfg.action_dim, fcs, cfg.action_per_frame, 1, device=device, dtype=dtype) + + # Timesteps + video_timesteps = self.video_scheduler.padded_timesteps + action_timesteps = self.action_scheduler.padded_timesteps + + # RoPE grid IDs + video_grid_id = get_mesh_id( + fcs // ps[0], lh // ps[1], lw // ps[2], 0, 1, frame_st_id, + ).to(device) + action_grid_id = get_mesh_id( + fcs, cfg.action_per_frame, 1, 1, 1, frame_st_id, action=True, + ).to(device) + + # Open cache window + transformer_cache.start(autoregressive_index) + + # --- Video denoise --- + for i, t in enumerate(tqdm(video_timesteps, desc="video denoise")): + last_step = i == len(video_timesteps) - 1 + latent_cond = init_latent[:, :, 0:1].to(dtype) if frame_st_id == 0 else None + + noisy = latents.clone() + if latent_cond is not None: + noisy[:, :, 0:1] = latent_cond + x = rearrange( + noisy, + 'b c (f p1) (h p2) (w p3) -> b (f h w) (c p1 p2 p3)', + p1=ps[0], p2=ps[1], p3=ps[2], + ) + + t_val = float(t) + n_tokens = x.shape[1] + ts = torch.full((1, n_tokens), t_val, dtype=torch.float32, device=device) + if latent_cond is not None and frame_st_id == 0: + cond_tokens = (lh // ps[1]) * (lw // ps[2]) + ts[:, :cond_tokens] = 0.0 + + pred = self.transformer.predict_flow( + x, ts, transformer_cache, input={"grid_id": video_grid_id}, + persist=last_step, + ) + + if not last_step: + pred = rearrange( + pred, + 'b (f h w) (c kt kh kw) -> b c (f kt) (h kh) (w kw)', + f=fcs // ps[0], h=lh // ps[1], w=lw // ps[2], + kt=ps[0], kh=ps[1], kw=ps[2], + ) + latents = self.video_scheduler.step(pred, t, latents) + + latents[:, :, 0:1] = latent_cond if frame_st_id == 0 else latents[:, :, 0:1] + + # --- Action denoise --- + for i, t in enumerate(tqdm(action_timesteps, desc="action denoise")): + last_step = i == len(action_timesteps) - 1 + action_cond = ( + torch.zeros(1, cfg.action_dim, 1, cfg.action_per_frame, 1, device=device, dtype=dtype) + if frame_st_id == 0 else None + ) + + noisy_a = actions.clone() + if action_cond is not None: + noisy_a[:, :, 0:1] = action_cond + noisy_a[:, ~action_mask] *= 0 + x_a = rearrange(noisy_a, 'b c f h w -> b (f h w) c') + + t_val = float(t) + n_tokens_a = x_a.shape[1] + ts_a = torch.full((1, n_tokens_a), t_val, dtype=torch.float32, device=device) + if action_cond is not None and frame_st_id == 0: + ts_a[:, :cfg.action_per_frame] = 0.0 + + pred = self.transformer.predict_action_flow( + x_a, ts_a, transformer_cache, input={"grid_id": action_grid_id}, + persist=last_step, + ) + + if not last_step: + pred = rearrange(pred, "b (f n) c -> b c f n 1", f=fcs) + actions = self.action_scheduler.step(pred, t, actions) + + actions[:, :, 0:1] = action_cond if frame_st_id == 0 else actions[:, :, 0:1] + + # Close cache window and commit + transformer_cache.finalize(autoregressive_index) + + actions[:, ~action_mask] *= 0 + return LingbotVAOutput(latent=latents, action=actions) + + def finalize( # type: ignore[override] + self, + autoregressive_index: int, + cache: StreamInferencePipelineCache, + ) -> None: + """No-op — cache is finalized within generate().""" + pass diff --git a/integrations/lingbot_va/lingbot_va/runner.py b/integrations/lingbot_va/lingbot_va/runner.py new file mode 100644 index 000000000..74daafe39 --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/runner.py @@ -0,0 +1,503 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Runner for the LingBot-VA Robotwin I2AV integration. + +Uses the native compiled LingBot-VA transformer wrapper for inference +with ``torch.compile`` acceleration. +""" + +from __future__ import annotations + +import gc +import html +import json +import os +import re +import time +from dataclasses import dataclass, field +from pathlib import Path + +import numpy as np +import torch +import torch.nn.functional as F +from loguru import logger +from PIL import Image + +from flashdreams.infra.runner import Runner, RunnerConfig +from lingbot_va.action import LingbotVAActionProcessor +from lingbot_va.constants import ( + DEFAULT_CHECKPOINT_ROOT, + DEFAULT_INPUT_IMAGE_DIR, + DEFAULT_OUTPUT_DIR, + DEFAULT_PROMPT, + ROBOTWIN_ACTION_DIM, + ROBOTWIN_ACTION_GUIDANCE_SCALE, + ROBOTWIN_ACTION_INFERENCE_STEPS, + ROBOTWIN_ACTION_PER_FRAME, + ROBOTWIN_ACTION_SNR_SHIFT, + ROBOTWIN_ATTENTION_WINDOW, + ROBOTWIN_FRAME_CHUNK_SIZE, + ROBOTWIN_GUIDANCE_SCALE, + ROBOTWIN_HEIGHT, + ROBOTWIN_OBS_CAM_KEYS, + ROBOTWIN_PATCH_SIZE, + ROBOTWIN_SNR_SHIFT, + ROBOTWIN_USED_ACTION_CHANNEL_IDS, + ROBOTWIN_VIDEO_INFERENCE_STEPS, + ROBOTWIN_WIDTH, +) +from lingbot_va.pipeline import LingbotVAInferencePipeline +from lingbot_va.utils import resolve_prompt + + +def _prompt_clean(text: str) -> str: + """Clean prompt text (inlined from diffusers prompt_clean).""" + try: + import ftfy + text = ftfy.fix_text(text) + except ImportError: + pass + text = html.unescape(html.unescape(text)) + text = re.sub(r"\s+", " ", text).strip() + return text + + + + +@dataclass(kw_only=True) +class LingbotVARobotwinRunnerConfig(RunnerConfig): + """User-facing config for the LingBot-VA Robotwin I2AV runner.""" + + _target: type["LingbotVARobotwinRunner"] = field( + default_factory=lambda: LingbotVARobotwinRunner + ) + + checkpoint_root: str | Path = DEFAULT_CHECKPOINT_ROOT + input_image_dir: Path = DEFAULT_INPUT_IMAGE_DIR + prompt: str | Path = DEFAULT_PROMPT + num_chunks: int = 10 + save_video: bool = True + save_actions: bool = True + metadata_only: bool = False + seed: int = 42 + enable_offload: bool = False + compile_network: bool = True + benchmark: bool = False + + pixel_height: int = ROBOTWIN_HEIGHT + pixel_width: int = ROBOTWIN_WIDTH + frame_chunk_size: int = ROBOTWIN_FRAME_CHUNK_SIZE + action_dim: int = ROBOTWIN_ACTION_DIM + action_per_frame: int = ROBOTWIN_ACTION_PER_FRAME + attn_window: int = ROBOTWIN_ATTENTION_WINDOW + guidance_scale: float = ROBOTWIN_GUIDANCE_SCALE + action_guidance_scale: float = ROBOTWIN_ACTION_GUIDANCE_SCALE + num_inference_steps: int = ROBOTWIN_VIDEO_INFERENCE_STEPS + action_num_inference_steps: int = ROBOTWIN_ACTION_INFERENCE_STEPS + snr_shift: float = ROBOTWIN_SNR_SHIFT + action_snr_shift: float = ROBOTWIN_ACTION_SNR_SHIFT + patch_size: tuple[int, int, int] = ROBOTWIN_PATCH_SIZE + + +class LingbotVARobotwinRunner( + Runner[LingbotVARobotwinRunnerConfig, LingbotVAInferencePipeline] +): + """Robotwin I2AV driver with full GPU inference.""" + + config: LingbotVARobotwinRunnerConfig + + # ------------------------------------------------------------------ + # helpers + # ------------------------------------------------------------------ + + def _required_image_paths(self) -> dict[str, Path]: + return { + key: self.config.input_image_dir / f"{key}.png" + for key in ROBOTWIN_OBS_CAM_KEYS + } + + def _validate_inputs(self) -> dict[str, Path]: + image_paths = self._required_image_paths() + missing = [p for p in image_paths.values() if not p.exists()] + if missing: + raise FileNotFoundError( + "Missing camera PNGs: " + ", ".join(str(p) for p in missing) + ) + return image_paths + + def _write_metadata_manifest(self) -> Path: + cfg = self.config + prompt = resolve_prompt(cfg.prompt) + image_paths = self._validate_inputs() + cfg.output_dir.mkdir(parents=True, exist_ok=True) + manifest_path = cfg.output_dir / "runtime.json" + manifest = { + "runner_name": cfg.runner_name, + "checkpoint_root": str(cfg.checkpoint_root), + "input_image_dir": str(cfg.input_image_dir), + "image_paths": {k: str(v) for k, v in image_paths.items()}, + "prompt": prompt, + "num_chunks": cfg.num_chunks, + "seed": cfg.seed, + "height": cfg.pixel_height, + "width": cfg.pixel_width, + } + manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8") + return manifest_path + + # ------------------------------------------------------------------ + # model loading + # ------------------------------------------------------------------ + + def _load_models(self, device: torch.device, dtype: torch.dtype): + """Load VAE(s), text encoder, tokenizer, and transformer weights.""" + from lingbot_va._loaders import ( + WanVAEStreamingWrapper, + load_text_encoder, + load_tokenizer, + load_vae, + ) + + cfg = self.config + ckpt = str(cfg.checkpoint_root) + enable_offload = cfg.enable_offload + vae_device = "cpu" if enable_offload else device + + self._vae = load_vae(os.path.join(ckpt, "vae"), dtype, vae_device) + self._streaming_vae = WanVAEStreamingWrapper(self._vae) + + self._streaming_vae_half = WanVAEStreamingWrapper(self._vae) + + self._tokenizer = load_tokenizer(os.path.join(ckpt, "tokenizer")) + self._text_encoder = load_text_encoder( + os.path.join(ckpt, "text_encoder"), dtype, + "cpu" if enable_offload else device, + ) + + # Propagate runner overrides to the pipeline's transformer config + self.pipeline.transformer.config.checkpoint_root = ckpt + self.pipeline.transformer.config.compile_network = cfg.compile_network + self.pipeline.transformer.load_model(device) + + # ------------------------------------------------------------------ + # prompt encoding (from upstream _get_t5_prompt_embeds / encode_prompt) + # ------------------------------------------------------------------ + + def _encode_prompt(self, prompt: str, device: torch.device, dtype: torch.dtype): + prompt = _prompt_clean(prompt) + text_inputs = self._tokenizer( + [prompt], + padding="max_length", + max_length=512, + truncation=True, + add_special_tokens=True, + return_attention_mask=True, + return_tensors="pt", + ) + ids, mask = text_inputs.input_ids, text_inputs.attention_mask + seq_len = mask.gt(0).sum(dim=1).long() + + enc_device = next(self._text_encoder.parameters()).device + embeds = self._text_encoder(ids.to(enc_device), mask.to(enc_device)).last_hidden_state + embeds = embeds.to(dtype=dtype, device=device) + embeds = [u[:v] for u, v in zip(embeds, seq_len)] + embeds = torch.stack([ + torch.cat([u, u.new_zeros(512 - u.size(0), u.size(1))]) for u in embeds + ], dim=0) + return embeds.to(device) + + # ------------------------------------------------------------------ + # observation encoding (from upstream _encode_obs) + # ------------------------------------------------------------------ + + def _encode_obs(self, obs_images: dict[str, np.ndarray], + device: torch.device, dtype: torch.dtype): + cfg = self.config + videos = [] + for k_i, k in enumerate(ROBOTWIN_OBS_CAM_KEYS): + if k_i == 0: + h_i, w_i = cfg.pixel_height, cfg.pixel_width + else: + h_i, w_i = cfg.pixel_height // 2, cfg.pixel_width // 2 + img = torch.from_numpy(obs_images[k]).float().permute(2, 0, 1).unsqueeze(1) + img = F.interpolate(img, size=(h_i, w_i), mode="bilinear", align_corners=False) + videos.append(img.unsqueeze(0)) + + videos_high = videos[0] / 255.0 * 2.0 - 1.0 + videos_wrist = torch.cat(videos[1:], dim=0) / 255.0 * 2.0 - 1.0 + + vae_device = next(self._streaming_vae.vae.parameters()).device + enc_high = self._streaming_vae.encode_chunk(videos_high.to(vae_device).to(dtype)) + enc_wrist = self._streaming_vae_half.encode_chunk(videos_wrist.to(vae_device).to(dtype)) + + enc_out = torch.cat([ + torch.cat(enc_wrist.split(1, dim=0), dim=-1), + enc_high, + ], dim=-2) + + mu, _ = torch.chunk(enc_out, 2, dim=1) + latents_mean = torch.tensor(self._vae.config.latents_mean).to(mu.device) + latents_std = torch.tensor(self._vae.config.latents_std).to(mu.device) + mu_norm = self._normalize_latents(mu, latents_mean, 1.0 / latents_std) + return mu_norm.to(device) + + @staticmethod + def _normalize_latents(latents, mean, std): + mean = mean.view(1, -1, 1, 1, 1).to(latents.device) + std = std.view(1, -1, 1, 1, 1).to(latents.device) + return ((latents.float() - mean) * std).to(latents) + + # ------------------------------------------------------------------ + # main entry + # ------------------------------------------------------------------ + + def run(self) -> None: + cfg = self.config + + if cfg.metadata_only: + mp = self._write_metadata_manifest() + logger.info("Wrote manifest -> {}", mp.resolve()) + return + + prompt = resolve_prompt(cfg.prompt) + image_paths = self._validate_inputs() + + device = torch.device(f"cuda:{self.local_rank}") + dtype = torch.bfloat16 + + # seed + torch.manual_seed(cfg.seed) + + # load models + logger.info("Loading models from {}", cfg.checkpoint_root) + self._load_models(device, dtype) + + # encode prompt + if cfg.benchmark: + torch.cuda.synchronize() + t_pipeline_start = time.perf_counter() + logger.info("Encoding prompt") + use_cfg = (cfg.guidance_scale > 1) or (cfg.action_guidance_scale > 1) + if cfg.enable_offload: + self._text_encoder = self._text_encoder.to(device) + prompt_embeds = self._encode_prompt(prompt, device, dtype) + neg_embeds = self._encode_prompt("", device, dtype) if use_cfg else prompt_embeds + if cfg.enable_offload: + self._text_encoder = self._text_encoder.cpu() + torch.cuda.empty_cache() + + # load observation images + obs_images = { + k: np.array(Image.open(str(p)).convert("RGB")) + for k, p in image_paths.items() + } + + # encode observation + logger.info("Encoding observation") + if cfg.enable_offload: + self._vae.to(device) + init_latent = self._encode_obs(obs_images, device, dtype) + if cfg.enable_offload: + self._vae.cpu() + torch.cuda.empty_cache() + + # initialize pipeline cache + logger.info("Initializing AR cache") + pipeline_cache = self.pipeline.initialize_cache( + text_embeddings=prompt_embeds, + negative_text_embeddings=neg_embeds if use_cfg else None, + batch_size=1, + ) + + # action mask + action_mask = torch.zeros(cfg.action_dim, dtype=torch.bool, device=device) + action_mask[list(ROBOTWIN_USED_ACTION_CHANNEL_IDS)] = True + + # AR loop + cfg.output_dir.mkdir(parents=True, exist_ok=True) + action_processor = LingbotVAActionProcessor() + pred_latents = [] + pred_actions = [] + + if cfg.benchmark: + torch.cuda.synchronize() + t_infer_start = time.perf_counter() + chunk_times = [] + + for chunk_id in range(cfg.num_chunks): + logger.info("Generating chunk {}/{}", chunk_id + 1, cfg.num_chunks) + if cfg.benchmark: + torch.cuda.synchronize() + t_chunk_start = time.perf_counter() + output = self.pipeline.generate( + autoregressive_index=chunk_id, + cache=pipeline_cache, + input={ + "init_latent": init_latent, + "action_mask": action_mask, + "device": device, + "dtype": dtype, + }, + ) + if cfg.benchmark: + torch.cuda.synchronize() + chunk_times.append(time.perf_counter() - t_chunk_start) + post_actions = action_processor.postprocess(output.action) + pred_latents.append(output.latent.detach().cpu()) + pred_actions.append(post_actions) + del output + torch.cuda.empty_cache() + + if cfg.benchmark: + torch.cuda.synchronize() + t_infer_end = time.perf_counter() + infer_elapsed = t_infer_end - t_infer_start + pipeline_elapsed = t_infer_end - t_pipeline_start + + timing_results = { + "method": "flashdreams-lingbot-va", + "num_chunks": cfg.num_chunks, + "total_pipeline_sec": round(pipeline_elapsed, 4), + "total_inference_sec": round(infer_elapsed, 4), + "per_chunk_sec": [round(t, 4) for t in chunk_times], + "avg_chunk_sec": round(sum(chunk_times) / len(chunk_times), 4), + "note": "total_pipeline = prompt_enc + obs_enc + inference; total_inference = AR loop only; excludes model loading and video decoding", + } + timing_path = cfg.output_dir / "timing_flashdreams.json" + timing_path.write_text(json.dumps(timing_results, indent=2)) + logger.info("Timing saved to {}", timing_path) + logger.info("[FlashDreams] Pipeline: {:.4f}s, AR loop: {:.4f}s, avg chunk: {:.4f}s", + pipeline_elapsed, infer_elapsed, timing_results["avg_chunk_sec"]) + + # save + all_latents = torch.cat(pred_latents, dim=2) + all_actions = torch.cat(pred_actions, dim=1).flatten(1).numpy() + + if cfg.save_actions: + np.save(str(cfg.output_dir / "actions.npy"), all_actions) + logger.info("Saved actions.npy") + + torch.save(all_latents, str(cfg.output_dir / "latents.pt")) + logger.info("Saved latents.pt") + + # optional video decode + if cfg.save_video: + logger.info("Decoding video — freeing inference models") + logger.info("VRAM before cleanup: {:.2f} GiB allocated", + torch.cuda.memory_allocated() / 1024**3) + # Free KV caches (self-attn + cross-attn) + tc = pipeline_cache.transformer_cache + for nc in [tc.network_cache, tc.network_cache_uncond]: + if nc is None: + continue + for bc in nc.block_caches: + bc.self_attn.reset() + bc.cross_attn.text.k = torch.empty(0) + bc.cross_attn.text.v = torch.empty(0) + del pipeline_cache + # Free streaming VAE caches (but keep self._vae on GPU for decode) + self._streaming_vae.clear_cache() + if self._streaming_vae_half: + self._streaming_vae_half.clear_cache() + # Move all models except VAE to CPU + self.pipeline.transformer.network.to("cpu") + if hasattr(self, '_text_encoder') and self._text_encoder is not None: + self._text_encoder.to("cpu") + del self._streaming_vae + del self._streaming_vae_half + del self._text_encoder + del prompt_embeds, neg_embeds, init_latent, action_mask + del pred_latents, pred_actions + gc.collect() + torch.cuda.empty_cache() + logger.info("VRAM after cleanup: {:.2f} GiB allocated", + torch.cuda.memory_allocated() / 1024**3) + + self._vae = self._vae.to(device).to(dtype) + + from diffusers.utils import export_to_video + from diffusers.video_processor import VideoProcessor + + vp = VideoProcessor(vae_scale_factor=1) + + # Denormalize latents + lat = all_latents.to(device).to(self._vae.dtype) + del all_latents + lat_mean = ( + torch.tensor(self._vae.config.latents_mean) + .view(1, self._vae.config.z_dim, 1, 1, 1) + .to(lat.device, lat.dtype) + ) + lat_std = ( + 1.0 / torch.tensor(self._vae.config.latents_std) + .view(1, self._vae.config.z_dim, 1, 1, 1) + .to(lat.device, lat.dtype) + ) + lat = lat / lat_std + lat_mean + + # Memory-efficient decode: process frame-by-frame using the VAE's + # causal conv3d streaming interface but offload decoded frames to CPU + # immediately to avoid accumulating the full pixel-space video on GPU. + vae = self._vae + num_latent_frames = lat.shape[2] + vae.clear_cache() + x = vae.post_quant_conv(lat) + del lat + decoded_frames = [] + for i in range(num_latent_frames): + vae._conv_idx = [0] + if i == 0: + frame = vae.decoder( + x[:, :, i:i+1, :, :], + feat_cache=vae._feat_map, + feat_idx=vae._conv_idx, + first_chunk=True, + ) + else: + frame = vae.decoder( + x[:, :, i:i+1, :, :], + feat_cache=vae._feat_map, + feat_idx=vae._conv_idx, + ) + decoded_frames.append(frame.detach().cpu()) + del frame + del x + vae.clear_cache() + torch.cuda.empty_cache() + + out = torch.cat(decoded_frames, dim=2) + del decoded_frames + if vae.config.patch_size is not None: + from diffusers.models.autoencoders.autoencoder_kl_wan import unpatchify + out = unpatchify(out, patch_size=vae.config.patch_size) + video = torch.clamp(out, min=-1.0, max=1.0) + del out + + video = vp.postprocess_video(video, output_type="np")[0] + export_to_video(video, str(cfg.output_dir / "demo.mp4"), fps=10) + logger.info("Saved demo.mp4") + + # runtime manifest + manifest = { + "runner_name": cfg.runner_name, + "checkpoint_root": str(cfg.checkpoint_root), + "prompt": prompt, + "num_chunks": cfg.num_chunks, + "seed": cfg.seed, + "timestamp": time.strftime("%Y%m%d_%H%M%S"), + } + (cfg.output_dir / "runtime.json").write_text(json.dumps(manifest, indent=2)) + logger.info("Done. Outputs in {}", cfg.output_dir.resolve()) diff --git a/integrations/lingbot_va/lingbot_va/scheduler.py b/integrations/lingbot_va/lingbot_va/scheduler.py new file mode 100644 index 000000000..ec7a5c08e --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/scheduler.py @@ -0,0 +1,130 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""LingBot-VA upstream-compatible flow-match scheduler.""" + +from __future__ import annotations + +from dataclasses import dataclass, field + +import torch +from torch import Tensor + +from flashdreams.infra.diffusion.scheduler import FlowPredictor, Scheduler, SchedulerConfig + + +def warp_sigmas(sigmas: Tensor, shift: float) -> Tensor: + """Apply upstream LingBot-VA sigma shift ``shift * s / (1 + (shift - 1) * s)``.""" + return shift * sigmas / (1.0 + (shift - 1.0) * sigmas) + + +@dataclass(kw_only=True) +class LingbotVAFlowMatchSchedulerConfig(SchedulerConfig): + """Config matching ``raw/lingbot-va/wan_va/utils/scheduler.py`` inference behavior.""" + + _target: type["LingbotVAFlowMatchScheduler"] = field( + default_factory=lambda: LingbotVAFlowMatchScheduler + ) + + num_inference_steps: int = 25 + num_train_timesteps: int = 1000 + shift: float = 5.0 + sigma_max: float = 1.0 + sigma_min: float = 0.0 + inverse_timesteps: bool = False + extra_one_step: bool = True + reverse_sigmas: bool = False + + +class LingbotVAFlowMatchScheduler(Scheduler): + """Explicit Euler flow-match scheduler used by LingBot-VA video/action loops.""" + + timesteps: Tensor + sigmas: Tensor + + def __init__(self, config: LingbotVAFlowMatchSchedulerConfig) -> None: + super().__init__(config) + self.config: LingbotVAFlowMatchSchedulerConfig = config + sigmas, timesteps = self.build_schedule(config) + self.register_buffer("sigmas", sigmas, persistent=False) + self.register_buffer("timesteps", timesteps, persistent=False) + + @staticmethod + def build_schedule(config: LingbotVAFlowMatchSchedulerConfig) -> tuple[Tensor, Tensor]: + sigma_start = config.sigma_min + (config.sigma_max - config.sigma_min) + if config.extra_one_step: + sigmas = torch.linspace( + sigma_start, + config.sigma_min, + config.num_inference_steps + 1, + dtype=torch.float32, + )[:-1] + else: + sigmas = torch.linspace( + sigma_start, + config.sigma_min, + config.num_inference_steps, + dtype=torch.float32, + ) + if config.inverse_timesteps: + sigmas = torch.flip(sigmas, dims=[0]) + sigmas = warp_sigmas(sigmas, config.shift) + if config.reverse_sigmas: + sigmas = 1.0 - sigmas + timesteps = sigmas * config.num_train_timesteps + return sigmas, timesteps + + @property + def padded_timesteps(self) -> Tensor: + """Timesteps with the trailing zero that upstream appends before the final KV update.""" + return torch.nn.functional.pad(self.timesteps, (0, 1), mode="constant", value=0) + + def step(self, model_output: Tensor, timestep: Tensor, sample: Tensor) -> Tensor: + """Apply one upstream Euler step in sigma space.""" + timestep_cpu = timestep.detach().to(device=self.timesteps.device, dtype=self.timesteps.dtype) + timestep_id = torch.argmin((self.timesteps - timestep_cpu).abs()) + sigma = self.sigmas[timestep_id].to(device=sample.device, dtype=sample.dtype) + if int(timestep_id.item()) + 1 >= self.sigmas.shape[0]: + sigma_next = torch.zeros((), device=sample.device, dtype=sample.dtype) + else: + sigma_next = self.sigmas[timestep_id + 1].to(device=sample.device, dtype=sample.dtype) + return sample + model_output * (sigma_next - sigma) + + def sample( + self, + initial_noise: Tensor, + predict_flow: FlowPredictor, + rng: torch.Generator | None = None, + ) -> Tensor: + """Run Euler denoising over ``timesteps``. ``rng`` is accepted for interface parity.""" + del rng + sample = initial_noise + for timestep in self.timesteps: + flow = predict_flow(sample, timestep.to(device=sample.device, dtype=sample.dtype)) + sample = self.step(flow, timestep, sample) + return sample + + def add_noise( + self, + clean_input: Tensor, + timestep: Tensor, + rng: torch.Generator | None = None, + ) -> Tensor: + """Apply upstream forward corruption at the nearest scheduler timestep.""" + noise = torch.empty_like(clean_input).normal_(generator=rng) + timestep_cpu = timestep.detach().to(device=self.timesteps.device, dtype=self.timesteps.dtype) + timestep_id = torch.argmin((self.timesteps - timestep_cpu).abs()) + sigma = self.sigmas[timestep_id].to(device=clean_input.device, dtype=clean_input.dtype) + return (1.0 - sigma) * clean_input + sigma * noise diff --git a/integrations/lingbot_va/lingbot_va/transformer/__init__.py b/integrations/lingbot_va/lingbot_va/transformer/__init__.py new file mode 100644 index 000000000..a9c5370fe --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/transformer/__init__.py @@ -0,0 +1,280 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Native LingBot-VA transformer with torch.compile support. + +Wraps ``WanVADiTNetwork`` in the flashdreams ``Transformer`` interface +with ``predict_flow`` (video) and ``predict_action_flow`` (action). +""" + +from __future__ import annotations + +import gc +import os +from dataclasses import dataclass, field +from typing import Any + +import torch +import torch.nn as nn +from torch import Tensor + +from flashdreams.core.checkpoint.load import load_checkpoint +from flashdreams.infra.diffusion.transformer import ( + Transformer, + TransformerAutoregressiveCache, + TransformerConfig, +) + +from lingbot_va.transformer.checkpoint import state_dict_transform +from lingbot_va.transformer.impl.network import ( + WanVADiTNetwork, + WanVADiTNetworkCache, + WanVADiTNetworkConfig, + compute_rope_freqs_from_grid, +) + + +# --------------------------------------------------------------------------- +# Cache +# --------------------------------------------------------------------------- + +@dataclass(kw_only=True) +class LingbotVATransformerCache(TransformerAutoregressiveCache): + """Per-rollout AR cache.""" + + network_cache: WanVADiTNetworkCache + network_cache_uncond: WanVADiTNetworkCache | None = None + autoregressive_index: int = -1 + + def start(self, autoregressive_index: int) -> None: + """Open cache window for the given AR step.""" + self.autoregressive_index = autoregressive_index + for bc in self.network_cache.block_caches: + bc.self_attn.before_update(autoregressive_index) + if self.network_cache_uncond is not None: + for bc in self.network_cache_uncond.block_caches: + bc.self_attn.before_update(autoregressive_index) + + def finalize(self, autoregressive_index: int) -> None: + """Close cache window and commit the AR step.""" + for bc in self.network_cache.block_caches: + bc.self_attn.after_update(autoregressive_index) + if self.network_cache_uncond is not None: + for bc in self.network_cache_uncond.block_caches: + bc.self_attn.after_update(autoregressive_index) + self.autoregressive_index = autoregressive_index + + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + +@dataclass(kw_only=True) +class LingbotVATransformerConfig(TransformerConfig): + """Config for the native LingBot-VA transformer.""" + + _target: type["LingbotVATransformer"] = field( + default_factory=lambda: LingbotVATransformer + ) + + network: WanVADiTNetworkConfig = field(default_factory=WanVADiTNetworkConfig) + checkpoint_root: str = "" + dtype: torch.dtype = torch.bfloat16 + compile_network: bool = True + guidance_scale: float = 5.0 + action_guidance_scale: float = 1.0 + + # Spatial layout + latent_height: int = 0 + latent_width: int = 0 + frame_chunk_size: int = 4 + action_per_frame: int = 16 + attn_window: int = 64 + + +# --------------------------------------------------------------------------- +# Transformer +# --------------------------------------------------------------------------- + +class LingbotVATransformer(Transformer[LingbotVATransformerCache]): + """Native LingBot-VA transformer with torch.compile support.""" + + def __init__(self, config: LingbotVATransformerConfig) -> None: + super().__init__(config) + self.config: LingbotVATransformerConfig = config + self._anchor = nn.Parameter(torch.empty(0)) + # Use object.__setattr__ to avoid nn.Module type checks on compiled modules + object.__setattr__(self, '_network', None) + + def load_model(self, device: torch.device) -> None: + """Build, load weights, and optionally compile the network.""" + cfg = self.config + net = WanVADiTNetwork(cfg.network) + net.eval() + + ckpt_path = os.path.join(cfg.checkpoint_root, "transformer") + idx_path = os.path.join(ckpt_path, "diffusion_pytorch_model.safetensors.index.json") + if os.path.exists(idx_path): + ckpt_path = idx_path + state_dict = load_checkpoint(ckpt_path, map_location="cpu") + state_dict = state_dict_transform(state_dict) + net.load_state_dict(state_dict) + del state_dict + gc.collect() + + net.update_parameters_after_loading_checkpoint() + net = net.to(dtype=cfg.dtype, device=device) + + if cfg.compile_network: + net._forward_blocks_video = torch.compile( + net._forward_blocks_video, + mode="max-autotune-no-cudagraphs", + fullgraph=True, + ) + net._forward_blocks_action = torch.compile( + net._forward_blocks_action, + mode="max-autotune-no-cudagraphs", + fullgraph=True, + ) + + object.__setattr__(self, '_network', net) + + @property + def network(self) -> WanVADiTNetwork: + assert self._network is not None, "Call load_model() first" + return self._network + + # ------------------------------------------------------------------ + # Cache management + # ------------------------------------------------------------------ + + @torch.no_grad() + def initialize_autoregressive_cache( + self, + *, + text_embeddings: Tensor, + negative_text_embeddings: Tensor | None = None, + batch_size: int = 1, + **_unused: Any, + ) -> LingbotVATransformerCache: + cfg = self.config + ps = cfg.network.patch_size + video_chunk = ( + (cfg.frame_chunk_size // ps[0]) + * (cfg.latent_height // ps[1]) + * (cfg.latent_width // ps[2]) + ) + action_chunk = cfg.frame_chunk_size * cfg.action_per_frame + window_slots = cfg.attn_window // 2 + + network_cache = self.network.initialize_cache( + text_embeddings=text_embeddings, + video_chunk=video_chunk, + action_chunk=action_chunk, + window_slots=window_slots, + batch_size=batch_size, + ) + + network_cache_uncond: WanVADiTNetworkCache | None = None + if cfg.guidance_scale > 1.0 or cfg.action_guidance_scale > 1.0: + assert negative_text_embeddings is not None + network_cache_uncond = self.network.initialize_cache( + text_embeddings=negative_text_embeddings, + video_chunk=video_chunk, + action_chunk=action_chunk, + window_slots=window_slots, + batch_size=batch_size, + ) + + return LingbotVATransformerCache( + network_cache=network_cache, + network_cache_uncond=network_cache_uncond, + ) + + # ------------------------------------------------------------------ + # Flow prediction + # ------------------------------------------------------------------ + + def predict_flow( + self, + noisy_latent: Tensor, + timestep: Tensor, + cache: LingbotVATransformerCache, + input: Any = None, + persist: bool = False, + ) -> Tensor: + """Video-mode flow prediction with optional CFG.""" + grid_id = input["grid_id"] + if grid_id.shape[0] == 4: + grid_id = grid_id[:3] + rope_freqs = compute_rope_freqs_from_grid( + grid_id, self.config.network.dim // self.config.network.num_heads + ).to(noisy_latent.device) + + flow_cond = self.network.forward_video( + noisy_latent, timestep, cache.network_cache, rope_freqs, persist=persist, + ) + + if cache.network_cache_uncond is not None: + flow_uncond = self.network.forward_video( + noisy_latent, timestep, cache.network_cache_uncond, rope_freqs, persist=persist, + ) + if self.config.guidance_scale > 1.0: + return flow_uncond + self.config.guidance_scale * (flow_cond - flow_uncond) + + return flow_cond + + def predict_action_flow( + self, + noisy_action: Tensor, + timestep: Tensor, + cache: LingbotVATransformerCache, + input: Any = None, + persist: bool = False, + ) -> Tensor: + """Action-mode flow prediction with optional CFG.""" + grid_id = input["grid_id"] + if grid_id.shape[0] == 4: + grid_id = grid_id[:3] + rope_freqs = compute_rope_freqs_from_grid( + grid_id, self.config.network.dim // self.config.network.num_heads + ).to(noisy_action.device) + + flow_cond = self.network.forward_action( + noisy_action, timestep, cache.network_cache, rope_freqs, persist=persist, + ) + + if cache.network_cache_uncond is not None: + flow_uncond = self.network.forward_action( + noisy_action, timestep, cache.network_cache_uncond, rope_freqs, persist=persist, + ) + if self.config.action_guidance_scale > 1.0: + return flow_uncond + self.config.action_guidance_scale * (flow_cond - flow_uncond) + + return flow_cond + + # ------------------------------------------------------------------ + # Abstract method stubs + # ------------------------------------------------------------------ + + @property + def latent_shape(self) -> tuple[int, ...]: + return (0,) + + def patchify_and_maybe_split_cp(self, x: Any) -> Any: + return x + + def unpatchify_and_maybe_gather_cp(self, x: Tensor) -> Tensor: + return x diff --git a/integrations/lingbot_va/lingbot_va/transformer/checkpoint.py b/integrations/lingbot_va/lingbot_va/transformer/checkpoint.py new file mode 100644 index 000000000..63fd15978 --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/transformer/checkpoint.py @@ -0,0 +1,94 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Checkpoint weight remapping from upstream LingBot-VA to native network.""" + +from __future__ import annotations + +import re +from collections import OrderedDict + +from torch import Tensor + +_BLOCK_RULES: list[tuple[str, str]] = [ + (r"^blocks\.(\d+)\.attn1\.to_q\.(.+)$", r"blocks.\1.self_attn.q.\2"), + (r"^blocks\.(\d+)\.attn1\.to_k\.(.+)$", r"blocks.\1.self_attn.k.\2"), + (r"^blocks\.(\d+)\.attn1\.to_v\.(.+)$", r"blocks.\1.self_attn.v.\2"), + (r"^blocks\.(\d+)\.attn1\.to_out\.0\.(.+)$", r"blocks.\1.self_attn.o.\2"), + (r"^blocks\.(\d+)\.attn1\.norm_q\.(.+)$", r"blocks.\1.self_attn.norm_q.\2"), + (r"^blocks\.(\d+)\.attn1\.norm_k\.(.+)$", r"blocks.\1.self_attn.norm_k.\2"), + (r"^blocks\.(\d+)\.attn2\.to_q\.(.+)$", r"blocks.\1.cross_attn.q.\2"), + (r"^blocks\.(\d+)\.attn2\.to_k\.(.+)$", r"blocks.\1.cross_attn.k.\2"), + (r"^blocks\.(\d+)\.attn2\.to_v\.(.+)$", r"blocks.\1.cross_attn.v.\2"), + (r"^blocks\.(\d+)\.attn2\.to_out\.0\.(.+)$", r"blocks.\1.cross_attn.o.\2"), + (r"^blocks\.(\d+)\.attn2\.norm_q\.(.+)$", r"blocks.\1.cross_attn.norm_q.\2"), + (r"^blocks\.(\d+)\.attn2\.norm_k\.(.+)$", r"blocks.\1.cross_attn.norm_k.\2"), + (r"^blocks\.(\d+)\.norm2\.(.+)$", r"blocks.\1.norm3.\2"), + (r"^blocks\.(\d+)\.ffn\.net\.0\.proj\.(.+)$", r"blocks.\1.ffn.0.\2"), + (r"^blocks\.(\d+)\.ffn\.net\.2\.(.+)$", r"blocks.\1.ffn.2.\2"), + (r"^blocks\.(\d+)\.scale_shift_table$", r"blocks.\1.modulation"), +] + +_TOPLEVEL_RULES: list[tuple[str, str]] = [ + (r"^patch_embedding_mlp\.(.+)$", r"patch_embedding.\1"), + (r"^condition_embedder\.time_embedder\.linear_1\.(.+)$", r"time_embedding.0.\1"), + (r"^condition_embedder\.time_embedder\.linear_2\.(.+)$", r"time_embedding.2.\1"), + (r"^condition_embedder\.time_proj\.(.+)$", r"time_projection.1.\1"), + (r"^condition_embedder\.text_embedder\.linear_1\.(.+)$", r"text_embedding.0.\1"), + (r"^condition_embedder\.text_embedder\.linear_2\.(.+)$", r"text_embedding.2.\1"), + (r"^action_embedder\.(.+)$", r"action_embedder.\1"), + (r"^action_proj_out\.(.+)$", r"action_head.\1"), + (r"^condition_embedder_action\.time_embedder\.linear_1\.(.+)$", r"action_time_embedding.0.\1"), + (r"^condition_embedder_action\.time_embedder\.linear_2\.(.+)$", r"action_time_embedding.2.\1"), + (r"^condition_embedder_action\.time_proj\.(.+)$", r"action_time_projection.1.\1"), + (r"^condition_embedder_action\.text_embedder\.linear_1\.(.+)$", r"action_text_embedding.0.\1"), + (r"^condition_embedder_action\.text_embedder\.linear_2\.(.+)$", r"action_text_embedding.2.\1"), + (r"^proj_out\.(.+)$", r"head.head.\1"), + (r"^scale_shift_table$", r"head.modulation"), +] + +_DROP_PREFIXES = ( + "patch_embedding.", + "norm_out.", + "rope.", + "condition_embedder.timesteps_proj.", + "condition_embedder_action.timesteps_proj.", +) + +_ALL_RULES = _BLOCK_RULES + _TOPLEVEL_RULES + + +def state_dict_transform(state_dict: dict[str, Tensor]) -> dict[str, Tensor]: + """Remap upstream LingBot-VA checkpoint keys to native WanVADiTNetwork keys.""" + remapped: dict[str, Tensor] = OrderedDict() + + for key, tensor in state_dict.items(): + if any(key.startswith(p) for p in _DROP_PREFIXES): + continue + + new_key: str | None = None + for pattern, replacement in _ALL_RULES: + if re.match(pattern, key): + new_key = re.sub(pattern, replacement, key) + break + + if new_key is None: + raise ValueError( + f"Unmapped checkpoint key: {key!r}. " + "Update the remapping rules in checkpoint.py." + ) + remapped[new_key] = tensor + + return remapped diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/__init__.py b/integrations/lingbot_va/lingbot_va/transformer/impl/__init__.py new file mode 100644 index 000000000..ce36cd7d0 --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/__init__.py @@ -0,0 +1,15 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/kvcache.py b/integrations/lingbot_va/lingbot_va/transformer/impl/kvcache.py new file mode 100644 index 000000000..b3de6bed9 --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/kvcache.py @@ -0,0 +1,157 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""VAKVCache — rolling-window KV cache for video-action transformers. + +Wraps a ``BlockKVCache`` with a compile-friendly read path: intermediate +denoising steps read committed cache + concat fresh tokens (no writes), +while the final step writes the full [video|action] chunk via BlockKVCache.update(). +""" + +from __future__ import annotations + +from dataclasses import dataclass + +import torch +from torch import Tensor + +from flashdreams.core.attention.kvcache import BlockKVCache + + +@dataclass +class VAKVCache: + """Rolling-window KV cache for video-action models. + + Lifecycle per AR step: + cache.before_update(chunk_idx) + ... intermediate denoising: read via committed_kv_plus_fresh (no cache mutation) + ... final video step: write_video(k, v) + ... final action step: write_action(k, v) → commits full chunk + cache.after_update(chunk_idx) + """ + + kv_cache: BlockKVCache + video_chunk: int + action_chunk: int + + @staticmethod + def create( + *, + video_chunk: int, + action_chunk: int, + window_slots: int, + batch_size: int, + num_heads: int, + head_dim: int, + sink_size: int = 0, + device: torch.device | str = "cuda", + dtype: torch.dtype = torch.bfloat16, + ) -> "VAKVCache": + """Construct a VAKVCache with the given dimensions.""" + slot_size = video_chunk + action_chunk + window_size = window_slots * slot_size + total_size = sink_size + window_size + kv_shape = (batch_size, total_size, num_heads, head_dim) + + kv_cache = BlockKVCache( + k_shape=kv_shape, + v_shape=kv_shape, + seq_dim=1, + chunk_size=slot_size, + window_size=window_size, + sink_size=sink_size, + device=device, + dtype=dtype, + ) + + return VAKVCache( + kv_cache=kv_cache, + video_chunk=video_chunk, + action_chunk=action_chunk, + ) + + @property + def n_committed_tokens(self) -> int: + """Number of committed tokens from prior AR steps.""" + return self.kv_cache._n_cached + + def before_update(self, chunk_idx: int) -> None: + """Open the update window for a new AR step.""" + self.kv_cache.before_update(chunk_idx) + + def after_update(self, chunk_idx: int) -> None: + """Close the update window and commit.""" + self.kv_cache.after_update(chunk_idx) + + def committed_kv_plus_fresh( + self, k_fresh: Tensor, v_fresh: Tensor + ) -> tuple[Tensor, Tensor]: + """Read-only: committed prior tokens + fresh current tokens. + + Used during intermediate denoising steps. Does NOT mutate the cache. + This path is compile-friendly (pure tensor ops, no side effects). + + Args: + k_fresh: Shape ``[batch, L_fresh, heads, head_dim]``. + v_fresh: Same shape as ``k_fresh``. + + Returns: + ``(full_k, full_v)`` for attention context. + """ + n = self.n_committed_tokens + committed_k = self.kv_cache._k[:, :n] + committed_v = self.kv_cache._v[:, :n] + return ( + torch.cat([committed_k, k_fresh], dim=1), + torch.cat([committed_v, v_fresh], dim=1), + ) + + def write_video(self, k: Tensor, v: Tensor) -> None: + """Write video KV to the current chunk (pads action with zeros). + + Called once on the final video denoising step. + + Args: + k: Shape ``[batch, video_chunk, heads, head_dim]``. + v: Same shape as ``k``. + """ + batch, _, heads, head_dim = k.shape + action_k = torch.zeros( + batch, self.action_chunk, heads, head_dim, + device=k.device, dtype=k.dtype, + ) + action_v = torch.zeros_like(action_k) + full_k = torch.cat([k, action_k], dim=1) + full_v = torch.cat([v, action_v], dim=1) + self.kv_cache.update(full_k, full_v) + + def write_action(self, k: Tensor, v: Tensor, video_k: Tensor, video_v: Tensor) -> None: + """Write full [video|action] KV to the current chunk (overwrite). + + Called once on the final action denoising step. + + Args: + k: Action K, shape ``[batch, action_chunk, heads, head_dim]``. + v: Action V, same shape. + video_k: Video K from the final video step. + video_v: Video V from the final video step. + """ + full_k = torch.cat([video_k, k], dim=1) + full_v = torch.cat([video_v, v], dim=1) + self.kv_cache.update(full_k, full_v) + + def reset(self) -> None: + """Reset to empty state.""" + self.kv_cache.reset() diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py b/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py new file mode 100644 index 000000000..1b5bf9c77 --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py @@ -0,0 +1,219 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""VA-specific transformer building blocks. + +All block computations take pre-extracted KV tensors as arguments (no cache +object access), enabling ``torch.compile(fullgraph=True)`` without graph breaks. +Cache read/write operations happen at the network level, outside the compiled graph. +""" + +from __future__ import annotations + +import math +from dataclasses import dataclass + +import torch +import torch.nn as nn +from torch import Tensor + +from flashdreams.core.attention import NativeAttention +from flashdreams.core.attention.rope import apply_rope_freqs +from flashdreams.recipes.wan.transformer.impl.modules import ( + Block, + BlockCache, + CrossAttnCache, + CrossAttention, + MultiHeadAttention, +) + +from lingbot_va.transformer.impl.kvcache import VAKVCache + + +# --------------------------------------------------------------------------- +# Cache +# --------------------------------------------------------------------------- + +@dataclass +class VABlockCache: + """Per-block cache for a VA transformer block.""" + + self_attn: VAKVCache + cross_attn: CrossAttnCache + + +# --------------------------------------------------------------------------- +# VASelfAttention +# --------------------------------------------------------------------------- + +class VASelfAttention(MultiHeadAttention): + """Self-attention that takes committed KV as plain tensors. + + All methods receive pre-sliced committed K/V and cross-attn K/V as + tensor arguments — no cache object access inside, fully compile-friendly. + """ + + def forward( + self, + x: Tensor, + committed_k: Tensor, + committed_v: Tensor, + rope_freqs: Tensor, + ) -> tuple[Tensor, Tensor, Tensor]: + """Compute self-attention against committed cache + fresh tokens. + + Args: + x: Input hidden states ``[batch, L, dim]``. + committed_k: Committed K from prior steps ``[batch, N, heads, head_dim]``. + committed_v: Committed V from prior steps ``[batch, N, heads, head_dim]``. + rope_freqs: RoPE frequencies ``[L, 1, 1, head_dim]``. + + Returns: + ``(attn_output, k_fresh, v_fresh)`` — output + fresh KV for optional cache write. + """ + batch_size = x.shape[0] + L = x.shape[1] + n, d = self.n_heads, self.head_dim + + k_fresh = self.norm_k(self.k(x)).reshape(batch_size, L, n, d) + v_fresh = self.v(x).reshape(batch_size, L, n, d) + if self.apply_rope_before_kvcache: + k_fresh = apply_rope_freqs(k_fresh, rope_freqs, interleaved=True) + + q = self.norm_q(self.q(x)).reshape(batch_size, L, n, d) + q = apply_rope_freqs(q, rope_freqs, interleaved=True) + + if committed_k.shape[1] > 0: + full_k = torch.cat([committed_k, k_fresh], dim=1) + full_v = torch.cat([committed_v, v_fresh], dim=1) + else: + full_k = k_fresh + full_v = v_fresh + + out = self.attn_op(q, full_k, full_v) + out = out.reshape(batch_size, L, n * d) + return self.o(out), k_fresh, v_fresh + + +# --------------------------------------------------------------------------- +# VABlock +# --------------------------------------------------------------------------- + +class VABlock(nn.Module): + """Transformer block for video-action models. + + Takes committed KV and cross-attn KV as tensor arguments. + No cache object access inside — fully compile-friendly. + """ + + modulation: nn.Parameter + + def __init__( + self, + dim: int, + ffn_dim: int, + num_heads: int, + cross_attn_norm: bool = True, + eps: float = 1e-6, + apply_rope_before_kvcache: bool = True, + cp_method: str = "ring", + ) -> None: + super().__init__() + self.dim = dim + + self.norm1 = nn.LayerNorm(dim, eps=eps, elementwise_affine=False) + self.self_attn = VASelfAttention( + query_dim=dim, + n_heads=num_heads, + head_dim=dim // num_heads, + eps=eps, + apply_rope_before_kvcache=apply_rope_before_kvcache, + cp_method=cp_method, + ) + self.norm3 = ( + nn.LayerNorm(dim, eps, elementwise_affine=True) + if cross_attn_norm + else nn.Identity() + ) + self.cross_attn = CrossAttention( + query_dim=dim, + n_heads=num_heads, + head_dim=dim // num_heads, + eps=eps, + cp_method=cp_method, + ) + self.norm2 = nn.LayerNorm(dim, eps=eps, elementwise_affine=False) + self.ffn = nn.Sequential( + nn.Linear(dim, ffn_dim), + nn.GELU(approximate="tanh"), + nn.Linear(ffn_dim, dim), + ) + self.modulation = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5) + self._parameters_updated_after_loading_checkpoint = False + + def update_parameters_after_loading_checkpoint(self) -> None: + if self._parameters_updated_after_loading_checkpoint: + return + self.modulation.data = self.modulation.data.squeeze(0) + self._parameters_updated_after_loading_checkpoint = True + + def forward( + self, + x: Tensor, + e: Tensor, + committed_k: Tensor, + committed_v: Tensor, + cross_k: Tensor, + cross_v: Tensor, + rope_freqs: Tensor, + ) -> tuple[Tensor, Tensor, Tensor]: + """Run one transformer block. + + Args: + x: Hidden states ``[batch, L, dim]``. + e: Modulation ``[batch, L, 6, dim]``. + committed_k: Prior committed K ``[batch, N, heads, head_dim]``. + committed_v: Prior committed V ``[batch, N, heads, head_dim]``. + cross_k: Cross-attn text K ``[batch, T, heads, head_dim]``. + cross_v: Cross-attn text V ``[batch, T, heads, head_dim]``. + rope_freqs: RoPE frequencies ``[L, 1, 1, head_dim]``. + + Returns: + ``(x, k_fresh, v_fresh)`` — updated hidden states + fresh KV. + """ + assert self._parameters_updated_after_loading_checkpoint + e_chunks = [c.squeeze(-2) for c in (self.modulation + e).chunk(6, dim=-2)] + + y = self.norm1(x) * (1 + e_chunks[1]) + e_chunks[0] + attn_out, k_fresh, v_fresh = self.self_attn( + y, committed_k, committed_v, rope_freqs + ) + x = x + (attn_out * e_chunks[2]) + + # Cross-attention with pre-extracted text KV + B, L, D = x.shape + n, d = self.self_attn.n_heads, self.self_attn.head_dim + y2 = self.norm3(x) + q2 = self.cross_attn.norm_q(self.cross_attn.q(y2)).reshape(B, L, n, d) + out2 = self.cross_attn.attn_op(q2, cross_k, cross_v) + out2 = out2.reshape(B, L, n * d) + x = x + self.cross_attn.o(out2) + + # FFN + y3 = self.norm2(x) * (1 + e_chunks[4]) + e_chunks[3] + y3 = self.ffn(y3) + x = x + (y3 * e_chunks[5]) + + return x, k_fresh, v_fresh diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/network.py b/integrations/lingbot_va/lingbot_va/transformer/impl/network.py new file mode 100644 index 000000000..e36b0ab89 --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/network.py @@ -0,0 +1,422 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""WanVADiTNetwork — native flashdreams DiT for LingBot-VA.""" + +from __future__ import annotations + +import math +from dataclasses import dataclass, field +from typing import Any + +import torch +import torch.nn as nn +from torch import Tensor + +from flashdreams.core.attention.rope import apply_rope_freqs +from flashdreams.recipes.wan.transformer.impl.modules import ( + Head, + sinusoidal_embedding_1d, +) + +from lingbot_va.transformer.impl.kvcache import VAKVCache +from lingbot_va.transformer.impl.modules import VABlock, VABlockCache + + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + +@dataclass +class WanVADiTNetworkConfig: + """Network config for the LingBot-VA video-action DiT.""" + + action_dim: int = 30 + action_per_frame: int = 16 + patch_size: tuple[int, int, int] = (1, 2, 2) + in_dim: int = 48 + out_dim: int = 48 + dim: int = 3072 + ffn_dim: int = 14336 + num_heads: int = 24 + num_layers: int = 30 + text_dim: int = 4096 + freq_dim: int = 256 + text_len: int = 512 + cross_attn_norm: bool = True + eps: float = 1e-6 + apply_rope_before_kvcache: bool = True + cp_method: str = "ring" + + +# --------------------------------------------------------------------------- +# Network cache +# --------------------------------------------------------------------------- + +@dataclass +class WanVADiTNetworkCache: + """Per-block caches for the entire network.""" + + block_caches: list[VABlockCache] + + def __getitem__(self, index: int) -> VABlockCache: + return self.block_caches[index] + + +# --------------------------------------------------------------------------- +# RoPE helper +# --------------------------------------------------------------------------- + +def compute_rope_freqs_from_grid( + grid_id: Tensor, + head_dim: int, + theta: float = 10000.0, +) -> Tensor: + """Compute RoPE frequencies from grid_id ``[3, L]`` (f, h, w). + + Returns ``[L, 1, 1, head_dim]`` for ``apply_rope_freqs(..., interleaved=True)``. + """ + f_dim = head_dim - 2 * (head_dim // 3) + h_dim = head_dim // 3 + w_dim = head_dim // 3 + device = grid_id.device + + f_base = 1.0 / (theta ** (torch.arange(0, f_dim, 2, device=device, dtype=torch.float64)[: f_dim // 2] / f_dim)) + h_base = 1.0 / (theta ** (torch.arange(0, h_dim, 2, device=device, dtype=torch.float64)[: h_dim // 2] / h_dim)) + w_base = 1.0 / (theta ** (torch.arange(0, w_dim, 2, device=device, dtype=torch.float64)[: w_dim // 2] / w_dim)) + + f_angles = grid_id[0].to(torch.float64).unsqueeze(-1) * f_base.unsqueeze(0) + h_angles = grid_id[1].to(torch.float64).unsqueeze(-1) * h_base.unsqueeze(0) + w_angles = grid_id[2].to(torch.float64).unsqueeze(-1) * w_base.unsqueeze(0) + + # Interleaved format: each angle duplicated as [theta_0, theta_0, theta_1, theta_1, ...] + # to match RotaryPositionEmbedding3D._cat_freqs(interleaved=True) + freqs = torch.cat([ + f_angles.repeat_interleave(2, dim=-1), + h_angles.repeat_interleave(2, dim=-1), + w_angles.repeat_interleave(2, dim=-1), + ], dim=-1).float() + + return freqs.unsqueeze(1).unsqueeze(1) # [L, 1, 1, head_dim] + + +# --------------------------------------------------------------------------- +# Network +# --------------------------------------------------------------------------- + +class WanVADiTNetwork(nn.Module): + """Video-Action DiT network with native flashdreams building blocks.""" + + def __init__(self, config: WanVADiTNetworkConfig) -> None: + super().__init__() + self.config = config + self.dim = config.dim + self.freq_dim = config.freq_dim + self.text_dim = config.text_dim + self.out_dim = config.out_dim + self.num_heads = config.num_heads + self.num_layers = config.num_layers + self.patch_size = config.patch_size + self.eps = config.eps + self.text_len = config.text_len + self.head_dim = config.dim // config.num_heads + + # Video path embeddings + in_channels = config.in_dim * math.prod(config.patch_size) + self.patch_embedding = nn.Linear(in_channels, self.dim) + self.text_embedding = nn.Sequential( + nn.Linear(config.text_dim, self.dim), + nn.GELU(approximate="tanh"), + nn.Linear(self.dim, self.dim), + ) + self.time_embedding = nn.Sequential( + nn.Linear(config.freq_dim, self.dim), + nn.SiLU(), + nn.Linear(self.dim, self.dim), + ) + self.time_projection = nn.Sequential( + nn.SiLU(), + nn.Linear(self.dim, self.dim * 6), + ) + + # Action path embeddings + self.action_embedder = nn.Linear(config.action_dim, self.dim) + self.action_time_embedding = nn.Sequential( + nn.Linear(config.freq_dim, self.dim), + nn.SiLU(), + nn.Linear(self.dim, self.dim), + ) + self.action_time_projection = nn.Sequential( + nn.SiLU(), + nn.Linear(self.dim, self.dim * 6), + ) + self.action_text_embedding = nn.Sequential( + nn.Linear(config.text_dim, self.dim), + nn.GELU(approximate="tanh"), + nn.Linear(self.dim, self.dim), + ) + + # Shared transformer blocks + self.blocks = nn.ModuleList([ + VABlock( + dim=self.dim, + ffn_dim=config.ffn_dim, + num_heads=config.num_heads, + cross_attn_norm=config.cross_attn_norm, + eps=config.eps, + apply_rope_before_kvcache=config.apply_rope_before_kvcache, + cp_method=config.cp_method, + ) + for _ in range(config.num_layers) + ]) + + # Video output head + self.head = Head(self.dim, config.out_dim, config.patch_size, config.eps) + + # Action output head (shares head.modulation for norm, separate projection) + self.action_head = nn.Linear(self.dim, config.action_dim) + + self._parameters_updated_after_loading_checkpoint = False + + def update_parameters_after_loading_checkpoint(self) -> None: + if self._parameters_updated_after_loading_checkpoint: + return + self._fuse_shuffle_op_into_last_layer() + for block in self.blocks: + assert isinstance(block, VABlock) + block.update_parameters_after_loading_checkpoint() + self.head.update_parameters_after_loading_checkpoint() + self._parameters_updated_after_loading_checkpoint = True + + def _fuse_shuffle_op_into_last_layer(self) -> None: + """Fuse channel shuffle into head.head weights (same as WanDiTNetwork).""" + from einops import rearrange + kt, kh, kw = self.patch_size + self.head.head.weight.data = rearrange( + self.head.head.weight, + "(kt kh kw c) in_dim -> (c kt kh kw) in_dim", + kt=kt, kh=kh, kw=kw, c=self.out_dim, + ).contiguous() + if self.head.head.bias is not None: + self.head.head.bias.data = rearrange( + self.head.head.bias, + "(kt kh kw c) -> (c kt kh kw)", + kt=kt, kh=kh, kw=kw, c=self.out_dim, + ).contiguous() + + def initialize_cache( + self, + *, + text_embeddings: Tensor, + video_chunk: int, + action_chunk: int, + window_slots: int, + batch_size: int, + ) -> WanVADiTNetworkCache: + """Build per-block caches.""" + context_text = self.text_embedding(text_embeddings) + block_caches: list[VABlockCache] = [] + for block in self.blocks: + assert isinstance(block, VABlock) + self_attn_cache = VAKVCache.create( + video_chunk=video_chunk, + action_chunk=action_chunk, + window_slots=window_slots, + batch_size=batch_size, + num_heads=self.num_heads, + head_dim=self.head_dim, + device=text_embeddings.device, + dtype=text_embeddings.dtype, + ) + cross_attn_cache = block.cross_attn.initialize_cache(context_text) + block_caches.append(VABlockCache( + self_attn=self_attn_cache, + cross_attn=cross_attn_cache, + )) + return WanVADiTNetworkCache(block_caches=block_caches) + + def _extract_cache_tensors( + self, cache: WanVADiTNetworkCache + ) -> tuple[Tensor, Tensor, Tensor, Tensor]: + """Extract committed KV and cross-attn text KV as plain tensors. + + Called outside the compiled graph to avoid BlockKVCache graph breaks. + + Returns: + (committed_k_stack, committed_v_stack, cross_k_stack, cross_v_stack) + All shapes: [num_layers, batch, seq_len, heads, head_dim] + """ + committed_k = torch.stack([ + bc.self_attn.kv_cache._k[:, :bc.self_attn.n_committed_tokens] + for bc in cache.block_caches + ]) + committed_v = torch.stack([ + bc.self_attn.kv_cache._v[:, :bc.self_attn.n_committed_tokens] + for bc in cache.block_caches + ]) + cross_k = torch.stack([ + bc.cross_attn.text._k[:, :bc.cross_attn.text._n_cached] + for bc in cache.block_caches + ]) + cross_v = torch.stack([ + bc.cross_attn.text._v[:, :bc.cross_attn.text._n_cached] + for bc in cache.block_caches + ]) + return committed_k, committed_v, cross_k, cross_v + + def _forward_blocks_video( + self, + x: Tensor, + timesteps: Tensor, + committed_k: Tensor, + committed_v: Tensor, + cross_k: Tensor, + cross_v: Tensor, + rope_freqs: Tensor, + ) -> tuple[Tensor, list[Tensor], list[Tensor]]: + """Pure-tensor video forward through all blocks. Compile-friendly. + + Args: + committed_k: [num_layers, batch, N, heads, head_dim] + committed_v: same + cross_k: [num_layers, batch, T, heads, head_dim] + cross_v: same + + Returns: + (head_output, k_fresh_list, v_fresh_list) + """ + x = self.patch_embedding(x) + e = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, timesteps).type_as(x)) + e0 = self.time_projection(e).unflatten(-1, (6, self.dim)) + block_e = e0 + head_e = e.unsqueeze(-2) + + k_list: list[Tensor] = [] + v_list: list[Tensor] = [] + for block_idx, block in enumerate(self.blocks): + x, k_fresh, v_fresh = block( + x, block_e, + committed_k[block_idx], committed_v[block_idx], + cross_k[block_idx], cross_v[block_idx], + rope_freqs, + ) + k_list.append(k_fresh) + v_list.append(v_fresh) + + return self.head(x, head_e), k_list, v_list + + def _forward_blocks_action( + self, + x: Tensor, + timesteps: Tensor, + committed_k: Tensor, + committed_v: Tensor, + cross_k: Tensor, + cross_v: Tensor, + rope_freqs: Tensor, + ) -> tuple[Tensor, list[Tensor], list[Tensor]]: + """Pure-tensor action forward through all blocks. Compile-friendly. + + Returns: + (action_output, k_fresh_list, v_fresh_list) + """ + x = self.action_embedder(x) + e = self.action_time_embedding(sinusoidal_embedding_1d(self.freq_dim, timesteps).type_as(x)) + e0 = self.action_time_projection(e).unflatten(-1, (6, self.dim)) + block_e = e0 + head_e = e.unsqueeze(-2) + + k_list: list[Tensor] = [] + v_list: list[Tensor] = [] + for block_idx, block in enumerate(self.blocks): + x, k_fresh, v_fresh = block( + x, block_e, + committed_k[block_idx], committed_v[block_idx], + cross_k[block_idx], cross_v[block_idx], + rope_freqs, + ) + k_list.append(k_fresh) + v_list.append(v_fresh) + + # Action output: shared modulation + separate projection + e_chunks = [c.squeeze(-2) for c in (self.head.modulation + head_e).chunk(2, dim=-2)] + x = self.head.norm(x) * (1 + e_chunks[1]) + e_chunks[0] + return self.action_head(x), k_list, v_list + + def forward_video( + self, + x: Tensor, + timesteps: Tensor, + cache: WanVADiTNetworkCache, + rope_freqs: Tensor, + persist: bool = False, + ) -> Tensor: + """Video-mode forward. + + Cache extraction and writes happen outside the compiled block loop. + + Returns: + Video flow ``[batch, L, prod(patch_size) * out_dim]``. + """ + assert self._parameters_updated_after_loading_checkpoint + + # Extract cache tensors (outside compile boundary) + committed_k, committed_v, cross_k, cross_v = self._extract_cache_tensors(cache) + + # Compiled block loop (pure tensors, no cache access) + output, k_list, v_list = self._forward_blocks_video( + x, timesteps, committed_k, committed_v, cross_k, cross_v, rope_freqs, + ) + + # Cache write (outside compile boundary) + if persist: + for block_idx, (k, v) in enumerate(zip(k_list, v_list)): + cache[block_idx].self_attn.write_video(k, v) + self._last_video_kv = list(zip(k_list, v_list)) + + return output + + def forward_action( + self, + x: Tensor, + timesteps: Tensor, + cache: WanVADiTNetworkCache, + rope_freqs: Tensor, + persist: bool = False, + ) -> Tensor: + """Action-mode forward. + + Cache extraction and writes happen outside the compiled block loop. + + Returns: + Action flow ``[batch, L, action_dim]``. + """ + assert self._parameters_updated_after_loading_checkpoint + + # Extract cache tensors (outside compile boundary) + committed_k, committed_v, cross_k, cross_v = self._extract_cache_tensors(cache) + + # Compiled block loop (pure tensors, no cache access) + output, k_list, v_list = self._forward_blocks_action( + x, timesteps, committed_k, committed_v, cross_k, cross_v, rope_freqs, + ) + + # Cache write (outside compile boundary) + if persist: + for block_idx, (k, v) in enumerate(zip(k_list, v_list)): + vk, vv = self._last_video_kv[block_idx] + cache[block_idx].self_attn.write_action(k, v, vk, vv) + + return output diff --git a/integrations/lingbot_va/lingbot_va/utils.py b/integrations/lingbot_va/lingbot_va/utils.py new file mode 100644 index 000000000..6f2825fc4 --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/utils.py @@ -0,0 +1,105 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Small upstream-compatible tensor helpers for LingBot-VA.""" + +from __future__ import annotations + +from pathlib import Path + +import torch +from torch import Tensor + + +def get_mesh_id( + f: int, + h: int, + w: int, + t: int, + f_w: int = 1, + f_shift: int = 0, + action: bool = False, +) -> Tensor: + """Return upstream LingBot-VA 4-row RoPE/grid ids. + + Args: + f: Number of frame positions. + h: Number of height/action-per-frame positions. + w: Number of width positions. + t: Stream type id: upstream uses 0 for video and 1 for action. + f_w: Frame-position multiplier. + f_shift: Autoregressive frame offset. + action: If true, use fractional action positions and sentinel ``h/w=-1``. + + Returns: + Tensor of shape ``[4, f * h * w]``. + """ + f_idx = torch.arange(f_shift, f + f_shift) * f_w + h_idx = torch.arange(h) + w_idx = torch.arange(w) + ff, hh, ww = torch.meshgrid(f_idx, h_idx, w_idx, indexing="ij") + if action: + ff_offset = (torch.ones([h]).cumsum(0) / (h + 1)).view(1, -1, 1) + ff = ff + ff_offset + hh = torch.ones_like(hh) * -1 + ww = torch.ones_like(ww) * -1 + grid_id = torch.cat( + [ + ff.unsqueeze(0), + hh.unsqueeze(0), + ww.unsqueeze(0), + ], + dim=0, + ).flatten(1) + return torch.cat([grid_id, torch.full_like(grid_id[:1], t)], dim=0) + + +def data_seq_to_patch( + patch_size: tuple[int, int, int], + data_seq: Tensor, + latent_num_frames: int, + latent_height: int, + latent_width: int, + batch_size: int = 1, +) -> Tensor: + """Invert LingBot-VA's patch-sequence output into ``[B, C, F, H, W]``.""" + p_t, p_h, p_w = patch_size + post_patch_num_frames = latent_num_frames // p_t + post_patch_height = latent_height // p_h + post_patch_width = latent_width // p_w + + data_patch = data_seq.reshape( + batch_size, + post_patch_num_frames, + post_patch_height, + post_patch_width, + p_t, + p_h, + p_w, + -1, + ) + data_patch = data_patch.permute(0, 7, 1, 4, 2, 5, 3, 6) + data_patch = data_patch.flatten(6, 7).flatten(4, 5).flatten(2, 3) + return data_patch + + +def resolve_prompt(value: str | Path) -> str: + """Resolve an inline prompt or a text file whose first non-empty line is used.""" + if isinstance(value, Path): + lines = [line.strip() for line in value.read_text().splitlines() if line.strip()] + assert lines, f"prompt file {value} has no non-empty lines" + return lines[0] + assert value, "prompt must be a non-empty string or a path" + return value diff --git a/integrations/lingbot_va/pyproject.toml b/integrations/lingbot_va/pyproject.toml new file mode 100644 index 000000000..f1430e51d --- /dev/null +++ b/integrations/lingbot_va/pyproject.toml @@ -0,0 +1,50 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +[build-system] +requires = ["setuptools>=69", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "flashdreams-lingbot-va" +version = "0.1.0" +description = "LingBot-VA Robotwin I2AV inference integration scaffold for flashdreams." +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "flashdreams", + "diffusers>=0.34", + "transformers>=5.0", + "einops", +] + +[tool.uv.sources] +flashdreams = { workspace = true } + +[project.optional-dependencies] +dev = [ + "pytest>=8.0", + "tomli>=2.0", +] + +[project.entry-points."flashdreams.runner_configs"] +"lingbot-va-robotwin-i2av" = "lingbot_va.config:RUNNER_LINGBOT_VA_ROBOTWIN_I2AV" + +[tool.setuptools.packages.find] +include = ["lingbot_va*"] +exclude = ["tests"] + +[tool.uv] +managed = true From 0174d7e9647ffba88e4ca8e9060ced765a12358c Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Tue, 25 Aug 2026 14:28:43 -0700 Subject: [PATCH 02/18] Fix LingBot CFG branch KV ownership Signed-off-by: Jonathan McCaffrey --- .../lingbot_va/transformer/__init__.py | 33 ++++- .../lingbot_va/transformer/impl/network.py | 26 +++- .../lingbot_va/tests/test_cfg_cache.py | 116 ++++++++++++++++++ 3 files changed, 165 insertions(+), 10 deletions(-) create mode 100644 integrations/lingbot_va/tests/test_cfg_cache.py diff --git a/integrations/lingbot_va/lingbot_va/transformer/__init__.py b/integrations/lingbot_va/lingbot_va/transformer/__init__.py index a9c5370fe..7bf9ac408 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/__init__.py +++ b/integrations/lingbot_va/lingbot_va/transformer/__init__.py @@ -39,6 +39,7 @@ from lingbot_va.transformer.checkpoint import state_dict_transform from lingbot_va.transformer.impl.network import ( + VideoKV, WanVADiTNetwork, WanVADiTNetworkCache, WanVADiTNetworkConfig, @@ -56,6 +57,10 @@ class LingbotVATransformerCache(TransformerAutoregressiveCache): network_cache: WanVADiTNetworkCache network_cache_uncond: WanVADiTNetworkCache | None = None + video_kv_cond: VideoKV | None = None + """Video KV produced by the conditional branch for the current chunk.""" + video_kv_uncond: VideoKV | None = None + """Video KV produced by the unconditional branch for the current chunk.""" autoregressive_index: int = -1 def start(self, autoregressive_index: int) -> None: @@ -223,14 +228,20 @@ def predict_flow( grid_id, self.config.network.dim // self.config.network.num_heads ).to(noisy_latent.device) - flow_cond = self.network.forward_video( + flow_cond, video_kv_cond = self.network.forward_video( noisy_latent, timestep, cache.network_cache, rope_freqs, persist=persist, ) + if persist: + assert video_kv_cond is not None + cache.video_kv_cond = video_kv_cond if cache.network_cache_uncond is not None: - flow_uncond = self.network.forward_video( + flow_uncond, video_kv_uncond = self.network.forward_video( noisy_latent, timestep, cache.network_cache_uncond, rope_freqs, persist=persist, ) + if persist: + assert video_kv_uncond is not None + cache.video_kv_uncond = video_kv_uncond if self.config.guidance_scale > 1.0: return flow_uncond + self.config.guidance_scale * (flow_cond - flow_uncond) @@ -253,13 +264,27 @@ def predict_action_flow( ).to(noisy_action.device) flow_cond = self.network.forward_action( - noisy_action, timestep, cache.network_cache, rope_freqs, persist=persist, + noisy_action, + timestep, + cache.network_cache, + rope_freqs, + video_kv=cache.video_kv_cond, + persist=persist, ) + if persist: + cache.video_kv_cond = None if cache.network_cache_uncond is not None: flow_uncond = self.network.forward_action( - noisy_action, timestep, cache.network_cache_uncond, rope_freqs, persist=persist, + noisy_action, + timestep, + cache.network_cache_uncond, + rope_freqs, + video_kv=cache.video_kv_uncond, + persist=persist, ) + if persist: + cache.video_kv_uncond = None if self.config.action_guidance_scale > 1.0: return flow_uncond + self.config.action_guidance_scale * (flow_cond - flow_uncond) diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/network.py b/integrations/lingbot_va/lingbot_va/transformer/impl/network.py index e36b0ab89..da8a98b3b 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/impl/network.py +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/network.py @@ -34,6 +34,9 @@ from lingbot_va.transformer.impl.kvcache import VAKVCache from lingbot_va.transformer.impl.modules import VABlock, VABlockCache +VideoKV = tuple[tuple[Tensor, Tensor], ...] +"""Per-block video keys and values retained for the matching action branch.""" + # --------------------------------------------------------------------------- # Config @@ -362,13 +365,14 @@ def forward_video( cache: WanVADiTNetworkCache, rope_freqs: Tensor, persist: bool = False, - ) -> Tensor: + ) -> tuple[Tensor, VideoKV | None]: """Video-mode forward. Cache extraction and writes happen outside the compiled block loop. Returns: - Video flow ``[batch, L, prod(patch_size) * out_dim]``. + Video flow ``[batch, L, prod(patch_size) * out_dim]`` and the + per-block video KV written by a persistent call. """ assert self._parameters_updated_after_loading_checkpoint @@ -381,12 +385,13 @@ def forward_video( ) # Cache write (outside compile boundary) + video_kv: VideoKV | None = None if persist: for block_idx, (k, v) in enumerate(zip(k_list, v_list)): cache[block_idx].self_attn.write_video(k, v) - self._last_video_kv = list(zip(k_list, v_list)) + video_kv = tuple(zip(k_list, v_list)) - return output + return output, video_kv def forward_action( self, @@ -394,6 +399,7 @@ def forward_action( timesteps: Tensor, cache: WanVADiTNetworkCache, rope_freqs: Tensor, + video_kv: VideoKV | None = None, persist: bool = False, ) -> Tensor: """Action-mode forward. @@ -415,8 +421,16 @@ def forward_action( # Cache write (outside compile boundary) if persist: - for block_idx, (k, v) in enumerate(zip(k_list, v_list)): - vk, vv = self._last_video_kv[block_idx] + assert video_kv is not None, ( + "A persistent action pass requires video KV from the same CFG branch." + ) + assert len(video_kv) == len(k_list), ( + f"Expected {len(k_list)} video KV pairs, got {len(video_kv)}." + ) + for block_idx, (k, v, branch_video_kv) in enumerate( + zip(k_list, v_list, video_kv) + ): + vk, vv = branch_video_kv cache[block_idx].self_attn.write_action(k, v, vk, vv) return output diff --git a/integrations/lingbot_va/tests/test_cfg_cache.py b/integrations/lingbot_va/tests/test_cfg_cache.py new file mode 100644 index 000000000..185f9b77c --- /dev/null +++ b/integrations/lingbot_va/tests/test_cfg_cache.py @@ -0,0 +1,116 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU regressions for LingBot-VA conditional and unconditional KV ownership.""" + +from typing import Any + +import pytest +import torch +from torch import Tensor, nn + +from lingbot_va.transformer import ( + LingbotVATransformer, + LingbotVATransformerCache, + LingbotVATransformerConfig, +) +from lingbot_va.transformer.impl.network import ( + VideoKV, + WanVADiTNetworkCache, + WanVADiTNetworkConfig, +) + +pytestmark = pytest.mark.ci_cpu + + +class _BranchRecordingNetwork(nn.Module): + """Record the video KV supplied to each action branch.""" + + def __init__( + self, + cond_cache: WanVADiTNetworkCache, + uncond_cache: WanVADiTNetworkCache, + ) -> None: + super().__init__() + self._cond_cache = cond_cache + self._uncond_cache = uncond_cache + self.action_video_kv: dict[str, VideoKV | None] = {} + + def _branch(self, cache: WanVADiTNetworkCache) -> tuple[str, float]: + if cache is self._cond_cache: + return "cond", 1.0 + assert cache is self._uncond_cache + return "uncond", -1.0 + + def forward_video( + self, + x: Tensor, + timesteps: Tensor, + cache: WanVADiTNetworkCache, + rope_freqs: Tensor, + persist: bool = False, + ) -> tuple[Tensor, VideoKV | None]: + del timesteps, rope_freqs + _, value = self._branch(cache) + video_kv = ((torch.tensor([value]), torch.tensor([value * 10])),) + return torch.full_like(x, value), video_kv if persist else None + + def forward_action( + self, + x: Tensor, + timesteps: Tensor, + cache: WanVADiTNetworkCache, + rope_freqs: Tensor, + video_kv: VideoKV | None = None, + persist: bool = False, + ) -> Tensor: + del timesteps, rope_freqs + branch, value = self._branch(cache) + if persist: + self.action_video_kv[branch] = video_kv + return torch.full_like(x, value * 3) + + +def test_cfg_action_branches_consume_their_matching_video_kv() -> None: + cond_cache = WanVADiTNetworkCache(block_caches=[]) + uncond_cache = WanVADiTNetworkCache(block_caches=[]) + config = LingbotVATransformerConfig( + network=WanVADiTNetworkConfig(dim=12, num_heads=1, num_layers=1), + guidance_scale=5.0, + action_guidance_scale=1.0, + compile_network=False, + ) + transformer = LingbotVATransformer(config) + network = _BranchRecordingNetwork(cond_cache, uncond_cache) + object.__setattr__(transformer, "_network", network) + cache = LingbotVATransformerCache( + network_cache=cond_cache, + network_cache_uncond=uncond_cache, + ) + noisy = torch.zeros(1, 1, 1) + timestep = torch.zeros(1, 1) + model_input: dict[str, Any] = {"grid_id": torch.zeros(3, 1)} + + video_flow = transformer.predict_flow( + noisy, + timestep, + cache, + input=model_input, + persist=True, + ) + action_flow = transformer.predict_action_flow( + noisy, + timestep, + cache, + input=model_input, + persist=True, + ) + + assert torch.equal(video_flow, torch.full_like(noisy, 9.0)) + assert torch.equal(action_flow, torch.full_like(noisy, 3.0)) + assert network.action_video_kv["cond"] is not None + assert network.action_video_kv["uncond"] is not None + assert network.action_video_kv["cond"][0][0].item() == 1.0 + assert network.action_video_kv["uncond"][0][0].item() == -1.0 + assert cache.video_kv_cond is None + assert cache.video_kv_uncond is None From efb6bf30f800e5913327abac679217aca31d3fe2 Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Tue, 25 Aug 2026 14:35:27 -0700 Subject: [PATCH 03/18] Fix LingBot checkpoint and action contracts Signed-off-by: Jonathan McCaffrey --- .../lingbot_va/lingbot_va/_loaders.py | 310 ++++++++++-------- integrations/lingbot_va/lingbot_va/action.py | 59 +++- integrations/lingbot_va/tests/test_action.py | 59 ++++ integrations/lingbot_va/tests/test_loaders.py | 96 ++++++ 4 files changed, 388 insertions(+), 136 deletions(-) create mode 100644 integrations/lingbot_va/tests/test_action.py create mode 100644 integrations/lingbot_va/tests/test_loaders.py diff --git a/integrations/lingbot_va/lingbot_va/_loaders.py b/integrations/lingbot_va/lingbot_va/_loaders.py index d0fa8cef9..123e9756f 100644 --- a/integrations/lingbot_va/lingbot_va/_loaders.py +++ b/integrations/lingbot_va/lingbot_va/_loaders.py @@ -14,54 +14,186 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -"""Model loading utilities adapted from upstream LingBot-VA (wan_va/modules/utils.py -and wan_va/utils/scheduler.py).""" + +"""Checkpoint resolution and model loading for LingBot-VA.""" from __future__ import annotations -import math +from pathlib import Path +from typing import Any import torch -from diffusers import AutoencoderKLWan -from transformers import T5TokenizerFast, UMT5EncoderModel - +from torch import Tensor, nn + +from flashdreams.core.io.hf import maybe_download_hf_repo_on_rank0 + +_CHECKPOINT_COMPONENTS = ("transformer", "vae", "text_encoder", "tokenizer") +"""Subdirectories required from a LingBot-VA checkpoint snapshot.""" + + +def validate_checkpoint_root(path: str | Path) -> Path: + """Validate and return a local LingBot-VA checkpoint root. + + Args: + path: Local snapshot directory. + + Returns: + Expanded checkpoint directory. + + Raises: + FileNotFoundError: The root or a required component is missing. + NotADirectoryError: The root is not a directory. + """ + root = Path(path).expanduser() + if not root.exists(): + raise FileNotFoundError(f"LingBot-VA checkpoint root does not exist: {root}") + if not root.is_dir(): + raise NotADirectoryError( + f"LingBot-VA checkpoint root is not a directory: {root}" + ) + missing = [ + component + for component in _CHECKPOINT_COMPONENTS + if not (root / component).is_dir() + ] + if missing: + raise FileNotFoundError( + f"LingBot-VA checkpoint root {root} is missing component " + + ", ".join(repr(component) for component in missing) + + "." + ) + return root + + +def resolve_checkpoint_root( + checkpoint_root: str | Path, + *, + revision: str | None = None, +) -> Path: + """Resolve a local root or revision-pinned Hugging Face repo to a snapshot. + + Existing paths are always local. Nonexistent absolute, tilde-prefixed, and + dot-prefixed paths fail locally; other strings are treated as Hugging Face + repository IDs. + + Args: + checkpoint_root: Local directory or Hugging Face repository ID. + revision: Optional Hugging Face revision. + + Returns: + Validated local snapshot directory. + """ + value = str(checkpoint_root) + expanded = Path(value).expanduser() + if expanded.exists(): + return validate_checkpoint_root(expanded) + if ( + isinstance(checkpoint_root, Path) + or expanded.is_absolute() + or value.startswith((".", "~")) + ): + raise FileNotFoundError( + f"LingBot-VA checkpoint root does not exist: {expanded}" + ) + + allow_patterns = [ + component + for name in _CHECKPOINT_COMPONENTS + for component in (name, f"{name}/*", f"{name}/**") + ] + maybe_download_hf_repo_on_rank0( + value, + revision=revision, + allow_patterns=allow_patterns, + ) + from huggingface_hub import snapshot_download -def load_vae(vae_path: str, torch_dtype: torch.dtype, torch_device): - vae = AutoencoderKLWan.from_pretrained(vae_path, torch_dtype=torch_dtype) + local_root = snapshot_download( + repo_id=value, + revision=revision, + allow_patterns=allow_patterns, + local_files_only=True, + ) + return validate_checkpoint_root(local_root) + + +def load_vae( + checkpoint_root: Path, + torch_dtype: torch.dtype, + torch_device: torch.device | str, +) -> nn.Module: + """Load the Wan VAE from a resolved snapshot.""" + from diffusers import AutoencoderKLWan + + vae = AutoencoderKLWan.from_pretrained( + checkpoint_root, + subfolder="vae", + torch_dtype=torch_dtype, + local_files_only=True, + ) return vae.to(torch_device) -def load_text_encoder(text_encoder_path: str, torch_dtype: torch.dtype, torch_device): +def load_text_encoder( + checkpoint_root: Path, + torch_dtype: torch.dtype, + torch_device: torch.device | str, +) -> nn.Module: + """Load the UMT5 text encoder from a resolved snapshot.""" + from transformers import UMT5EncoderModel + text_encoder = UMT5EncoderModel.from_pretrained( - text_encoder_path, torch_dtype=torch_dtype + checkpoint_root, + subfolder="text_encoder", + torch_dtype=torch_dtype, + local_files_only=True, ) return text_encoder.to(torch_device) -def load_tokenizer(tokenizer_path: str): - return T5TokenizerFast.from_pretrained(tokenizer_path) +def load_tokenizer(checkpoint_root: Path) -> Any: + """Load the T5 tokenizer from a resolved snapshot.""" + from transformers import T5TokenizerFast + return T5TokenizerFast.from_pretrained( + checkpoint_root, + subfolder="tokenizer", + local_files_only=True, + ) -def patchify(x: torch.Tensor, patch_size: int | None) -> torch.Tensor: + +def patchify(x: Tensor, patch_size: int | None) -> Tensor: + """Fold a spatial VAE patch into the channel dimension.""" if patch_size is None or patch_size == 1: return x batch_size, channels, frames, height, width = x.shape x = x.view( - batch_size, channels, frames, - height // patch_size, patch_size, - width // patch_size, patch_size, + batch_size, + channels, + frames, + height // patch_size, + patch_size, + width // patch_size, + patch_size, ) x = x.permute(0, 1, 6, 4, 2, 3, 5).contiguous() - x = x.view( - batch_size, channels * patch_size * patch_size, frames, - height // patch_size, width // patch_size, + return x.view( + batch_size, + channels * patch_size * patch_size, + frames, + height // patch_size, + width // patch_size, ) - return x class WanVAEStreamingWrapper: + """Keep independent causal encoder state around a shared Wan VAE.""" - def __init__(self, vae_model): + def __init__(self, vae_model: nn.Module) -> None: + """ + Args: + vae_model: Wan VAE whose encoder and quantization projection are used. + """ self.vae = vae_model self.encoder = vae_model.encoder self.quant_conv = vae_model.quant_conv @@ -69,115 +201,35 @@ def __init__(self, vae_model): if hasattr(self.vae, "_cached_conv_counts"): self.enc_conv_num = self.vae._cached_conv_counts["encoder"] else: - count = 0 - for m in self.encoder.modules(): - if m.__class__.__name__ == "WanCausalConv3d": - count += 1 - self.enc_conv_num = count - + self.enc_conv_num = sum( + module.__class__.__name__ == "WanCausalConv3d" + for module in self.encoder.modules() + ) self.clear_cache() - def clear_cache(self): - self.feat_cache = [None] * self.enc_conv_num + def clear_cache(self) -> None: + """Discard causal encoder features from the previous observation.""" + self.feat_cache: list[Tensor | None] = [None] * self.enc_conv_num - def encode_chunk(self, x_chunk: torch.Tensor) -> torch.Tensor: - if ( - hasattr(self.vae.config, "patch_size") - and self.vae.config.patch_size is not None - ): - x_chunk = patchify(x_chunk, self.vae.config.patch_size) + def encode_chunk(self, x_chunk: Tensor) -> Tensor: + """Encode one observation chunk while advancing causal feature state.""" + patch_size = getattr(self.vae.config, "patch_size", None) + if patch_size is not None: + x_chunk = patchify(x_chunk, patch_size) feat_idx = [0] - out = self.encoder(x_chunk, feat_cache=self.feat_cache, feat_idx=feat_idx) - enc = self.quant_conv(out) - return enc - - -# --------------------------------------------------------------------------- -# Upstream FlowMatchScheduler (wan_va/utils/scheduler.py) -# --------------------------------------------------------------------------- - - -class FlowMatchScheduler: - - def __init__( - self, - num_inference_steps=100, - num_train_timesteps=1000, - shift=3.0, - sigma_max=1.0, - sigma_min=0.003 / 1.002, - inverse_timesteps=False, - extra_one_step=False, - reverse_sigmas=False, - exponential_shift=False, - exponential_shift_mu=None, - shift_terminal=None, - ): - self.num_train_timesteps = num_train_timesteps - self.shift = shift - self.sigma_max = sigma_max - self.sigma_min = sigma_min - self.inverse_timesteps = inverse_timesteps - self.extra_one_step = extra_one_step - self.reverse_sigmas = reverse_sigmas - self.exponential_shift = exponential_shift - self.exponential_shift_mu = exponential_shift_mu - self.shift_terminal = shift_terminal - self.set_timesteps(num_inference_steps) - - def set_timesteps(self, num_inference_steps=100, denoising_strength=1.0, - training=False, shift=None, dynamic_shift_len=None): - if shift is not None: - self.shift = shift - sigma_start = self.sigma_min + (self.sigma_max - self.sigma_min) * denoising_strength - if self.extra_one_step: - self.sigmas = torch.linspace(sigma_start, self.sigma_min, - num_inference_steps + 1)[:-1] - else: - self.sigmas = torch.linspace(sigma_start, self.sigma_min, - num_inference_steps) - if self.inverse_timesteps: - self.sigmas = torch.flip(self.sigmas, dims=[0]) - if self.exponential_shift: - mu = (self.calculate_shift(dynamic_shift_len) - if dynamic_shift_len is not None else self.exponential_shift_mu) - self.sigmas = math.exp(mu) / (math.exp(mu) + (1 / self.sigmas - 1)) - else: - self.sigmas = self.shift * self.sigmas / (1 + (self.shift - 1) * self.sigmas) - if self.shift_terminal is not None: - one_minus_z = 1 - self.sigmas - scale_factor = one_minus_z[-1] / (1 - self.shift_terminal) - self.sigmas = 1 - (one_minus_z / scale_factor) - if self.reverse_sigmas: - self.sigmas = 1 - self.sigmas - self.timesteps = self.sigmas * self.num_train_timesteps - - def step(self, model_output, timestep, sample, to_final=False, **kwargs): - if isinstance(timestep, torch.Tensor): - timestep = timestep.cpu() - timestep_id = torch.argmin((self.timesteps - timestep).abs()) - sigma = self.sigmas[timestep_id] - if to_final or timestep_id + 1 >= len(self.timesteps): - sigma_ = 1 if (self.inverse_timesteps or self.reverse_sigmas) else 0 - else: - sigma_ = self.sigmas[timestep_id + 1] - prev_sample = sample + model_output * (sigma_ - sigma) - return prev_sample - - def add_noise(self, original_samples, noise, timestep, t_dim=2): - if isinstance(timestep, torch.Tensor): - timestep = timestep.cpu() - timestep = timestep[None] - timestep_id = torch.argmin((self.timesteps[:, None] - timestep).abs(), dim=0) - shape = [1] * noise.ndim - shape[t_dim] = timestep_id.shape[0] - sigma = self.sigmas[timestep_id].to(original_samples).view(shape) - sample = (1 - sigma) * original_samples + sigma * noise - return sample - - def calculate_shift(self, image_seq_len, base_seq_len=256, - max_seq_len=8192, base_shift=0.5, max_shift=0.9): - m = (max_shift - base_shift) / (max_seq_len - base_seq_len) - b = base_shift - m * base_seq_len - mu = image_seq_len * m + b - return mu + output = self.encoder( + x_chunk, + feat_cache=self.feat_cache, + feat_idx=feat_idx, + ) + return self.quant_conv(output) + + +__all__ = [ + "WanVAEStreamingWrapper", + "load_text_encoder", + "load_tokenizer", + "load_vae", + "resolve_checkpoint_root", + "validate_checkpoint_root", +] diff --git a/integrations/lingbot_va/lingbot_va/action.py b/integrations/lingbot_va/lingbot_va/action.py index a2782cf40..c35fbb7d3 100644 --- a/integrations/lingbot_va/lingbot_va/action.py +++ b/integrations/lingbot_va/lingbot_va/action.py @@ -40,6 +40,22 @@ class LingbotVAActionProcessorConfig: q99: tuple[float, ...] = ROBOTWIN_ACTION_Q99 norm_method: str = "quantiles" + def __post_init__(self) -> None: + """Reject inconsistent action schemas before tensor processing starts.""" + if self.action_dim <= 0: + raise ValueError("action_dim must be positive") + if len(self.q01) != self.action_dim or len(self.q99) != self.action_dim: + raise ValueError("q01 and q99 must each contain action_dim values") + if len(set(self.used_action_channel_ids)) != len(self.used_action_channel_ids): + raise ValueError("used_action_channel_ids must be unique") + if any( + channel < 0 or channel >= self.action_dim + for channel in self.used_action_channel_ids + ): + raise ValueError("used_action_channel_ids must be within action_dim") + if self.norm_method != "quantiles": + raise ValueError(f"unsupported norm_method: {self.norm_method!r}") + @dataclass class LingbotVAActionProcessor: @@ -62,10 +78,18 @@ def action_mask(self, *, device: torch.device | None = None) -> Tensor: return mask def q01_tensor(self, *, device: torch.device | None = None) -> Tensor: - return torch.tensor(self.config.q01, dtype=torch.float32, device=device).reshape(-1, 1, 1) + return torch.tensor( + self.config.q01, + dtype=torch.float32, + device=device, + ).reshape(-1, 1, 1) def q99_tensor(self, *, device: torch.device | None = None) -> Tensor: - return torch.tensor(self.config.q99, dtype=torch.float32, device=device).reshape(-1, 1, 1) + return torch.tensor( + self.config.q99, + dtype=torch.float32, + device=device, + ).reshape(-1, 1, 1) def preprocess(self, action: Tensor) -> Tensor: """Normalize a raw action tensor of shape ``[C_used_or_full, F, H]``. @@ -75,8 +99,17 @@ def preprocess(self, action: Tensor) -> Tensor: quantile-normalize to roughly ``[-1, 1]``, then return ``[1, action_dim, F, H, 1]``. """ - assert action.ndim == 3, f"expected [C, F, H], got {tuple(action.shape)}" - padded = torch.nn.functional.pad(action, [0, 0, 0, 0, 0, 1], mode="constant", value=0) + expected_channels = len(self.config.used_action_channel_ids) + if action.ndim != 3 or action.shape[0] != expected_channels: + raise ValueError( + f"expected [{expected_channels}, F, H], got {tuple(action.shape)}" + ) + padded = torch.nn.functional.pad( + action, + [0, 0, 0, 0, 0, 1], + mode="constant", + value=0, + ) expanded = padded[list(self.inverse_used_action_channel_ids)] if self.config.norm_method != "quantiles": raise NotImplementedError(self.config.norm_method) @@ -98,13 +131,25 @@ def postprocess(self, action: Tensor) -> Tensor: action: Tensor with shape ``[B, 30, F, H, 1]``. Returns: - Tensor with shape ``[len(used_action_channel_ids), F, H]``. + Tensor with shape ``[F * H, len(used_action_channel_ids)]``. """ - assert action.ndim == 5, f"expected [B, C, F, H, W], got {tuple(action.shape)}" + if ( + action.ndim != 5 + or action.shape[0] != 1 + or action.shape[1] != self.config.action_dim + or action.shape[-1] != 1 + ): + raise ValueError( + f"expected [1, {self.config.action_dim}, F, H, 1], " + f"got {tuple(action.shape)}" + ) action_cpu = action[0, ..., 0].detach().cpu() if self.config.norm_method != "quantiles": raise NotImplementedError(self.config.norm_method) q01 = self.q01_tensor() q99 = self.q99_tensor() denorm = (action_cpu + 1.0) / 2.0 * (q99 - q01 + 1e-6) + q01 - return denorm[list(self.config.used_action_channel_ids)] + selected = denorm[list(self.config.used_action_channel_ids)] + return selected.permute(1, 2, 0).reshape( + -1, len(self.config.used_action_channel_ids) + ) diff --git a/integrations/lingbot_va/tests/test_action.py b/integrations/lingbot_va/tests/test_action.py new file mode 100644 index 000000000..d42079ad5 --- /dev/null +++ b/integrations/lingbot_va/tests/test_action.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU tests for the Robotwin action tensor boundary.""" + +import pytest +import torch + +from lingbot_va.action import ( + LingbotVAActionProcessor, + LingbotVAActionProcessorConfig, +) + +pytestmark = pytest.mark.ci_cpu + + +def test_action_mask_selects_only_robotwin_channels() -> None: + processor = LingbotVAActionProcessor() + + mask = processor.action_mask() + + assert mask.dtype == torch.bool + assert mask.shape == (processor.config.action_dim,) + assert mask.sum().item() == len(processor.config.used_action_channel_ids) + + +def test_action_preprocess_postprocess_round_trip() -> None: + processor = LingbotVAActionProcessor() + channel_count = len(processor.config.used_action_channel_ids) + raw = torch.linspace(-0.25, 0.25, channel_count * 2 * 3).reshape( + channel_count, + 2, + 3, + ) + + model_action = processor.preprocess(raw) + restored = processor.postprocess(model_action) + + assert model_action.shape == (1, processor.config.action_dim, 2, 3, 1) + assert restored.shape == (6, channel_count) + torch.testing.assert_close(restored, raw.permute(1, 2, 0).reshape(6, channel_count)) + + +def test_action_processor_rejects_inconsistent_schema() -> None: + with pytest.raises(ValueError, match="q01 and q99"): + LingbotVAActionProcessorConfig(action_dim=2, q01=(0.0,), q99=(1.0, 1.0)) + + with pytest.raises(ValueError, match="unique"): + LingbotVAActionProcessorConfig(used_action_channel_ids=(0, 0)) + + +def test_action_processor_rejects_wrong_input_shape() -> None: + processor = LingbotVAActionProcessor() + + with pytest.raises(ValueError, match="expected"): + processor.preprocess(torch.zeros(15, 2, 3)) + + with pytest.raises(ValueError, match="expected"): + processor.postprocess(torch.zeros(2, 30, 2, 3, 1)) diff --git a/integrations/lingbot_va/tests/test_loaders.py b/integrations/lingbot_va/tests/test_loaders.py new file mode 100644 index 000000000..f14bddeb1 --- /dev/null +++ b/integrations/lingbot_va/tests/test_loaders.py @@ -0,0 +1,96 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU tests for local and Hugging Face LingBot checkpoint resolution.""" + +from pathlib import Path +from typing import Any + +import huggingface_hub +import pytest + +import lingbot_va._loaders as loaders + +pytestmark = pytest.mark.ci_cpu + +_COMPONENTS = ("transformer", "vae", "text_encoder", "tokenizer") + + +def _checkpoint_root(path: Path) -> Path: + path.mkdir() + for component in _COMPONENTS: + (path / component).mkdir() + return path + + +def test_validate_checkpoint_root_accepts_complete_local_snapshot( + tmp_path: Path, +) -> None: + checkpoint_root = _checkpoint_root(tmp_path / "checkpoint") + + assert loaders.validate_checkpoint_root(checkpoint_root) == checkpoint_root + + +def test_validate_checkpoint_root_names_missing_component(tmp_path: Path) -> None: + checkpoint_root = _checkpoint_root(tmp_path / "checkpoint") + (checkpoint_root / "tokenizer").rmdir() + + with pytest.raises(FileNotFoundError, match="tokenizer"): + loaders.validate_checkpoint_root(checkpoint_root) + + +def test_resolve_local_checkpoint_never_downloads( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + checkpoint_root = _checkpoint_root(tmp_path / "checkpoint") + + def unexpected_download(*args: Any, **kwargs: Any) -> None: + raise AssertionError((args, kwargs)) + + monkeypatch.setattr(loaders, "maybe_download_hf_repo_on_rank0", unexpected_download) + monkeypatch.setattr(huggingface_hub, "snapshot_download", unexpected_download) + + assert loaders.resolve_checkpoint_root(checkpoint_root) == checkpoint_root + + +def test_resolve_remote_checkpoint_propagates_revision( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + checkpoint_root = _checkpoint_root(tmp_path / "snapshot") + calls: list[tuple[str, dict[str, Any]]] = [] + + def record_preload(repo_id: str, **kwargs: Any) -> None: + calls.append((repo_id, kwargs)) + + def record_snapshot(**kwargs: Any) -> str: + calls.append(("snapshot_download", kwargs)) + return str(checkpoint_root) + + monkeypatch.setattr(loaders, "maybe_download_hf_repo_on_rank0", record_preload) + monkeypatch.setattr(huggingface_hub, "snapshot_download", record_snapshot) + + resolved = loaders.resolve_checkpoint_root("owner/repo", revision="deadbeef") + + assert resolved == checkpoint_root + assert calls[0][0] == "owner/repo" + assert calls[0][1]["revision"] == "deadbeef" + assert calls[1][1]["repo_id"] == "owner/repo" + assert calls[1][1]["revision"] == "deadbeef" + assert calls[1][1]["local_files_only"] is True + + +def test_resolve_nonexistent_explicit_path_fails_without_download( + tmp_path: Path, + monkeypatch: pytest.MonkeyPatch, +) -> None: + missing = tmp_path / "missing" + + def unexpected_download(*args: Any, **kwargs: Any) -> None: + raise AssertionError((args, kwargs)) + + monkeypatch.setattr(loaders, "maybe_download_hf_repo_on_rank0", unexpected_download) + + with pytest.raises(FileNotFoundError, match="does not exist"): + loaders.resolve_checkpoint_root(missing) From 6ed272b85b015d86a0b2fd5455a0254331be0502 Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Tue, 25 Aug 2026 14:43:57 -0700 Subject: [PATCH 04/18] Extract LingBot one-run inference engine Signed-off-by: Jonathan McCaffrey --- integrations/lingbot_va/lingbot_va/config.py | 27 - integrations/lingbot_va/lingbot_va/engine.py | 620 ++++++++++++++++++ .../lingbot_va/lingbot_va/pipeline.py | 1 - integrations/lingbot_va/lingbot_va/runner.py | 503 -------------- .../lingbot_va/transformer/impl/modules.py | 4 - .../lingbot_va/transformer/impl/network.py | 4 +- integrations/lingbot_va/pyproject.toml | 11 +- integrations/lingbot_va/tests/test_engine.py | 135 ++++ uv.lock | 31 + 9 files changed, 792 insertions(+), 544 deletions(-) create mode 100644 integrations/lingbot_va/lingbot_va/engine.py delete mode 100644 integrations/lingbot_va/lingbot_va/runner.py create mode 100644 integrations/lingbot_va/tests/test_engine.py diff --git a/integrations/lingbot_va/lingbot_va/config.py b/integrations/lingbot_va/lingbot_va/config.py index f3324162d..0605ee2ec 100644 --- a/integrations/lingbot_va/lingbot_va/config.py +++ b/integrations/lingbot_va/lingbot_va/config.py @@ -18,10 +18,8 @@ from __future__ import annotations from flashdreams.infra.diffusion.model import DiffusionModelConfig -from flashdreams.infra.runner import RunnerConfig from lingbot_va.constants import ( DEFAULT_CHECKPOINT_ROOT, - DEFAULT_OUTPUT_DIR, ROBOTWIN_ACTION_INFERENCE_STEPS, ROBOTWIN_ACTION_PER_FRAME, ROBOTWIN_ACTION_SNR_SHIFT, @@ -39,7 +37,6 @@ RUNNER_NAME_ROBOTWIN_I2AV, ) from lingbot_va.pipeline import LingbotVAInferencePipelineConfig -from lingbot_va.runner import LingbotVARobotwinRunnerConfig from lingbot_va.scheduler import LingbotVAFlowMatchSchedulerConfig from lingbot_va.transformer import LingbotVATransformerConfig @@ -81,30 +78,6 @@ shift=ROBOTWIN_ACTION_SNR_SHIFT, ), ) -"""Robotwin I2AV pipeline config shell. - -This config is intentionally no-instantiate safe; GPU inference remains gated -until the native LingBot-VA DiT/VAE path is ported. -""" - -RUNNER_LINGBOT_VA_ROBOTWIN_I2AV = LingbotVARobotwinRunnerConfig( - runner_name=PIPELINE_LINGBOT_VA_ROBOTWIN_I2AV.name, - description=( - "LingBot-VA Robotwin I2AV inference scaffold " - "(three-camera Robotwin config; native DiT port pending)." - ), - pipeline=PIPELINE_LINGBOT_VA_ROBOTWIN_I2AV, - output_dir=DEFAULT_OUTPUT_DIR, - checkpoint_root=DEFAULT_CHECKPOINT_ROOT, - num_inference_steps=ROBOTWIN_VIDEO_INFERENCE_STEPS, - action_num_inference_steps=ROBOTWIN_ACTION_INFERENCE_STEPS, - snr_shift=ROBOTWIN_SNR_SHIFT, - action_snr_shift=ROBOTWIN_ACTION_SNR_SHIFT, -) - PIPELINE_CONFIGS: dict[str, LingbotVAInferencePipelineConfig] = { PIPELINE_LINGBOT_VA_ROBOTWIN_I2AV.name: PIPELINE_LINGBOT_VA_ROBOTWIN_I2AV, } -RUNNER_CONFIGS: dict[str, RunnerConfig] = { - RUNNER_LINGBOT_VA_ROBOTWIN_I2AV.runner_name: RUNNER_LINGBOT_VA_ROBOTWIN_I2AV, -} diff --git a/integrations/lingbot_va/lingbot_va/engine.py b/integrations/lingbot_va/lingbot_va/engine.py new file mode 100644 index 000000000..5cd5244f8 --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/engine.py @@ -0,0 +1,620 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Session-owned LingBot-VA Robotwin inference engine.""" + +from __future__ import annotations + +import gc +import html +import re +import time +from collections.abc import Mapping +from dataclasses import dataclass +from enum import Enum, auto +from pathlib import Path +from typing import Any + +import numpy as np +import torch +import torch.nn.functional as F +from PIL import Image +from torch import Tensor + +from flashdreams.infra.config import derive_config +from lingbot_va._loaders import ( + WanVAEStreamingWrapper, + load_text_encoder, + load_tokenizer, + load_vae, + resolve_checkpoint_root, +) +from lingbot_va.action import LingbotVAActionProcessor +from lingbot_va.config import PIPELINE_LINGBOT_VA_ROBOTWIN_I2AV +from lingbot_va.constants import ( + DEFAULT_CHECKPOINT_ROOT, + DEFAULT_INPUT_IMAGE_DIR, + DEFAULT_PROMPT, + ROBOTWIN_ACTION_GUIDANCE_SCALE, + ROBOTWIN_ACTION_INFERENCE_STEPS, + ROBOTWIN_ACTION_SNR_SHIFT, + ROBOTWIN_FRAME_CHUNK_SIZE, + ROBOTWIN_GUIDANCE_SCALE, + ROBOTWIN_HEIGHT, + ROBOTWIN_OBS_CAM_KEYS, + ROBOTWIN_SNR_SHIFT, + ROBOTWIN_VIDEO_INFERENCE_STEPS, + ROBOTWIN_WIDTH, +) +from lingbot_va.pipeline import LingbotVAInferencePipelineConfig +from lingbot_va.utils import resolve_prompt + + +@dataclass(frozen=True, slots=True, kw_only=True) +class LingbotVAEngineConfig: + """Resolved model and input settings for one destructive engine run.""" + + checkpoint_root: str | Path = DEFAULT_CHECKPOINT_ROOT + checkpoint_revision: str | None = None + input_image_dir: Path = DEFAULT_INPUT_IMAGE_DIR + prompt: str | Path = DEFAULT_PROMPT + num_chunks: int = 10 + seed: int = 42 + device: str = "cuda:0" + dtype: torch.dtype = torch.bfloat16 + enable_offload: bool = False + compile_network: bool = True + guidance_scale: float = ROBOTWIN_GUIDANCE_SCALE + action_guidance_scale: float = ROBOTWIN_ACTION_GUIDANCE_SCALE + video_inference_steps: int = ROBOTWIN_VIDEO_INFERENCE_STEPS + action_inference_steps: int = ROBOTWIN_ACTION_INFERENCE_STEPS + video_snr_shift: float = ROBOTWIN_SNR_SHIFT + action_snr_shift: float = ROBOTWIN_ACTION_SNR_SHIFT + + def __post_init__(self) -> None: + """Reject invalid user-controlled settings before loading weights.""" + if not str(self.checkpoint_root): + raise ValueError("checkpoint_root must not be empty") + if self.num_chunks <= 0: + raise ValueError("num_chunks must be positive") + if self.video_inference_steps <= 0 or self.action_inference_steps <= 0: + raise ValueError("inference step counts must be positive") + if self.video_snr_shift <= 0 or self.action_snr_shift <= 0: + raise ValueError("SNR shifts must be positive") + if self.guidance_scale < 0 or self.action_guidance_scale < 0: + raise ValueError("guidance scales must be non-negative") + + +@dataclass(frozen=True, slots=True) +class LingbotVAEngineOutput: + """CPU outputs from one complete Robotwin rollout.""" + + video: Tensor + """Decoded video in ``[T, C, H, W]`` layout and the ``[-1, 1]`` range.""" + + actions: Tensor + """Denormalized actions in ``[step, channel]`` layout.""" + + metrics: Mapping[str, float] + """Measured phase durations and peak allocated CUDA bytes.""" + + +class LingbotVAEngineState(Enum): + """Legal lifecycle states for a one-run engine.""" + + NEW = auto() + RUNNING = auto() + FINISHED = auto() + CLOSED = auto() + + +def required_image_paths(input_image_dir: Path) -> dict[str, Path]: + """Return the canonical Robotwin camera-key-to-PNG mapping.""" + return {key: input_image_dir / f"{key}.png" for key in ROBOTWIN_OBS_CAM_KEYS} + + +def validate_input_images(input_image_dir: Path) -> dict[str, Path]: + """Validate all three Robotwin camera PNGs without loading model state.""" + image_paths = required_image_paths(input_image_dir) + missing = [path for path in image_paths.values() if not path.is_file()] + if missing: + raise FileNotFoundError( + "Missing Robotwin camera PNGs: " + ", ".join(str(path) for path in missing) + ) + return image_paths + + +def validate_device(device_name: str) -> torch.device: + """Resolve a requested device and fail early when CUDA is unavailable.""" + device = torch.device(device_name) + if device.type == "cuda": + if not torch.cuda.is_available(): + raise RuntimeError( + f"CUDA device requested but CUDA is unavailable: {device}" + ) + if device.index is not None and device.index >= torch.cuda.device_count(): + raise ValueError( + f"CUDA device index {device.index} is unavailable; " + f"found {torch.cuda.device_count()} device(s)." + ) + return device + + +def build_pipeline_config( + config: LingbotVAEngineConfig, + checkpoint_root: Path, +) -> LingbotVAInferencePipelineConfig: + """Apply every effective engine override to a copied pipeline config.""" + return derive_config( + PIPELINE_LINGBOT_VA_ROBOTWIN_I2AV, + checkpoint_root=str(checkpoint_root), + enable_sync_and_profile=False, + diffusion_model={ + "seed": config.seed, + "transformer": { + "checkpoint_root": str(checkpoint_root), + "dtype": config.dtype, + "compile_network": config.compile_network, + "guidance_scale": config.guidance_scale, + "action_guidance_scale": config.action_guidance_scale, + }, + "scheduler": { + "num_inference_steps": config.video_inference_steps, + "shift": config.video_snr_shift, + }, + }, + action_scheduler={ + "num_inference_steps": config.action_inference_steps, + "shift": config.action_snr_shift, + }, + ) + + +def _prompt_clean(text: str) -> str: + """Apply the upstream double HTML decode and whitespace cleanup.""" + try: + import ftfy + + text = ftfy.fix_text(text) + except ImportError: + pass + return re.sub(r"\s+", " ", html.unescape(html.unescape(text))).strip() + + +class LingbotVAEngine: + """Own model components for exactly one complete LingBot-VA rollout. + + Video decoding requires releasing the denoising pipeline first. Therefore + this object cannot be reused after :meth:`run`; a session reset creates a + new engine. + """ + + def __init__(self, config: LingbotVAEngineConfig) -> None: + """ + Args: + config: Immutable settings for the rollout. + """ + self.config = config + self._state = LingbotVAEngineState.NEW + self._device: torch.device | None = None + self._pipeline: Any | None = None + self._pipeline_cache: Any | None = None + self._vae: Any | None = None + self._text_encoder: Any | None = None + self._tokenizer: Any | None = None + self._streaming_vae: WanVAEStreamingWrapper | None = None + self._streaming_vae_half: WanVAEStreamingWrapper | None = None + + @property + def state(self) -> LingbotVAEngineState: + """Return the current one-run lifecycle state.""" + return self._state + + def run(self) -> LingbotVAEngineOutput: + """Run one rollout and return decoded video, actions, and metrics.""" + if self._state is not LingbotVAEngineState.NEW: + raise RuntimeError( + f"LingbotVAEngine.run() requires NEW state, got {self._state.name}." + ) + self._state = LingbotVAEngineState.RUNNING + try: + output = self._run_impl() + except BaseException: + try: + self.close() + except Exception: + pass + raise + self._state = LingbotVAEngineState.FINISHED + return output + + def _run_impl(self) -> LingbotVAEngineOutput: + """Execute the real model path after lifecycle validation.""" + started = time.perf_counter() + image_paths = validate_input_images(self.config.input_image_dir) + prompt = resolve_prompt(self.config.prompt) + device = validate_device(self.config.device) + checkpoint_root = resolve_checkpoint_root( + self.config.checkpoint_root, + revision=self.config.checkpoint_revision, + ) + self._device = device + + if device.type == "cuda": + torch.cuda.reset_peak_memory_stats(device) + torch.manual_seed(self.config.seed) + self._load_models(checkpoint_root, device) + + prompt_started = time.perf_counter() + prompt_embeds, negative_prompt_embeds = self._prepare_prompt(prompt, device) + prompt_seconds = time.perf_counter() - prompt_started + + observation_started = time.perf_counter() + init_latent = self._prepare_observation(image_paths, device) + observation_seconds = time.perf_counter() - observation_started + + denoise_started = time.perf_counter() + latents, actions = self._generate_chunks( + prompt_embeds, + negative_prompt_embeds, + init_latent, + device, + ) + denoise_seconds = time.perf_counter() - denoise_started + + del prompt_embeds, negative_prompt_embeds, init_latent + self._release_denoising_state() + + decode_started = time.perf_counter() + video = self._decode_video(latents, device) + decode_seconds = time.perf_counter() - decode_started + self._release_vae() + + peak_allocated = 0.0 + if device.type == "cuda": + peak_allocated = float(torch.cuda.max_memory_allocated(device)) + return LingbotVAEngineOutput( + video=video, + actions=actions, + metrics={ + "prompt_encode_seconds": prompt_seconds, + "observation_encode_seconds": observation_seconds, + "denoise_seconds": denoise_seconds, + "decode_seconds": decode_seconds, + "total_seconds": time.perf_counter() - started, + "peak_allocated_bytes": peak_allocated, + }, + ) + + def _load_models(self, checkpoint_root: Path, device: torch.device) -> None: + """Load shared VAE, text components, and the native transformer.""" + component_device = torch.device("cpu") if self.config.enable_offload else device + self._vae = load_vae(checkpoint_root, self.config.dtype, component_device) + self._streaming_vae = WanVAEStreamingWrapper(self._vae) + self._streaming_vae_half = WanVAEStreamingWrapper(self._vae) + self._tokenizer = load_tokenizer(checkpoint_root) + self._text_encoder = load_text_encoder( + checkpoint_root, + self.config.dtype, + component_device, + ) + pipeline_config = build_pipeline_config(self.config, checkpoint_root) + self._pipeline = pipeline_config.setup() + self._pipeline.transformer.load_model(device) + + def _prepare_prompt( + self, + prompt: str, + device: torch.device, + ) -> tuple[Tensor, Tensor]: + """Encode positive and, when required, negative text embeddings.""" + if self._text_encoder is None: + raise RuntimeError("text encoder is not loaded") + if self.config.enable_offload: + self._text_encoder.to(device) + positive = self._encode_prompt(prompt, device) + use_cfg = ( + self.config.guidance_scale > 1.0 or self.config.action_guidance_scale > 1.0 + ) + negative = self._encode_prompt("", device) if use_cfg else positive + if self.config.enable_offload: + self._text_encoder.to("cpu") + self._empty_cuda_cache() + return positive, negative + + def _encode_prompt(self, prompt: str, device: torch.device) -> Tensor: + """Encode one cleaned prompt with the upstream 512-token contract.""" + if self._tokenizer is None or self._text_encoder is None: + raise RuntimeError("text components are not loaded") + text_inputs = self._tokenizer( + [_prompt_clean(prompt)], + padding="max_length", + max_length=512, + truncation=True, + add_special_tokens=True, + return_attention_mask=True, + return_tensors="pt", + ) + ids = text_inputs.input_ids + mask = text_inputs.attention_mask + sequence_lengths = mask.gt(0).sum(dim=1).long() + encoder_device = next(self._text_encoder.parameters()).device + embeddings = self._text_encoder( + ids.to(encoder_device), + mask.to(encoder_device), + ).last_hidden_state + embeddings = embeddings.to(dtype=self.config.dtype, device=device) + trimmed = [item[:length] for item, length in zip(embeddings, sequence_lengths)] + padded = [ + torch.cat( + [item, item.new_zeros(512 - item.size(0), item.size(1))], + dim=0, + ) + for item in trimmed + ] + return torch.stack(padded, dim=0) + + def _prepare_observation( + self, + image_paths: Mapping[str, Path], + device: torch.device, + ) -> Tensor: + """Load and encode the high and two wrist camera observations.""" + if self._vae is None: + raise RuntimeError("VAE is not loaded") + if self.config.enable_offload: + self._vae.to(device) + images = { + key: np.asarray(Image.open(path).convert("RGB")) + for key, path in image_paths.items() + } + output = self._encode_observation(images, device) + if self.config.enable_offload: + self._vae.to("cpu") + self._empty_cuda_cache() + return output + + def _encode_observation( + self, + observation_images: Mapping[str, np.ndarray], + device: torch.device, + ) -> Tensor: + """Encode the three-camera Robotwin spatial arrangement.""" + if ( + self._vae is None + or self._streaming_vae is None + or self._streaming_vae_half is None + ): + raise RuntimeError("VAE streaming wrappers are not loaded") + videos = [] + for camera_index, key in enumerate(ROBOTWIN_OBS_CAM_KEYS): + if camera_index == 0: + height, width = ROBOTWIN_HEIGHT, ROBOTWIN_WIDTH + else: + height, width = ROBOTWIN_HEIGHT // 2, ROBOTWIN_WIDTH // 2 + image = ( + torch.from_numpy(observation_images[key].copy()) + .float() + .permute(2, 0, 1) + .unsqueeze(1) + ) + image = F.interpolate( + image, + size=(height, width), + mode="bilinear", + align_corners=False, + ) + videos.append(image.unsqueeze(0)) + + high_video = videos[0] / 255.0 * 2.0 - 1.0 + wrist_video = torch.cat(videos[1:], dim=0) / 255.0 * 2.0 - 1.0 + vae_device = next(self._vae.parameters()).device + encoded_high = self._streaming_vae.encode_chunk( + high_video.to(device=vae_device, dtype=self.config.dtype) + ) + encoded_wrist = self._streaming_vae_half.encode_chunk( + wrist_video.to(device=vae_device, dtype=self.config.dtype) + ) + encoded = torch.cat( + [ + torch.cat(encoded_wrist.split(1, dim=0), dim=-1), + encoded_high, + ], + dim=-2, + ) + mean, _ = torch.chunk(encoded, 2, dim=1) + latent_mean = torch.as_tensor( + self._vae.config.latents_mean, + device=mean.device, + ).view(1, -1, 1, 1, 1) + latent_inverse_std = ( + 1.0 + / torch.as_tensor( + self._vae.config.latents_std, + device=mean.device, + ) + ).view(1, -1, 1, 1, 1) + normalized = ((mean.float() - latent_mean) * latent_inverse_std).to(mean) + return normalized.to(device) + + def _generate_chunks( + self, + prompt_embeddings: Tensor, + negative_prompt_embeddings: Tensor, + init_latent: Tensor, + device: torch.device, + ) -> tuple[Tensor, Tensor]: + """Generate all video/action chunks before destructive decode teardown.""" + if self._pipeline is None: + raise RuntimeError("pipeline is not loaded") + use_cfg = ( + self.config.guidance_scale > 1.0 or self.config.action_guidance_scale > 1.0 + ) + self._pipeline_cache = self._pipeline.initialize_cache( + text_embeddings=prompt_embeddings, + negative_text_embeddings=(negative_prompt_embeddings if use_cfg else None), + batch_size=1, + ) + action_processor = LingbotVAActionProcessor() + action_mask = action_processor.action_mask(device=device) + predicted_latents: list[Tensor] = [] + predicted_actions: list[Tensor] = [] + for chunk_index in range(self.config.num_chunks): + output = self._pipeline.generate( + autoregressive_index=chunk_index, + cache=self._pipeline_cache, + input={ + "init_latent": init_latent, + "action_mask": action_mask, + "device": device, + "dtype": self.config.dtype, + }, + ) + predicted_latents.append(output.latent.detach().cpu()) + predicted_actions.append(action_processor.postprocess(output.action)) + del output + self._empty_cuda_cache() + return ( + torch.cat(predicted_latents, dim=2), + torch.cat(predicted_actions, dim=0), + ) + + def _release_denoising_state(self) -> None: + """Release cache, DiT, text, and streaming encoder state before decode.""" + cache = self._pipeline_cache + if cache is not None: + transformer_cache = getattr(cache, "transformer_cache", None) + if transformer_cache is not None: + for network_cache in ( + getattr(transformer_cache, "network_cache", None), + getattr(transformer_cache, "network_cache_uncond", None), + ): + if network_cache is None: + continue + for block_cache in network_cache.block_caches: + block_cache.self_attn.reset() + block_cache.cross_attn.text.k = torch.empty(0) + block_cache.cross_attn.text.v = torch.empty(0) + self._pipeline_cache = None + + for wrapper in (self._streaming_vae, self._streaming_vae_half): + if wrapper is not None: + wrapper.clear_cache() + self._streaming_vae = None + self._streaming_vae_half = None + + if self._pipeline is not None: + transformer = self._pipeline.transformer + network = getattr(transformer, "_network", None) + if network is not None: + network.to("cpu") + object.__setattr__(transformer, "_network", None) + self._pipeline = None + if self._text_encoder is not None: + self._text_encoder.to("cpu") + self._text_encoder = None + self._tokenizer = None + gc.collect() + self._empty_cuda_cache() + + def _decode_video(self, latents: Tensor, device: torch.device) -> Tensor: + """Decode accumulated latent frames while offloading each frame to CPU.""" + if self._vae is None: + raise RuntimeError("VAE is not loaded") + vae = self._vae.to(device=device, dtype=self.config.dtype) + latent = latents.to(device=device, dtype=self.config.dtype) + latent_mean = torch.as_tensor( + vae.config.latents_mean, + device=device, + dtype=self.config.dtype, + ).view(1, vae.config.z_dim, 1, 1, 1) + latent_inverse_std = ( + 1.0 + / torch.as_tensor( + vae.config.latents_std, + device=device, + dtype=self.config.dtype, + ) + ).view(1, vae.config.z_dim, 1, 1, 1) + latent = latent / latent_inverse_std + latent_mean + + vae.clear_cache() + decoded_input = vae.post_quant_conv(latent) + del latent + decoded_frames: list[Tensor] = [] + for frame_index in range(decoded_input.shape[2]): + vae._conv_idx = [0] + decoder_kwargs: dict[str, Any] = { + "feat_cache": vae._feat_map, + "feat_idx": vae._conv_idx, + } + if frame_index == 0: + decoder_kwargs["first_chunk"] = True + frame = vae.decoder( + decoded_input[:, :, frame_index : frame_index + 1], + **decoder_kwargs, + ) + decoded_frames.append(frame.detach().cpu()) + del frame + del decoded_input + vae.clear_cache() + self._empty_cuda_cache() + + decoded = torch.cat(decoded_frames, dim=2) + patch_size = getattr(vae.config, "patch_size", None) + if patch_size is not None: + from diffusers.models.autoencoders.autoencoder_kl_wan import unpatchify + + decoded = unpatchify(decoded, patch_size=patch_size) + return decoded[0].clamp(-1.0, 1.0).permute(1, 0, 2, 3).contiguous() + + def _release_vae(self) -> None: + """Release the final model component after decoded tensors reach CPU.""" + if self._vae is not None: + self._vae.to("cpu") + if hasattr(self._vae, "clear_cache"): + self._vae.clear_cache() + self._vae = None + gc.collect() + self._empty_cuda_cache() + + def _empty_cuda_cache(self) -> None: + """Drop unused CUDA allocations when this engine targets CUDA.""" + if self._device is not None and self._device.type == "cuda": + torch.cuda.empty_cache() + + def close(self) -> None: + """Idempotently release partially or fully initialized model state.""" + if self._state is LingbotVAEngineState.CLOSED: + return + try: + self._release_denoising_state() + finally: + try: + self._release_vae() + finally: + self._state = LingbotVAEngineState.CLOSED + + +def expected_output_shape(config: LingbotVAEngineConfig) -> tuple[int, int, int, int]: + """Return the fixed natural decoded video shape for one rollout.""" + return ( + config.num_chunks * ROBOTWIN_FRAME_CHUNK_SIZE, + 3, + ROBOTWIN_HEIGHT, + ROBOTWIN_WIDTH, + ) + + +__all__ = [ + "LingbotVAEngine", + "LingbotVAEngineConfig", + "LingbotVAEngineOutput", + "LingbotVAEngineState", + "build_pipeline_config", + "expected_output_shape", + "required_image_paths", + "validate_device", + "validate_input_images", +] diff --git a/integrations/lingbot_va/lingbot_va/pipeline.py b/integrations/lingbot_va/lingbot_va/pipeline.py index 925beb808..3ba256355 100644 --- a/integrations/lingbot_va/lingbot_va/pipeline.py +++ b/integrations/lingbot_va/lingbot_va/pipeline.py @@ -21,7 +21,6 @@ from typing import Any, NamedTuple import torch -import torch.nn.functional as F from einops import rearrange from torch import Tensor from tqdm import tqdm diff --git a/integrations/lingbot_va/lingbot_va/runner.py b/integrations/lingbot_va/lingbot_va/runner.py deleted file mode 100644 index 74daafe39..000000000 --- a/integrations/lingbot_va/lingbot_va/runner.py +++ /dev/null @@ -1,503 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou -# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Runner for the LingBot-VA Robotwin I2AV integration. - -Uses the native compiled LingBot-VA transformer wrapper for inference -with ``torch.compile`` acceleration. -""" - -from __future__ import annotations - -import gc -import html -import json -import os -import re -import time -from dataclasses import dataclass, field -from pathlib import Path - -import numpy as np -import torch -import torch.nn.functional as F -from loguru import logger -from PIL import Image - -from flashdreams.infra.runner import Runner, RunnerConfig -from lingbot_va.action import LingbotVAActionProcessor -from lingbot_va.constants import ( - DEFAULT_CHECKPOINT_ROOT, - DEFAULT_INPUT_IMAGE_DIR, - DEFAULT_OUTPUT_DIR, - DEFAULT_PROMPT, - ROBOTWIN_ACTION_DIM, - ROBOTWIN_ACTION_GUIDANCE_SCALE, - ROBOTWIN_ACTION_INFERENCE_STEPS, - ROBOTWIN_ACTION_PER_FRAME, - ROBOTWIN_ACTION_SNR_SHIFT, - ROBOTWIN_ATTENTION_WINDOW, - ROBOTWIN_FRAME_CHUNK_SIZE, - ROBOTWIN_GUIDANCE_SCALE, - ROBOTWIN_HEIGHT, - ROBOTWIN_OBS_CAM_KEYS, - ROBOTWIN_PATCH_SIZE, - ROBOTWIN_SNR_SHIFT, - ROBOTWIN_USED_ACTION_CHANNEL_IDS, - ROBOTWIN_VIDEO_INFERENCE_STEPS, - ROBOTWIN_WIDTH, -) -from lingbot_va.pipeline import LingbotVAInferencePipeline -from lingbot_va.utils import resolve_prompt - - -def _prompt_clean(text: str) -> str: - """Clean prompt text (inlined from diffusers prompt_clean).""" - try: - import ftfy - text = ftfy.fix_text(text) - except ImportError: - pass - text = html.unescape(html.unescape(text)) - text = re.sub(r"\s+", " ", text).strip() - return text - - - - -@dataclass(kw_only=True) -class LingbotVARobotwinRunnerConfig(RunnerConfig): - """User-facing config for the LingBot-VA Robotwin I2AV runner.""" - - _target: type["LingbotVARobotwinRunner"] = field( - default_factory=lambda: LingbotVARobotwinRunner - ) - - checkpoint_root: str | Path = DEFAULT_CHECKPOINT_ROOT - input_image_dir: Path = DEFAULT_INPUT_IMAGE_DIR - prompt: str | Path = DEFAULT_PROMPT - num_chunks: int = 10 - save_video: bool = True - save_actions: bool = True - metadata_only: bool = False - seed: int = 42 - enable_offload: bool = False - compile_network: bool = True - benchmark: bool = False - - pixel_height: int = ROBOTWIN_HEIGHT - pixel_width: int = ROBOTWIN_WIDTH - frame_chunk_size: int = ROBOTWIN_FRAME_CHUNK_SIZE - action_dim: int = ROBOTWIN_ACTION_DIM - action_per_frame: int = ROBOTWIN_ACTION_PER_FRAME - attn_window: int = ROBOTWIN_ATTENTION_WINDOW - guidance_scale: float = ROBOTWIN_GUIDANCE_SCALE - action_guidance_scale: float = ROBOTWIN_ACTION_GUIDANCE_SCALE - num_inference_steps: int = ROBOTWIN_VIDEO_INFERENCE_STEPS - action_num_inference_steps: int = ROBOTWIN_ACTION_INFERENCE_STEPS - snr_shift: float = ROBOTWIN_SNR_SHIFT - action_snr_shift: float = ROBOTWIN_ACTION_SNR_SHIFT - patch_size: tuple[int, int, int] = ROBOTWIN_PATCH_SIZE - - -class LingbotVARobotwinRunner( - Runner[LingbotVARobotwinRunnerConfig, LingbotVAInferencePipeline] -): - """Robotwin I2AV driver with full GPU inference.""" - - config: LingbotVARobotwinRunnerConfig - - # ------------------------------------------------------------------ - # helpers - # ------------------------------------------------------------------ - - def _required_image_paths(self) -> dict[str, Path]: - return { - key: self.config.input_image_dir / f"{key}.png" - for key in ROBOTWIN_OBS_CAM_KEYS - } - - def _validate_inputs(self) -> dict[str, Path]: - image_paths = self._required_image_paths() - missing = [p for p in image_paths.values() if not p.exists()] - if missing: - raise FileNotFoundError( - "Missing camera PNGs: " + ", ".join(str(p) for p in missing) - ) - return image_paths - - def _write_metadata_manifest(self) -> Path: - cfg = self.config - prompt = resolve_prompt(cfg.prompt) - image_paths = self._validate_inputs() - cfg.output_dir.mkdir(parents=True, exist_ok=True) - manifest_path = cfg.output_dir / "runtime.json" - manifest = { - "runner_name": cfg.runner_name, - "checkpoint_root": str(cfg.checkpoint_root), - "input_image_dir": str(cfg.input_image_dir), - "image_paths": {k: str(v) for k, v in image_paths.items()}, - "prompt": prompt, - "num_chunks": cfg.num_chunks, - "seed": cfg.seed, - "height": cfg.pixel_height, - "width": cfg.pixel_width, - } - manifest_path.write_text(json.dumps(manifest, indent=2), encoding="utf-8") - return manifest_path - - # ------------------------------------------------------------------ - # model loading - # ------------------------------------------------------------------ - - def _load_models(self, device: torch.device, dtype: torch.dtype): - """Load VAE(s), text encoder, tokenizer, and transformer weights.""" - from lingbot_va._loaders import ( - WanVAEStreamingWrapper, - load_text_encoder, - load_tokenizer, - load_vae, - ) - - cfg = self.config - ckpt = str(cfg.checkpoint_root) - enable_offload = cfg.enable_offload - vae_device = "cpu" if enable_offload else device - - self._vae = load_vae(os.path.join(ckpt, "vae"), dtype, vae_device) - self._streaming_vae = WanVAEStreamingWrapper(self._vae) - - self._streaming_vae_half = WanVAEStreamingWrapper(self._vae) - - self._tokenizer = load_tokenizer(os.path.join(ckpt, "tokenizer")) - self._text_encoder = load_text_encoder( - os.path.join(ckpt, "text_encoder"), dtype, - "cpu" if enable_offload else device, - ) - - # Propagate runner overrides to the pipeline's transformer config - self.pipeline.transformer.config.checkpoint_root = ckpt - self.pipeline.transformer.config.compile_network = cfg.compile_network - self.pipeline.transformer.load_model(device) - - # ------------------------------------------------------------------ - # prompt encoding (from upstream _get_t5_prompt_embeds / encode_prompt) - # ------------------------------------------------------------------ - - def _encode_prompt(self, prompt: str, device: torch.device, dtype: torch.dtype): - prompt = _prompt_clean(prompt) - text_inputs = self._tokenizer( - [prompt], - padding="max_length", - max_length=512, - truncation=True, - add_special_tokens=True, - return_attention_mask=True, - return_tensors="pt", - ) - ids, mask = text_inputs.input_ids, text_inputs.attention_mask - seq_len = mask.gt(0).sum(dim=1).long() - - enc_device = next(self._text_encoder.parameters()).device - embeds = self._text_encoder(ids.to(enc_device), mask.to(enc_device)).last_hidden_state - embeds = embeds.to(dtype=dtype, device=device) - embeds = [u[:v] for u, v in zip(embeds, seq_len)] - embeds = torch.stack([ - torch.cat([u, u.new_zeros(512 - u.size(0), u.size(1))]) for u in embeds - ], dim=0) - return embeds.to(device) - - # ------------------------------------------------------------------ - # observation encoding (from upstream _encode_obs) - # ------------------------------------------------------------------ - - def _encode_obs(self, obs_images: dict[str, np.ndarray], - device: torch.device, dtype: torch.dtype): - cfg = self.config - videos = [] - for k_i, k in enumerate(ROBOTWIN_OBS_CAM_KEYS): - if k_i == 0: - h_i, w_i = cfg.pixel_height, cfg.pixel_width - else: - h_i, w_i = cfg.pixel_height // 2, cfg.pixel_width // 2 - img = torch.from_numpy(obs_images[k]).float().permute(2, 0, 1).unsqueeze(1) - img = F.interpolate(img, size=(h_i, w_i), mode="bilinear", align_corners=False) - videos.append(img.unsqueeze(0)) - - videos_high = videos[0] / 255.0 * 2.0 - 1.0 - videos_wrist = torch.cat(videos[1:], dim=0) / 255.0 * 2.0 - 1.0 - - vae_device = next(self._streaming_vae.vae.parameters()).device - enc_high = self._streaming_vae.encode_chunk(videos_high.to(vae_device).to(dtype)) - enc_wrist = self._streaming_vae_half.encode_chunk(videos_wrist.to(vae_device).to(dtype)) - - enc_out = torch.cat([ - torch.cat(enc_wrist.split(1, dim=0), dim=-1), - enc_high, - ], dim=-2) - - mu, _ = torch.chunk(enc_out, 2, dim=1) - latents_mean = torch.tensor(self._vae.config.latents_mean).to(mu.device) - latents_std = torch.tensor(self._vae.config.latents_std).to(mu.device) - mu_norm = self._normalize_latents(mu, latents_mean, 1.0 / latents_std) - return mu_norm.to(device) - - @staticmethod - def _normalize_latents(latents, mean, std): - mean = mean.view(1, -1, 1, 1, 1).to(latents.device) - std = std.view(1, -1, 1, 1, 1).to(latents.device) - return ((latents.float() - mean) * std).to(latents) - - # ------------------------------------------------------------------ - # main entry - # ------------------------------------------------------------------ - - def run(self) -> None: - cfg = self.config - - if cfg.metadata_only: - mp = self._write_metadata_manifest() - logger.info("Wrote manifest -> {}", mp.resolve()) - return - - prompt = resolve_prompt(cfg.prompt) - image_paths = self._validate_inputs() - - device = torch.device(f"cuda:{self.local_rank}") - dtype = torch.bfloat16 - - # seed - torch.manual_seed(cfg.seed) - - # load models - logger.info("Loading models from {}", cfg.checkpoint_root) - self._load_models(device, dtype) - - # encode prompt - if cfg.benchmark: - torch.cuda.synchronize() - t_pipeline_start = time.perf_counter() - logger.info("Encoding prompt") - use_cfg = (cfg.guidance_scale > 1) or (cfg.action_guidance_scale > 1) - if cfg.enable_offload: - self._text_encoder = self._text_encoder.to(device) - prompt_embeds = self._encode_prompt(prompt, device, dtype) - neg_embeds = self._encode_prompt("", device, dtype) if use_cfg else prompt_embeds - if cfg.enable_offload: - self._text_encoder = self._text_encoder.cpu() - torch.cuda.empty_cache() - - # load observation images - obs_images = { - k: np.array(Image.open(str(p)).convert("RGB")) - for k, p in image_paths.items() - } - - # encode observation - logger.info("Encoding observation") - if cfg.enable_offload: - self._vae.to(device) - init_latent = self._encode_obs(obs_images, device, dtype) - if cfg.enable_offload: - self._vae.cpu() - torch.cuda.empty_cache() - - # initialize pipeline cache - logger.info("Initializing AR cache") - pipeline_cache = self.pipeline.initialize_cache( - text_embeddings=prompt_embeds, - negative_text_embeddings=neg_embeds if use_cfg else None, - batch_size=1, - ) - - # action mask - action_mask = torch.zeros(cfg.action_dim, dtype=torch.bool, device=device) - action_mask[list(ROBOTWIN_USED_ACTION_CHANNEL_IDS)] = True - - # AR loop - cfg.output_dir.mkdir(parents=True, exist_ok=True) - action_processor = LingbotVAActionProcessor() - pred_latents = [] - pred_actions = [] - - if cfg.benchmark: - torch.cuda.synchronize() - t_infer_start = time.perf_counter() - chunk_times = [] - - for chunk_id in range(cfg.num_chunks): - logger.info("Generating chunk {}/{}", chunk_id + 1, cfg.num_chunks) - if cfg.benchmark: - torch.cuda.synchronize() - t_chunk_start = time.perf_counter() - output = self.pipeline.generate( - autoregressive_index=chunk_id, - cache=pipeline_cache, - input={ - "init_latent": init_latent, - "action_mask": action_mask, - "device": device, - "dtype": dtype, - }, - ) - if cfg.benchmark: - torch.cuda.synchronize() - chunk_times.append(time.perf_counter() - t_chunk_start) - post_actions = action_processor.postprocess(output.action) - pred_latents.append(output.latent.detach().cpu()) - pred_actions.append(post_actions) - del output - torch.cuda.empty_cache() - - if cfg.benchmark: - torch.cuda.synchronize() - t_infer_end = time.perf_counter() - infer_elapsed = t_infer_end - t_infer_start - pipeline_elapsed = t_infer_end - t_pipeline_start - - timing_results = { - "method": "flashdreams-lingbot-va", - "num_chunks": cfg.num_chunks, - "total_pipeline_sec": round(pipeline_elapsed, 4), - "total_inference_sec": round(infer_elapsed, 4), - "per_chunk_sec": [round(t, 4) for t in chunk_times], - "avg_chunk_sec": round(sum(chunk_times) / len(chunk_times), 4), - "note": "total_pipeline = prompt_enc + obs_enc + inference; total_inference = AR loop only; excludes model loading and video decoding", - } - timing_path = cfg.output_dir / "timing_flashdreams.json" - timing_path.write_text(json.dumps(timing_results, indent=2)) - logger.info("Timing saved to {}", timing_path) - logger.info("[FlashDreams] Pipeline: {:.4f}s, AR loop: {:.4f}s, avg chunk: {:.4f}s", - pipeline_elapsed, infer_elapsed, timing_results["avg_chunk_sec"]) - - # save - all_latents = torch.cat(pred_latents, dim=2) - all_actions = torch.cat(pred_actions, dim=1).flatten(1).numpy() - - if cfg.save_actions: - np.save(str(cfg.output_dir / "actions.npy"), all_actions) - logger.info("Saved actions.npy") - - torch.save(all_latents, str(cfg.output_dir / "latents.pt")) - logger.info("Saved latents.pt") - - # optional video decode - if cfg.save_video: - logger.info("Decoding video — freeing inference models") - logger.info("VRAM before cleanup: {:.2f} GiB allocated", - torch.cuda.memory_allocated() / 1024**3) - # Free KV caches (self-attn + cross-attn) - tc = pipeline_cache.transformer_cache - for nc in [tc.network_cache, tc.network_cache_uncond]: - if nc is None: - continue - for bc in nc.block_caches: - bc.self_attn.reset() - bc.cross_attn.text.k = torch.empty(0) - bc.cross_attn.text.v = torch.empty(0) - del pipeline_cache - # Free streaming VAE caches (but keep self._vae on GPU for decode) - self._streaming_vae.clear_cache() - if self._streaming_vae_half: - self._streaming_vae_half.clear_cache() - # Move all models except VAE to CPU - self.pipeline.transformer.network.to("cpu") - if hasattr(self, '_text_encoder') and self._text_encoder is not None: - self._text_encoder.to("cpu") - del self._streaming_vae - del self._streaming_vae_half - del self._text_encoder - del prompt_embeds, neg_embeds, init_latent, action_mask - del pred_latents, pred_actions - gc.collect() - torch.cuda.empty_cache() - logger.info("VRAM after cleanup: {:.2f} GiB allocated", - torch.cuda.memory_allocated() / 1024**3) - - self._vae = self._vae.to(device).to(dtype) - - from diffusers.utils import export_to_video - from diffusers.video_processor import VideoProcessor - - vp = VideoProcessor(vae_scale_factor=1) - - # Denormalize latents - lat = all_latents.to(device).to(self._vae.dtype) - del all_latents - lat_mean = ( - torch.tensor(self._vae.config.latents_mean) - .view(1, self._vae.config.z_dim, 1, 1, 1) - .to(lat.device, lat.dtype) - ) - lat_std = ( - 1.0 / torch.tensor(self._vae.config.latents_std) - .view(1, self._vae.config.z_dim, 1, 1, 1) - .to(lat.device, lat.dtype) - ) - lat = lat / lat_std + lat_mean - - # Memory-efficient decode: process frame-by-frame using the VAE's - # causal conv3d streaming interface but offload decoded frames to CPU - # immediately to avoid accumulating the full pixel-space video on GPU. - vae = self._vae - num_latent_frames = lat.shape[2] - vae.clear_cache() - x = vae.post_quant_conv(lat) - del lat - decoded_frames = [] - for i in range(num_latent_frames): - vae._conv_idx = [0] - if i == 0: - frame = vae.decoder( - x[:, :, i:i+1, :, :], - feat_cache=vae._feat_map, - feat_idx=vae._conv_idx, - first_chunk=True, - ) - else: - frame = vae.decoder( - x[:, :, i:i+1, :, :], - feat_cache=vae._feat_map, - feat_idx=vae._conv_idx, - ) - decoded_frames.append(frame.detach().cpu()) - del frame - del x - vae.clear_cache() - torch.cuda.empty_cache() - - out = torch.cat(decoded_frames, dim=2) - del decoded_frames - if vae.config.patch_size is not None: - from diffusers.models.autoencoders.autoencoder_kl_wan import unpatchify - out = unpatchify(out, patch_size=vae.config.patch_size) - video = torch.clamp(out, min=-1.0, max=1.0) - del out - - video = vp.postprocess_video(video, output_type="np")[0] - export_to_video(video, str(cfg.output_dir / "demo.mp4"), fps=10) - logger.info("Saved demo.mp4") - - # runtime manifest - manifest = { - "runner_name": cfg.runner_name, - "checkpoint_root": str(cfg.checkpoint_root), - "prompt": prompt, - "num_chunks": cfg.num_chunks, - "seed": cfg.seed, - "timestamp": time.strftime("%Y%m%d_%H%M%S"), - } - (cfg.output_dir / "runtime.json").write_text(json.dumps(manifest, indent=2)) - logger.info("Done. Outputs in {}", cfg.output_dir.resolve()) diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py b/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py index 1b5bf9c77..12273fb98 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py @@ -22,18 +22,14 @@ from __future__ import annotations -import math from dataclasses import dataclass import torch import torch.nn as nn from torch import Tensor -from flashdreams.core.attention import NativeAttention from flashdreams.core.attention.rope import apply_rope_freqs from flashdreams.recipes.wan.transformer.impl.modules import ( - Block, - BlockCache, CrossAttnCache, CrossAttention, MultiHeadAttention, diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/network.py b/integrations/lingbot_va/lingbot_va/transformer/impl/network.py index da8a98b3b..8a022181c 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/impl/network.py +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/network.py @@ -18,14 +18,12 @@ from __future__ import annotations import math -from dataclasses import dataclass, field -from typing import Any +from dataclasses import dataclass import torch import torch.nn as nn from torch import Tensor -from flashdreams.core.attention.rope import apply_rope_freqs from flashdreams.recipes.wan.transformer.impl.modules import ( Head, sinusoidal_embedding_1d, diff --git a/integrations/lingbot_va/pyproject.toml b/integrations/lingbot_va/pyproject.toml index f1430e51d..40a463fbb 100644 --- a/integrations/lingbot_va/pyproject.toml +++ b/integrations/lingbot_va/pyproject.toml @@ -20,14 +20,16 @@ build-backend = "setuptools.build_meta" [project] name = "flashdreams-lingbot-va" version = "0.1.0" -description = "LingBot-VA Robotwin I2AV inference integration scaffold for flashdreams." +description = "LingBot-VA Robotwin I2AV model integration for FlashDreams V2 applications." readme = "README.md" requires-python = ">=3.10" dependencies = [ "flashdreams", - "diffusers>=0.34", - "transformers>=5.0", + # The engine uses Wan VAE private streaming fields; retest before widening. + "diffusers>=0.38,<0.39", "einops", + "Pillow>=10", + "transformers>=5.0,<6", ] [tool.uv.sources] @@ -39,9 +41,6 @@ dev = [ "tomli>=2.0", ] -[project.entry-points."flashdreams.runner_configs"] -"lingbot-va-robotwin-i2av" = "lingbot_va.config:RUNNER_LINGBOT_VA_ROBOTWIN_I2AV" - [tool.setuptools.packages.find] include = ["lingbot_va*"] exclude = ["tests"] diff --git a/integrations/lingbot_va/tests/test_engine.py b/integrations/lingbot_va/tests/test_engine.py new file mode 100644 index 000000000..b8556757e --- /dev/null +++ b/integrations/lingbot_va/tests/test_engine.py @@ -0,0 +1,135 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""CPU tests for LingBot engine configuration and one-run lifecycle.""" + +from pathlib import Path + +import pytest +import torch + +from lingbot_va.constants import ROBOTWIN_OBS_CAM_KEYS +from lingbot_va.engine import ( + LingbotVAEngine, + LingbotVAEngineConfig, + LingbotVAEngineOutput, + LingbotVAEngineState, + build_pipeline_config, + expected_output_shape, + validate_input_images, +) + +pytestmark = pytest.mark.ci_cpu + + +class _StandInEngine(LingbotVAEngine): + """Return fixed CPU tensors without loading external model packages.""" + + def _run_impl(self) -> LingbotVAEngineOutput: + return LingbotVAEngineOutput( + video=torch.zeros(2, 3, 4, 5), + actions=torch.zeros(32, 16), + metrics={"total_seconds": 0.0}, + ) + + +class _FailingEngine(LingbotVAEngine): + """Fail after entering the RUNNING state.""" + + def _run_impl(self) -> LingbotVAEngineOutput: + raise RuntimeError("inference failed") + + def _release_denoising_state(self) -> None: + raise ValueError("cleanup failed") + + +def test_pipeline_config_applies_every_model_override(tmp_path: Path) -> None: + config = LingbotVAEngineConfig( + seed=17, + compile_network=False, + guidance_scale=2.5, + action_guidance_scale=1.5, + video_inference_steps=7, + action_inference_steps=9, + video_snr_shift=4.0, + action_snr_shift=2.0, + ) + + resolved = build_pipeline_config(config, tmp_path) + + assert resolved.checkpoint_root == str(tmp_path) + assert resolved.enable_sync_and_profile is False + assert resolved.diffusion_model.seed == 17 + assert resolved.diffusion_model.transformer.checkpoint_root == str(tmp_path) + assert resolved.diffusion_model.transformer.compile_network is False + assert resolved.diffusion_model.transformer.guidance_scale == 2.5 + assert resolved.diffusion_model.transformer.action_guidance_scale == 1.5 + assert resolved.diffusion_model.scheduler.num_inference_steps == 7 + assert resolved.diffusion_model.scheduler.shift == 4.0 + assert resolved.action_scheduler.num_inference_steps == 9 + assert resolved.action_scheduler.shift == 2.0 + + +def test_engine_is_one_run_and_close_is_idempotent() -> None: + engine = _StandInEngine(LingbotVAEngineConfig()) + + output = engine.run() + + assert output.video.shape == (2, 3, 4, 5) + assert output.actions.shape == (32, 16) + assert engine.state is LingbotVAEngineState.FINISHED + with pytest.raises(RuntimeError, match="requires NEW state"): + engine.run() + + engine.close() + engine.close() + assert engine.state is LingbotVAEngineState.CLOSED + + +def test_engine_failure_closes_partial_state() -> None: + engine = _FailingEngine(LingbotVAEngineConfig()) + + with pytest.raises(RuntimeError, match="inference failed"): + engine.run() + + assert engine.state is LingbotVAEngineState.CLOSED + + +def test_validate_input_images_names_all_missing_cameras(tmp_path: Path) -> None: + with pytest.raises(FileNotFoundError) as error: + validate_input_images(tmp_path) + + for key in ROBOTWIN_OBS_CAM_KEYS: + assert f"{key}.png" in str(error.value) + + +def test_validate_input_images_returns_camera_mapping(tmp_path: Path) -> None: + for key in ROBOTWIN_OBS_CAM_KEYS: + (tmp_path / f"{key}.png").touch() + + image_paths = validate_input_images(tmp_path) + + assert tuple(image_paths) == ROBOTWIN_OBS_CAM_KEYS + + +def test_expected_shape_scales_only_with_chunk_count() -> None: + config = LingbotVAEngineConfig(num_chunks=3) + + assert expected_output_shape(config) == (6, 3, 256, 320) + + +@pytest.mark.parametrize( + ("changes", "message"), + [ + ({"num_chunks": 0}, "num_chunks"), + ({"video_inference_steps": 0}, "step counts"), + ({"video_snr_shift": 0.0}, "SNR shifts"), + ({"guidance_scale": -1.0}, "guidance scales"), + ], +) +def test_engine_config_rejects_invalid_values( + changes: dict[str, int | float], + message: str, +) -> None: + with pytest.raises(ValueError, match=message): + LingbotVAEngineConfig(**changes) # type: ignore[arg-type] diff --git a/uv.lock b/uv.lock index 04900a52b..d57b8fbd4 100644 --- a/uv.lock +++ b/uv.lock @@ -29,6 +29,7 @@ members = [ "flashdreams-flashvsr", "flashdreams-hy-worldplay", "flashdreams-lingbot", + "flashdreams-lingbot-va", "flashdreams-null-model", "flashdreams-omnidreams", "flashdreams-red-screen", @@ -1345,6 +1346,36 @@ requires-dist = [ ] provides-extras = ["dev"] +[[package]] +name = "flashdreams-lingbot-va" +version = "0.1.0" +source = { editable = "integrations/lingbot_va" } +dependencies = [ + { name = "diffusers" }, + { name = "einops" }, + { name = "flashdreams" }, + { name = "pillow" }, + { name = "transformers" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "tomli" }, +] + +[package.metadata] +requires-dist = [ + { name = "diffusers", specifier = ">=0.38,<0.39" }, + { name = "einops" }, + { name = "flashdreams", editable = "flashdreams" }, + { name = "pillow", specifier = ">=10" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, + { name = "tomli", marker = "extra == 'dev'", specifier = ">=2.0" }, + { name = "transformers", specifier = ">=5.0,<6" }, +] +provides-extras = ["dev"] + [[package]] name = "flashdreams-null-model" version = "0.1.0" From 771e1c2ea876ed0a733d5f46ca2dffac0a2574ad Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Tue, 25 Aug 2026 14:49:23 -0700 Subject: [PATCH 05/18] Add LingBot Robotwin V2 application Signed-off-by: Jonathan McCaffrey --- integrations_v2/lingbot_va/README.md | 5 + .../lingbot_va/lingbot_va_v2/__init__.py | 20 + .../lingbot_va/lingbot_va_v2/app.py | 419 ++++++++++++++++++ .../lingbot_va_v2/tests/test_app.py | 347 +++++++++++++++ integrations_v2/lingbot_va/pyproject.toml | 30 ++ uv.lock | 16 + 6 files changed, 837 insertions(+) create mode 100644 integrations_v2/lingbot_va/README.md create mode 100644 integrations_v2/lingbot_va/lingbot_va_v2/__init__.py create mode 100644 integrations_v2/lingbot_va/lingbot_va_v2/app.py create mode 100644 integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py create mode 100644 integrations_v2/lingbot_va/pyproject.toml diff --git a/integrations_v2/lingbot_va/README.md b/integrations_v2/lingbot_va/README.md new file mode 100644 index 000000000..79c899f30 --- /dev/null +++ b/integrations_v2/lingbot_va/README.md @@ -0,0 +1,5 @@ +# LingBot-VA V2 application + +This package adapts the LingBot-VA Robotwin I2AV model integration to the +FlashDreams V2 application, session, and model-loop APIs. See the model +package's README for checkpoint and input details. diff --git a/integrations_v2/lingbot_va/lingbot_va_v2/__init__.py b/integrations_v2/lingbot_va/lingbot_va_v2/__init__.py new file mode 100644 index 000000000..111b01066 --- /dev/null +++ b/integrations_v2/lingbot_va/lingbot_va_v2/__init__.py @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""LingBot-VA Robotwin application for the FlashDreams V2 API.""" + +from lingbot_va_v2.app import create_app + +__all__ = ["create_app"] diff --git a/integrations_v2/lingbot_va/lingbot_va_v2/app.py b/integrations_v2/lingbot_va/lingbot_va_v2/app.py new file mode 100644 index 000000000..c27b66a9d --- /dev/null +++ b/integrations_v2/lingbot_va/lingbot_va_v2/app.py @@ -0,0 +1,419 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""LingBot-VA Robotwin I2AV application for the FlashDreams V2 API.""" + +from __future__ import annotations + +import argparse +from collections.abc import Callable, Mapping, Sequence +from dataclasses import dataclass +from pathlib import Path +from typing import Protocol + +from flashdreams.api_v2.application import IApplication +from flashdreams.api_v2.loop import IModelLoop +from flashdreams.api_v2.session import ISession +from flashdreams.runtime_v2.session_desc import ( + BackpressureMode, + PresentationMode, + SessionDesc, +) +from flashdreams.runtime_v2.step_result import StepResult +from flashdreams.runtime_v2.tensor_artifact import ( + TensorArtifactOutput, + TensorArtifactSchema, +) +from flashdreams.runtime_v2.user_input_events import UserInputEvents +from flashdreams.runtime_v2.video_tensor import VideoTensorLayout +from lingbot_va._loaders import validate_checkpoint_root +from lingbot_va.constants import ( + DEFAULT_CHECKPOINT_ROOT, + DEFAULT_INPUT_IMAGE_DIR, + DEFAULT_PROMPT, + ROBOTWIN_ACTION_DIM, + ROBOTWIN_ACTION_GUIDANCE_SCALE, + ROBOTWIN_ACTION_INFERENCE_STEPS, + ROBOTWIN_ACTION_PER_FRAME, + ROBOTWIN_ACTION_SNR_SHIFT, + ROBOTWIN_FRAME_CHUNK_SIZE, + ROBOTWIN_GUIDANCE_SCALE, + ROBOTWIN_HEIGHT, + ROBOTWIN_SNR_SHIFT, + ROBOTWIN_USED_ACTION_CHANNEL_IDS, + ROBOTWIN_VIDEO_INFERENCE_STEPS, + ROBOTWIN_WIDTH, +) +from lingbot_va.engine import ( + LingbotVAEngine, + LingbotVAEngineConfig, + LingbotVAEngineOutput, + expected_output_shape, + validate_device, + validate_input_images, +) +from lingbot_va.utils import resolve_prompt + +_FRAMES_PER_SECOND = 10 +"""Native Robotwin video playback rate.""" + +ACTIONS_SCHEMA = TensorArtifactSchema( + name="actions", + dimension_names=("step", "channel"), + concatenate_axis=0, +) +"""Generic tensor artifact schema for denormalized Robotwin actions.""" + + +class LingbotVAEngineLike(Protocol): + """Minimal engine boundary used by the V2 adapter and CPU stand-ins.""" + + def run(self) -> LingbotVAEngineOutput: + """Generate one complete rollout.""" + ... + + def close(self) -> None: + """Release partially or fully initialized model state.""" + ... + + +EngineFactory = Callable[[LingbotVAEngineConfig], LingbotVAEngineLike] +"""Create a session-owned engine from immutable application config.""" + + +@dataclass(slots=True) +class LingbotVAModelState: + """Mutable state owned exclusively by the model loop.""" + + config: LingbotVAEngineConfig + session_desc: SessionDesc + engine_factory: EngineFactory + engine: LingbotVAEngineLike | None = None + generated: bool = False + + +class LingbotVAModelLoop(IModelLoop[LingbotVAModelState]): + """Generate one complete video/action rollout in one honest model step.""" + + def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: + del events + if self.state.generated: + raise RuntimeError("LingBot-VA has already generated this rollout.") + if self.state.engine is None: + self.state.engine = self.state.engine_factory(self.state.config) + output = self.state.engine.run() + _validate_engine_output(self.state.config, output) + self.state.generated = True + return [ + StepResult( + step_index=step_index, + output=output.video, + frame_count=output.video.shape[0], + output_layout=self.state.session_desc.output_layout, + metrics=dict(output.metrics), + tensor_artifacts=( + TensorArtifactOutput( + schema=ACTIONS_SCHEMA, + tensor=output.actions, + ), + ), + ) + ] + + def is_finished(self) -> bool: + return self.state.generated + + def reset(self) -> None: + """Discard the destructive engine; the next step creates a new one.""" + self.close() + self.state.generated = False + + def close(self) -> None: + """Idempotently close the session-owned engine.""" + engine = self.state.engine + self.state.engine = None + if engine is not None: + engine.close() + + +class LingbotVASession(ISession): + """Own one isolated, resettable LingBot-VA rollout.""" + + def __init__( + self, + config: LingbotVAEngineConfig, + session_desc: SessionDesc, + engine_factory: EngineFactory, + ) -> None: + """ + Args: + config: Immutable model and input settings. + session_desc: Canonical Robotwin output description. + engine_factory: Factory used lazily on the model thread. + """ + self._config = config + self._session_desc = session_desc + self._engine_factory = engine_factory + self._model_loop: LingbotVAModelLoop | None = None + + def init(self) -> None: + """Register one finite model loop; model loading remains lazy.""" + if self._model_loop is not None: + raise RuntimeError("LingbotVASession.init() may only run once.") + model_loop = self.register_model_loop( + LingbotVAModelLoop, + state=LingbotVAModelState( + config=self._config, + session_desc=self._session_desc, + engine_factory=self._engine_factory, + ), + ) + assert isinstance(model_loop, LingbotVAModelLoop) + self._model_loop = model_loop + + @property + def session_desc(self) -> SessionDesc: + return self._session_desc + + def close(self) -> None: + """Close a loop even when the runtime never started it.""" + if self._model_loop is not None: + self._model_loop.close() + + +class LingbotVAApplication(IApplication): + """Parse LingBot settings and create session-owned one-run engines.""" + + def __init__(self, engine_factory: EngineFactory = LingbotVAEngine) -> None: + """ + Args: + engine_factory: Injectable model boundary for CPU lifecycle tests. + """ + self._engine_factory = engine_factory + self._config: LingbotVAEngineConfig | None = None + + def session_desc(self) -> SessionDesc: + """Describe natural Robotwin outputs without parsing args or loading models.""" + return _session_desc() + + def init(self, commandline_args: Sequence[str]) -> None: + """Parse effective overrides and validate startup state without weights.""" + args = _parse_args(commandline_args) + prompt: str | Path = args.prompt_file or args.prompt + config = LingbotVAEngineConfig( + checkpoint_root=args.checkpoint_root, + checkpoint_revision=args.checkpoint_revision, + input_image_dir=args.input_image_dir, + prompt=prompt, + num_chunks=args.num_chunks, + seed=args.seed, + device=args.device, + enable_offload=args.enable_offload, + compile_network=args.compile_network, + guidance_scale=args.guidance_scale, + action_guidance_scale=args.action_guidance_scale, + video_inference_steps=args.video_inference_steps, + action_inference_steps=args.action_inference_steps, + video_snr_shift=args.video_snr_shift, + action_snr_shift=args.action_snr_shift, + ) + validate_device(config.device) + _validate_checkpoint_reference(config.checkpoint_root) + validate_input_images(config.input_image_dir) + resolve_prompt(config.prompt) + self._config = config + + def create_session(self, session_desc: SessionDesc) -> ISession: + """Create an uninitialized session for the exact Robotwin contract.""" + if self._config is None: + raise RuntimeError( + "LingbotVAApplication.init() must run before create_session()." + ) + canonical = _session_desc() + _validate_requested_session(session_desc, canonical) + return LingbotVASession( + self._config, + canonical, + self._engine_factory, + ) + + +def _parse_args(commandline_args: Sequence[str]) -> argparse.Namespace: + """Parse only settings that alter the effective model run.""" + parser = argparse.ArgumentParser( + prog="lingbot-va-robotwin-i2av", + description="Generate Robotwin video and actions with LingBot-VA.", + ) + parser.add_argument("--checkpoint-root", default=DEFAULT_CHECKPOINT_ROOT) + parser.add_argument("--checkpoint-revision") + parser.add_argument( + "--input-image-dir", + type=Path, + default=DEFAULT_INPUT_IMAGE_DIR, + ) + prompt_group = parser.add_mutually_exclusive_group() + prompt_group.add_argument("--prompt", default=DEFAULT_PROMPT) + prompt_group.add_argument("--prompt-file", type=Path) + parser.add_argument("--num-chunks", type=int, default=10) + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--device", default="cuda:0") + parser.add_argument( + "--compile", + dest="compile_network", + action=argparse.BooleanOptionalAction, + default=True, + ) + parser.add_argument("--enable-offload", action="store_true") + parser.add_argument( + "--guidance-scale", + type=float, + default=ROBOTWIN_GUIDANCE_SCALE, + ) + parser.add_argument( + "--action-guidance-scale", + type=float, + default=ROBOTWIN_ACTION_GUIDANCE_SCALE, + ) + parser.add_argument( + "--video-inference-steps", + type=int, + default=ROBOTWIN_VIDEO_INFERENCE_STEPS, + ) + parser.add_argument( + "--action-inference-steps", + type=int, + default=ROBOTWIN_ACTION_INFERENCE_STEPS, + ) + parser.add_argument( + "--video-snr-shift", + type=float, + default=ROBOTWIN_SNR_SHIFT, + ) + parser.add_argument( + "--action-snr-shift", + type=float, + default=ROBOTWIN_ACTION_SNR_SHIFT, + ) + return parser.parse_args(list(commandline_args)) + + +def _session_desc() -> SessionDesc: + """Build the canonical natural Robotwin V2 session contract.""" + return SessionDesc( + output_layout=VideoTensorLayout.tchw, + backpressure_mode=BackpressureMode.BLOCK, + presentation_mode=PresentationMode.ONLY_PRESENT_NEW, + frames_per_second_for_ui=_FRAMES_PER_SECOND, + frames_per_second_for_step=_FRAMES_PER_SECOND, + video_width=ROBOTWIN_WIDTH, + video_height=ROBOTWIN_HEIGHT, + tensor_artifact_schemas=(ACTIONS_SCHEMA,), + metadata={ + "action_dim": ROBOTWIN_ACTION_DIM, + "action_channel_ids": ROBOTWIN_USED_ACTION_CHANNEL_IDS, + }, + ) + + +def _validate_requested_session( + requested: SessionDesc, + canonical: SessionDesc, +) -> None: + """Reject runtime requests that would misdescribe fixed Robotwin output.""" + fields = ( + "output_layout", + "backpressure_mode", + "presentation_mode", + "frames_per_second_for_ui", + "frames_per_second_for_step", + "video_width", + "video_height", + "tensor_artifact_schemas", + ) + mismatches = [ + field_name + for field_name in fields + if getattr(requested, field_name) != getattr(canonical, field_name) + ] + if mismatches: + raise ValueError( + "LingBot-VA requires its natural Robotwin session contract; " + "mismatched field(s): " + ", ".join(mismatches) + "." + ) + + +def _validate_checkpoint_reference(checkpoint_root: str | Path) -> None: + """Validate existing/explicit local roots while accepting remote repo IDs.""" + value = str(checkpoint_root) + expanded = Path(value).expanduser() + if expanded.exists(): + validate_checkpoint_root(expanded) + return + if ( + isinstance(checkpoint_root, Path) + or expanded.is_absolute() + or value.startswith((".", "~")) + ): + raise FileNotFoundError( + f"LingBot-VA checkpoint root does not exist: {expanded}" + ) + + +def _validate_engine_output( + config: LingbotVAEngineConfig, + output: LingbotVAEngineOutput, +) -> None: + """Keep incorrect model shapes from reaching generic runtime sinks.""" + expected_video = expected_output_shape(config) + if tuple(output.video.shape) != expected_video: + raise ValueError( + f"LingBot-VA engine returned video shape {tuple(output.video.shape)}; " + f"expected {expected_video}." + ) + expected_action_shape = ( + config.num_chunks * ROBOTWIN_FRAME_CHUNK_SIZE * ROBOTWIN_ACTION_PER_FRAME, + len(ROBOTWIN_USED_ACTION_CHANNEL_IDS), + ) + if tuple(output.actions.shape) != expected_action_shape: + raise ValueError( + f"LingBot-VA engine returned action shape {tuple(output.actions.shape)}; " + f"expected {expected_action_shape}." + ) + if not output.video.is_floating_point() or not output.actions.is_floating_point(): + raise TypeError("LingBot-VA video and action outputs must be floating point.") + _validate_metrics(output.metrics) + + +def _validate_metrics(metrics: Mapping[str, float]) -> None: + """Require numeric model metrics before constructing a StepResult.""" + if any( + isinstance(value, bool) or not isinstance(value, (int, float)) + for value in metrics.values() + ): + raise TypeError("LingBot-VA engine metrics must be numeric.") + + +def create_app() -> IApplication: + """Return a new uninitialized LingBot-VA V2 application.""" + return LingbotVAApplication() + + +__all__ = [ + "ACTIONS_SCHEMA", + "LingbotVAApplication", + "LingbotVAModelLoop", + "LingbotVASession", + "create_app", +] diff --git a/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py b/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py new file mode 100644 index 000000000..9950763ff --- /dev/null +++ b/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py @@ -0,0 +1,347 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CPU lifecycle tests for the LingBot-VA V2 application.""" + +from __future__ import annotations + +from dataclasses import replace +from pathlib import Path + +import numpy as np +import pytest +import torch + +from flashdreams.api_v2.application import IApplication +from flashdreams.api_v2.client_window import IClientWindow +from flashdreams.runtime_v2.session_desc import ( + BackpressureMode, + PresentationMode, + SessionDesc, +) +from flashdreams.runtime_v2.session_runner import run_session +from flashdreams.runtime_v2.step_result import StepResult +from flashdreams.runtime_v2.tensor_artifact_output_sink import ( + TensorArtifactOutputSink, +) +from flashdreams.runtime_v2.user_input_events import UserInputEvents +from flashdreams.runtime_v2.video_tensor import VideoTensorLayout +from lingbot_va.constants import ROBOTWIN_OBS_CAM_KEYS +from lingbot_va.engine import LingbotVAEngineConfig, LingbotVAEngineOutput +from lingbot_va_v2.app import ( + ACTIONS_SCHEMA, + LingbotVAApplication, + create_app, +) + +pytestmark = pytest.mark.ci_cpu + + +class _FakeEngine: + """Emit deterministic natural-shape CPU outputs.""" + + def __init__( + self, + config: LingbotVAEngineConfig, + *, + video_shape: tuple[int, ...] | None = None, + ) -> None: + self.config = config + self.closed = False + self.runs = 0 + frame_count = config.num_chunks * 2 + action_steps = config.num_chunks * 32 + self._output = LingbotVAEngineOutput( + video=torch.zeros( + video_shape or (frame_count, 3, 256, 320), + dtype=torch.float32, + ), + actions=torch.arange(action_steps * 16, dtype=torch.float32).reshape( + action_steps, + 16, + ), + metrics={"total_seconds": 1.25}, + ) + + def run(self) -> LingbotVAEngineOutput: + self.runs += 1 + return self._output + + def close(self) -> None: + self.closed = True + + +class _FakeEngineFactory: + """Record every session/reset-owned engine instance.""" + + def __init__(self, *, video_shape: tuple[int, ...] | None = None) -> None: + self.engines: list[_FakeEngine] = [] + self._video_shape = video_shape + + def __call__(self, config: LingbotVAEngineConfig) -> _FakeEngine: + engine = _FakeEngine(config, video_shape=self._video_shape) + self.engines.append(engine) + return engine + + +class _RecordingWindow(IClientWindow): + """Collect outputs while reporting no user input.""" + + def __init__(self) -> None: + self.session_desc: SessionDesc | None = None + self.results: list[StepResult] = [] + self.closed = False + + def get_user_input_events(self) -> UserInputEvents: + return UserInputEvents([]) + + def open(self, session_desc: SessionDesc) -> None: + self.session_desc = session_desc + + def write(self, result: StepResult) -> None: + self.results.append(result) + + def close(self) -> None: + self.closed = True + + +def _input_dir(tmp_path: Path) -> Path: + input_dir = tmp_path / "inputs" + input_dir.mkdir() + for key in ROBOTWIN_OBS_CAM_KEYS: + (input_dir / f"{key}.png").touch() + return input_dir + + +def _init_app( + tmp_path: Path, + factory: _FakeEngineFactory, + *extra_args: str, +) -> LingbotVAApplication: + app = LingbotVAApplication(factory) + app.init( + [ + "--device", + "cpu", + "--input-image-dir", + str(_input_dir(tmp_path)), + "--num-chunks", + "1", + "--no-compile", + *extra_args, + ] + ) + return app + + +def _step(session: object, step_index: int = 0) -> StepResult: + model_loop = session.model_loop # type: ignore[attr-defined] + results = model_loop.step(step_index, UserInputEvents([])) + assert isinstance(results, list) + return results[0] + + +def test_create_app_returns_v2_application() -> None: + assert isinstance(create_app(), IApplication) + + +def test_session_desc_is_cheap_and_natural_before_init() -> None: + factory = _FakeEngineFactory() + app = LingbotVAApplication(factory) + + session_desc = app.session_desc() + + assert factory.engines == [] + assert session_desc.output_layout is VideoTensorLayout.tchw + assert session_desc.backpressure_mode is BackpressureMode.BLOCK + assert session_desc.presentation_mode is PresentationMode.ONLY_PRESENT_NEW + assert session_desc.frames_per_second_for_ui == 10 + assert session_desc.frames_per_second_for_step == 10 + assert (session_desc.video_width, session_desc.video_height) == (320, 256) + assert session_desc.tensor_artifact_schemas == (ACTIONS_SCHEMA,) + + +def test_application_init_validates_without_creating_engine(tmp_path: Path) -> None: + factory = _FakeEngineFactory() + + app = _init_app(tmp_path, factory) + + assert factory.engines == [] + session = app.create_session(app.session_desc()) + assert factory.engines == [] + session.init() + assert factory.engines == [] + + +def test_one_step_returns_video_actions_and_metrics(tmp_path: Path) -> None: + factory = _FakeEngineFactory() + app = _init_app(tmp_path, factory) + session = app.create_session(app.session_desc()) + session.init() + + result = _step(session) + + assert result.output.shape == (2, 3, 256, 320) + assert result.output_layout is VideoTensorLayout.tchw + assert result.frame_count == 2 + assert result.metrics == {"total_seconds": 1.25} + assert len(result.tensor_artifacts) == 1 + actions = result.tensor_artifacts[0] + assert actions.schema is ACTIONS_SCHEMA + assert actions.tensor.shape == (32, 16) + assert session.model_loop.is_finished() + + +def test_reset_closes_engine_and_lazily_creates_another(tmp_path: Path) -> None: + factory = _FakeEngineFactory() + app = _init_app(tmp_path, factory) + session = app.create_session(app.session_desc()) + session.init() + _step(session) + + session.model_loop.reset() + + assert factory.engines[0].closed + assert not session.model_loop.is_finished() + assert len(factory.engines) == 1 + _step(session) + assert len(factory.engines) == 2 + session.close() + assert factory.engines[1].closed + + +def test_create_session_rejects_misdescribed_robotwin_output(tmp_path: Path) -> None: + app = _init_app(tmp_path, _FakeEngineFactory()) + + with pytest.raises(ValueError, match="video_width"): + app.create_session(replace(app.session_desc(), video_width=640)) + + +def test_create_session_before_init_fails() -> None: + app = LingbotVAApplication(_FakeEngineFactory()) + + with pytest.raises(RuntimeError, match="init"): + app.create_session(app.session_desc()) + + +def test_init_rejects_missing_camera_inputs(tmp_path: Path) -> None: + app = LingbotVAApplication(_FakeEngineFactory()) + + with pytest.raises(FileNotFoundError, match="camera PNGs"): + app.init( + [ + "--device", + "cpu", + "--input-image-dir", + str(tmp_path), + ] + ) + + +def test_init_rejects_nonexistent_explicit_checkpoint(tmp_path: Path) -> None: + app = LingbotVAApplication(_FakeEngineFactory()) + input_dir = _input_dir(tmp_path) + + with pytest.raises(FileNotFoundError, match="checkpoint root"): + app.init( + [ + "--device", + "cpu", + "--input-image-dir", + str(input_dir), + "--checkpoint-root", + str(tmp_path / "missing-checkpoint"), + ] + ) + + +def test_every_cli_override_reaches_engine_config(tmp_path: Path) -> None: + factory = _FakeEngineFactory() + app = _init_app( + tmp_path, + factory, + "--checkpoint-root", + "owner/repo", + "--checkpoint-revision", + "revision-1", + "--prompt", + "do the thing", + "--seed", + "9", + "--enable-offload", + "--guidance-scale", + "3.5", + "--action-guidance-scale", + "2.5", + "--video-inference-steps", + "7", + "--action-inference-steps", + "8", + "--video-snr-shift", + "4.5", + "--action-snr-shift", + "1.5", + ) + session = app.create_session(app.session_desc()) + session.init() + + _step(session) + config = factory.engines[0].config + + assert config.checkpoint_root == "owner/repo" + assert config.checkpoint_revision == "revision-1" + assert config.prompt == "do the thing" + assert config.seed == 9 + assert config.enable_offload is True + assert config.compile_network is False + assert config.guidance_scale == 3.5 + assert config.action_guidance_scale == 2.5 + assert config.video_inference_steps == 7 + assert config.action_inference_steps == 8 + assert config.video_snr_shift == 4.5 + assert config.action_snr_shift == 1.5 + + +def test_model_loop_rejects_wrong_engine_video_shape(tmp_path: Path) -> None: + factory = _FakeEngineFactory(video_shape=(1, 3, 256, 320)) + app = _init_app(tmp_path, factory) + session = app.create_session(app.session_desc()) + session.init() + + with pytest.raises(ValueError, match="video shape"): + _step(session) + + session.close() + assert factory.engines[0].closed + + +def test_runtime_routes_actions_through_generic_sink(tmp_path: Path) -> None: + factory = _FakeEngineFactory() + app = _init_app(tmp_path, factory) + session = app.create_session(app.session_desc()) + window = _RecordingWindow() + artifact_dir = tmp_path / "artifacts" + + run_session( + session, + window, + tensor_artifact_output_sink=TensorArtifactOutputSink(artifact_dir), + ) + + assert window.closed + assert factory.engines[0].closed + actions = np.load(artifact_dir / "actions.npy") + assert actions.shape == (32, 16) + np.testing.assert_array_equal(actions, np.arange(32 * 16).reshape(32, 16)) diff --git a/integrations_v2/lingbot_va/pyproject.toml b/integrations_v2/lingbot_va/pyproject.toml new file mode 100644 index 000000000..f4485f86e --- /dev/null +++ b/integrations_v2/lingbot_va/pyproject.toml @@ -0,0 +1,30 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +[build-system] +requires = ["setuptools>=69", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "flashdreams-lingbot-va-v2" +version = "0.1.0" +description = "LingBot-VA Robotwin I2AV application for the FlashDreams V2 API." +readme = "README.md" +requires-python = ">=3.10" +dependencies = [ + "flashdreams", + "flashdreams-lingbot-va", +] + +[tool.uv.sources] +flashdreams = { workspace = true } +flashdreams-lingbot-va = { workspace = true } + +[project.entry-points."flashdreams.applications_v2"] +"lingbot-va-robotwin-i2av" = "lingbot_va_v2.app:create_app" + +[tool.setuptools.packages.find] +include = ["lingbot_va_v2*"] + +[tool.uv] +managed = true diff --git a/uv.lock b/uv.lock index d57b8fbd4..bec7c2ebf 100644 --- a/uv.lock +++ b/uv.lock @@ -30,6 +30,7 @@ members = [ "flashdreams-hy-worldplay", "flashdreams-lingbot", "flashdreams-lingbot-va", + "flashdreams-lingbot-va-v2", "flashdreams-null-model", "flashdreams-omnidreams", "flashdreams-red-screen", @@ -1376,6 +1377,21 @@ requires-dist = [ ] provides-extras = ["dev"] +[[package]] +name = "flashdreams-lingbot-va-v2" +version = "0.1.0" +source = { editable = "integrations_v2/lingbot_va" } +dependencies = [ + { name = "flashdreams" }, + { name = "flashdreams-lingbot-va" }, +] + +[package.metadata] +requires-dist = [ + { name = "flashdreams", editable = "flashdreams" }, + { name = "flashdreams-lingbot-va", editable = "integrations/lingbot_va" }, +] + [[package]] name = "flashdreams-null-model" version = "0.1.0" From 5459f0db6001b37c2d3c5001980dff8945fb9eb1 Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Tue, 25 Aug 2026 15:39:31 -0700 Subject: [PATCH 06/18] Fix LingBot video-action parity Signed-off-by: Jonathan McCaffrey --- .../lingbot_va/lingbot_va/_loaders.py | 52 +++++---- .../lingbot_va/lingbot_va/constants.py | 4 +- integrations/lingbot_va/lingbot_va/engine.py | 45 ++++++-- .../lingbot_va/lingbot_va/pipeline.py | 17 +-- .../lingbot_va/transformer/__init__.py | 26 +++-- .../lingbot_va/transformer/impl/modules.py | 3 +- .../lingbot_va/transformer/impl/network.py | 24 +++-- integrations/lingbot_va/tests/test_action.py | 12 +++ .../lingbot_va/tests/test_cfg_cache.py | 102 ++++++++++++++++++ integrations/lingbot_va/tests/test_engine.py | 18 +++- integrations/lingbot_va/tests/test_loaders.py | 12 +++ .../lingbot_va_v2/tests/test_app.py | 15 +-- 12 files changed, 269 insertions(+), 61 deletions(-) diff --git a/integrations/lingbot_va/lingbot_va/_loaders.py b/integrations/lingbot_va/lingbot_va/_loaders.py index 123e9756f..90016ea1b 100644 --- a/integrations/lingbot_va/lingbot_va/_loaders.py +++ b/integrations/lingbot_va/lingbot_va/_loaders.py @@ -19,8 +19,9 @@ from __future__ import annotations +import importlib from pathlib import Path -from typing import Any +from typing import Any, cast import torch from torch import Tensor, nn @@ -108,12 +109,14 @@ def resolve_checkpoint_root( ) from huggingface_hub import snapshot_download - local_root = snapshot_download( - repo_id=value, - revision=revision, - allow_patterns=allow_patterns, - local_files_only=True, - ) + snapshot_kwargs: dict[str, Any] = { + "repo_id": value, + "allow_patterns": allow_patterns, + "local_files_only": True, + } + if revision is not None: + snapshot_kwargs["revision"] = revision + local_root = snapshot_download(**snapshot_kwargs) return validate_checkpoint_root(local_root) @@ -125,11 +128,14 @@ def load_vae( """Load the Wan VAE from a resolved snapshot.""" from diffusers import AutoencoderKLWan - vae = AutoencoderKLWan.from_pretrained( - checkpoint_root, - subfolder="vae", - torch_dtype=torch_dtype, - local_files_only=True, + vae = cast( + nn.Module, + AutoencoderKLWan.from_pretrained( + checkpoint_root, + subfolder="vae", + torch_dtype=torch_dtype, + local_files_only=True, + ), ) return vae.to(torch_device) @@ -142,20 +148,26 @@ def load_text_encoder( """Load the UMT5 text encoder from a resolved snapshot.""" from transformers import UMT5EncoderModel - text_encoder = UMT5EncoderModel.from_pretrained( - checkpoint_root, - subfolder="text_encoder", - torch_dtype=torch_dtype, - local_files_only=True, + text_encoder = cast( + nn.Module, + UMT5EncoderModel.from_pretrained( + checkpoint_root, + subfolder="text_encoder", + torch_dtype=torch_dtype, + local_files_only=True, + ), ) return text_encoder.to(torch_device) def load_tokenizer(checkpoint_root: Path) -> Any: """Load the T5 tokenizer from a resolved snapshot.""" - from transformers import T5TokenizerFast + tokenizer_class = getattr( + importlib.import_module("transformers"), + "T5TokenizerFast", + ) - return T5TokenizerFast.from_pretrained( + return tokenizer_class.from_pretrained( checkpoint_root, subfolder="tokenizer", local_files_only=True, @@ -189,7 +201,7 @@ def patchify(x: Tensor, patch_size: int | None) -> Tensor: class WanVAEStreamingWrapper: """Keep independent causal encoder state around a shared Wan VAE.""" - def __init__(self, vae_model: nn.Module) -> None: + def __init__(self, vae_model: Any) -> None: """ Args: vae_model: Wan VAE whose encoder and quantization projection are used. diff --git a/integrations/lingbot_va/lingbot_va/constants.py b/integrations/lingbot_va/lingbot_va/constants.py index 2b40f082c..80d17ff06 100644 --- a/integrations/lingbot_va/lingbot_va/constants.py +++ b/integrations/lingbot_va/lingbot_va/constants.py @@ -30,7 +30,9 @@ ROBOTWIN_HEIGHT = 256 ROBOTWIN_WIDTH = 320 -ROBOTWIN_ATTENTION_WINDOW = 64 +ROBOTWIN_COMPOSITE_HEIGHT = ROBOTWIN_HEIGHT + ROBOTWIN_HEIGHT // 2 +ROBOTWIN_VAE_TEMPORAL_SCALE = 4 +ROBOTWIN_ATTENTION_WINDOW = 72 ROBOTWIN_FRAME_CHUNK_SIZE = 2 ROBOTWIN_ACTION_DIM = 30 ROBOTWIN_ACTION_PER_FRAME = 16 diff --git a/integrations/lingbot_va/lingbot_va/engine.py b/integrations/lingbot_va/lingbot_va/engine.py index 5cd5244f8..8bd5f6271 100644 --- a/integrations/lingbot_va/lingbot_va/engine.py +++ b/integrations/lingbot_va/lingbot_va/engine.py @@ -1,6 +1,18 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """Session-owned LingBot-VA Robotwin inference engine.""" @@ -39,12 +51,14 @@ ROBOTWIN_ACTION_GUIDANCE_SCALE, ROBOTWIN_ACTION_INFERENCE_STEPS, ROBOTWIN_ACTION_SNR_SHIFT, + ROBOTWIN_COMPOSITE_HEIGHT, ROBOTWIN_FRAME_CHUNK_SIZE, ROBOTWIN_GUIDANCE_SCALE, ROBOTWIN_HEIGHT, ROBOTWIN_OBS_CAM_KEYS, ROBOTWIN_SNR_SHIFT, ROBOTWIN_VIDEO_INFERENCE_STEPS, + ROBOTWIN_VAE_TEMPORAL_SCALE, ROBOTWIN_WIDTH, ) from lingbot_va.pipeline import LingbotVAInferencePipelineConfig @@ -242,7 +256,13 @@ def _run_impl(self) -> LingbotVAEngineOutput: self._device = device if device.type == "cuda": - torch.cuda.reset_peak_memory_stats(device) + # PyTorch 2.12 can reject reset_peak_memory_stats before the CUDA + # context exists. Resolve the current device inside its context + # first so every session reports its own peak rather than a + # process-lifetime high-water mark. + with torch.cuda.device(device): + torch.cuda.current_device() + torch.cuda.reset_peak_memory_stats() torch.manual_seed(self.config.seed) self._load_models(checkpoint_root, device) @@ -278,11 +298,11 @@ def _run_impl(self) -> LingbotVAEngineOutput: video=video, actions=actions, metrics={ - "prompt_encode_seconds": prompt_seconds, - "observation_encode_seconds": observation_seconds, - "denoise_seconds": denoise_seconds, - "decode_seconds": decode_seconds, - "total_seconds": time.perf_counter() - started, + "prompt_encode_s": prompt_seconds, + "observation_encode_s": observation_seconds, + "denoise_s": denoise_seconds, + "decode_s": decode_seconds, + "total_s": time.perf_counter() - started, "peak_allocated_bytes": peak_allocated, }, ) @@ -567,7 +587,14 @@ def _decode_video(self, latents: Tensor, device: torch.device) -> Tensor: from diffusers.models.autoencoders.autoencoder_kl_wan import unpatchify decoded = unpatchify(decoded, patch_size=patch_size) - return decoded[0].clamp(-1.0, 1.0).permute(1, 0, 2, 3).contiguous() + if tuple(decoded.shape[-2:]) != (ROBOTWIN_COMPOSITE_HEIGHT, ROBOTWIN_WIDTH): + raise ValueError( + "LingBot-VA VAE returned composite spatial shape " + f"{tuple(decoded.shape[-2:])}; expected " + f"{(ROBOTWIN_COMPOSITE_HEIGHT, ROBOTWIN_WIDTH)}." + ) + high_camera = decoded[..., -ROBOTWIN_HEIGHT:, :] + return high_camera[0].clamp(-1.0, 1.0).permute(1, 0, 2, 3).contiguous() def _release_vae(self) -> None: """Release the final model component after decoded tensors reach CPU.""" @@ -599,8 +626,10 @@ def close(self) -> None: def expected_output_shape(config: LingbotVAEngineConfig) -> tuple[int, int, int, int]: """Return the fixed natural decoded video shape for one rollout.""" + latent_frames = config.num_chunks * ROBOTWIN_FRAME_CHUNK_SIZE + decoded_frames = (latent_frames - 1) * ROBOTWIN_VAE_TEMPORAL_SCALE + 1 return ( - config.num_chunks * ROBOTWIN_FRAME_CHUNK_SIZE, + decoded_frames, 3, ROBOTWIN_HEIGHT, ROBOTWIN_WIDTH, diff --git a/integrations/lingbot_va/lingbot_va/pipeline.py b/integrations/lingbot_va/lingbot_va/pipeline.py index 3ba256355..300fec79a 100644 --- a/integrations/lingbot_va/lingbot_va/pipeline.py +++ b/integrations/lingbot_va/lingbot_va/pipeline.py @@ -18,7 +18,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import Any, NamedTuple +from typing import Any, NamedTuple, cast import torch from einops import rearrange @@ -90,11 +90,11 @@ def __init__(self, config: LingbotVAInferencePipelineConfig) -> None: @property def transformer(self) -> LingbotVATransformer: - return self.diffusion_model.transformer # type: ignore[return-value] + return cast(LingbotVATransformer, self.diffusion_model.transformer) @property def video_scheduler(self) -> LingbotVAFlowMatchScheduler: - return self.diffusion_model.scheduler # type: ignore[return-value] + return cast(LingbotVAFlowMatchScheduler, self.diffusion_model.scheduler) def initialize_cache( # type: ignore[override] self, @@ -138,7 +138,10 @@ def generate( # type: ignore[override] """ assert input is not None cfg = self.config - transformer_cache: LingbotVATransformerCache = cache.transformer_cache # type: ignore[assignment] + transformer_cache = cast( + LingbotVATransformerCache, + cache.transformer_cache, + ) init_latent = input["init_latent"] action_mask = input["action_mask"] @@ -205,7 +208,8 @@ def generate( # type: ignore[override] ) latents = self.video_scheduler.step(pred, t, latents) - latents[:, :, 0:1] = latent_cond if frame_st_id == 0 else latents[:, :, 0:1] + if latent_cond is not None: + latents[:, :, 0:1] = latent_cond # --- Action denoise --- for i, t in enumerate(tqdm(action_timesteps, desc="action denoise")): @@ -236,7 +240,8 @@ def generate( # type: ignore[override] pred = rearrange(pred, "b (f n) c -> b c f n 1", f=fcs) actions = self.action_scheduler.step(pred, t, actions) - actions[:, :, 0:1] = action_cond if frame_st_id == 0 else actions[:, :, 0:1] + if action_cond is not None: + actions[:, :, 0:1] = action_cond # Close cache window and commit transformer_cache.finalize(autoregressive_index) diff --git a/integrations/lingbot_va/lingbot_va/transformer/__init__.py b/integrations/lingbot_va/lingbot_va/transformer/__init__.py index 7bf9ac408..bdb7b75f1 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/__init__.py +++ b/integrations/lingbot_va/lingbot_va/transformer/__init__.py @@ -116,6 +116,8 @@ class LingbotVATransformerConfig(TransformerConfig): class LingbotVATransformer(Transformer[LingbotVATransformerCache]): """Native LingBot-VA transformer with torch.compile support.""" + _network: WanVADiTNetwork | None + def __init__(self, config: LingbotVATransformerConfig) -> None: super().__init__(config) self.config: LingbotVATransformerConfig = config @@ -143,15 +145,23 @@ def load_model(self, device: torch.device) -> None: net = net.to(dtype=cfg.dtype, device=device) if cfg.compile_network: - net._forward_blocks_video = torch.compile( - net._forward_blocks_video, - mode="max-autotune-no-cudagraphs", - fullgraph=True, + object.__setattr__( + net, + "_forward_blocks_video", + torch.compile( + net._forward_blocks_video, + mode="max-autotune-no-cudagraphs", + fullgraph=True, + ), ) - net._forward_blocks_action = torch.compile( - net._forward_blocks_action, - mode="max-autotune-no-cudagraphs", - fullgraph=True, + object.__setattr__( + net, + "_forward_blocks_action", + torch.compile( + net._forward_blocks_action, + mode="max-autotune-no-cudagraphs", + fullgraph=True, + ), ) object.__setattr__(self, '_network', net) diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py b/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py index 12273fb98..97c7e3c6e 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py @@ -23,6 +23,7 @@ from __future__ import annotations from dataclasses import dataclass +from typing import Literal import torch import torch.nn as nn @@ -124,7 +125,7 @@ def __init__( cross_attn_norm: bool = True, eps: float = 1e-6, apply_rope_before_kvcache: bool = True, - cp_method: str = "ring", + cp_method: Literal["ring", "ulysses"] = "ring", ) -> None: super().__init__() self.dim = dim diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/network.py b/integrations/lingbot_va/lingbot_va/transformer/impl/network.py index 8a022181c..fa1adffa0 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/impl/network.py +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/network.py @@ -19,6 +19,7 @@ import math from dataclasses import dataclass +from typing import Literal import torch import torch.nn as nn @@ -59,7 +60,7 @@ class WanVADiTNetworkConfig: cross_attn_norm: bool = True eps: float = 1e-6 apply_rope_before_kvcache: bool = True - cp_method: str = "ring" + cp_method: Literal["ring", "ulysses"] = "ring" # --------------------------------------------------------------------------- @@ -402,15 +403,28 @@ def forward_action( ) -> Tensor: """Action-mode forward. - Cache extraction and writes happen outside the compiled block loop. + Cache extraction, current-video concatenation, and writes happen outside + the compiled block loop. Every action denoise pass attends to committed + prior chunks, the matching branch's current video, and its fresh action + tokens, in that order. Returns: Action flow ``[batch, L, action_dim]``. """ assert self._parameters_updated_after_loading_checkpoint + if video_kv is None: + raise ValueError("Action attention requires current-chunk video KV.") + if len(video_kv) != len(self.blocks): + raise ValueError( + f"Expected {len(self.blocks)} video KV pairs, got {len(video_kv)}." + ) # Extract cache tensors (outside compile boundary) committed_k, committed_v, cross_k, cross_v = self._extract_cache_tensors(cache) + video_k = torch.stack([key for key, _ in video_kv]) + video_v = torch.stack([value for _, value in video_kv]) + committed_k = torch.cat([committed_k, video_k], dim=2) + committed_v = torch.cat([committed_v, video_v], dim=2) # Compiled block loop (pure tensors, no cache access) output, k_list, v_list = self._forward_blocks_action( @@ -419,12 +433,6 @@ def forward_action( # Cache write (outside compile boundary) if persist: - assert video_kv is not None, ( - "A persistent action pass requires video KV from the same CFG branch." - ) - assert len(video_kv) == len(k_list), ( - f"Expected {len(k_list)} video KV pairs, got {len(video_kv)}." - ) for block_idx, (k, v, branch_video_kv) in enumerate( zip(k_list, v_list, video_kv) ): diff --git a/integrations/lingbot_va/tests/test_action.py b/integrations/lingbot_va/tests/test_action.py index d42079ad5..64802fd53 100644 --- a/integrations/lingbot_va/tests/test_action.py +++ b/integrations/lingbot_va/tests/test_action.py @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """CPU tests for the Robotwin action tensor boundary.""" diff --git a/integrations/lingbot_va/tests/test_cfg_cache.py b/integrations/lingbot_va/tests/test_cfg_cache.py index 185f9b77c..392ce0562 100644 --- a/integrations/lingbot_va/tests/test_cfg_cache.py +++ b/integrations/lingbot_va/tests/test_cfg_cache.py @@ -1,8 +1,21 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """CPU regressions for LingBot-VA conditional and unconditional KV ownership.""" +from types import SimpleNamespace from typing import Any import pytest @@ -16,6 +29,7 @@ ) from lingbot_va.transformer.impl.network import ( VideoKV, + WanVADiTNetwork, WanVADiTNetworkCache, WanVADiTNetworkConfig, ) @@ -114,3 +128,91 @@ def test_cfg_action_branches_consume_their_matching_video_kv() -> None: assert network.action_video_kv["uncond"][0][0].item() == -1.0 assert cache.video_kv_cond is None assert cache.video_kv_uncond is None + + +def test_action_block_loop_attends_to_committed_then_current_video_kv() -> None: + config = WanVADiTNetworkConfig( + dim=12, + ffn_dim=24, + num_heads=1, + num_layers=1, + text_dim=8, + freq_dim=4, + ) + network = WanVADiTNetwork(config) + network._parameters_updated_after_loading_checkpoint = True + prior_k = torch.tensor([[[[1.0] * 12]]]) + prior_v = torch.tensor([[[[2.0] * 12]]]) + current_video_k = torch.tensor([[[[3.0] * 12]]]) + current_video_v = torch.tensor([[[[4.0] * 12]]]) + text_k = torch.zeros(1, 1, 1, 12) + text_v = torch.zeros_like(text_k) + cache = WanVADiTNetworkCache( + block_caches=[ + SimpleNamespace( + self_attn=SimpleNamespace( + n_committed_tokens=1, + kv_cache=SimpleNamespace(_k=prior_k, _v=prior_v), + ), + cross_attn=SimpleNamespace( + text=SimpleNamespace(_k=text_k, _v=text_v, _n_cached=1), + ), + ) + ] + ) + captured: dict[str, Tensor] = {} + + def record_action_inputs( + x: Tensor, + timesteps: Tensor, + committed_k: Tensor, + committed_v: Tensor, + cross_k: Tensor, + cross_v: Tensor, + rope_freqs: Tensor, + ) -> tuple[Tensor, list[Tensor], list[Tensor]]: + del timesteps, cross_k, cross_v, rope_freqs + captured["k"] = committed_k + captured["v"] = committed_v + fresh = torch.zeros(1, 1, 1, 12) + return x, [fresh], [fresh] + + object.__setattr__(network, "_forward_blocks_action", record_action_inputs) + + output = network.forward_action( + torch.zeros(1, 1, 12), + torch.zeros(1, 1), + cache, + torch.zeros(1, 1, 1, 12), + video_kv=((current_video_k, current_video_v),), + ) + + assert output.shape == (1, 1, 12) + assert captured["k"].shape == (1, 1, 2, 1, 12) + assert captured["v"].shape == (1, 1, 2, 1, 12) + torch.testing.assert_close(captured["k"][0, 0, 0], prior_k[0, 0]) + torch.testing.assert_close(captured["k"][0, 0, 1], current_video_k[0, 0]) + torch.testing.assert_close(captured["v"][0, 0, 0], prior_v[0, 0]) + torch.testing.assert_close(captured["v"][0, 0, 1], current_video_v[0, 0]) + + +def test_action_block_loop_requires_current_video_kv() -> None: + network = WanVADiTNetwork( + WanVADiTNetworkConfig( + dim=12, + ffn_dim=24, + num_heads=1, + num_layers=1, + text_dim=8, + freq_dim=4, + ) + ) + network._parameters_updated_after_loading_checkpoint = True + + with pytest.raises(ValueError, match="current-chunk video KV"): + network.forward_action( + torch.zeros(1, 1, 12), + torch.zeros(1, 1), + WanVADiTNetworkCache(block_caches=[]), + torch.zeros(1, 1, 1, 12), + ) diff --git a/integrations/lingbot_va/tests/test_engine.py b/integrations/lingbot_va/tests/test_engine.py index b8556757e..7144dd27b 100644 --- a/integrations/lingbot_va/tests/test_engine.py +++ b/integrations/lingbot_va/tests/test_engine.py @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """CPU tests for LingBot engine configuration and one-run lifecycle.""" @@ -29,7 +41,7 @@ def _run_impl(self) -> LingbotVAEngineOutput: return LingbotVAEngineOutput( video=torch.zeros(2, 3, 4, 5), actions=torch.zeros(32, 16), - metrics={"total_seconds": 0.0}, + metrics={"total_s": 0.0}, ) @@ -68,6 +80,8 @@ def test_pipeline_config_applies_every_model_override(tmp_path: Path) -> None: assert resolved.diffusion_model.scheduler.shift == 4.0 assert resolved.action_scheduler.num_inference_steps == 9 assert resolved.action_scheduler.shift == 2.0 + assert resolved.attn_window == 72 + assert resolved.diffusion_model.transformer.attn_window == 72 def test_engine_is_one_run_and_close_is_idempotent() -> None: @@ -115,7 +129,7 @@ def test_validate_input_images_returns_camera_mapping(tmp_path: Path) -> None: def test_expected_shape_scales_only_with_chunk_count() -> None: config = LingbotVAEngineConfig(num_chunks=3) - assert expected_output_shape(config) == (6, 3, 256, 320) + assert expected_output_shape(config) == (21, 3, 256, 320) @pytest.mark.parametrize( diff --git a/integrations/lingbot_va/tests/test_loaders.py b/integrations/lingbot_va/tests/test_loaders.py index f14bddeb1..e0884e017 100644 --- a/integrations/lingbot_va/tests/test_loaders.py +++ b/integrations/lingbot_va/tests/test_loaders.py @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """CPU tests for local and Hugging Face LingBot checkpoint resolution.""" diff --git a/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py b/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py index 9950763ff..d0f7e677a 100644 --- a/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py +++ b/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py @@ -19,6 +19,7 @@ from dataclasses import replace from pathlib import Path +from typing import Any import numpy as np import pytest @@ -61,7 +62,7 @@ def __init__( self.config = config self.closed = False self.runs = 0 - frame_count = config.num_chunks * 2 + frame_count = config.num_chunks * 8 - 3 action_steps = config.num_chunks * 32 self._output = LingbotVAEngineOutput( video=torch.zeros( @@ -72,7 +73,7 @@ def __init__( action_steps, 16, ), - metrics={"total_seconds": 1.25}, + metrics={"total_s": 1.25}, ) def run(self) -> LingbotVAEngineOutput: @@ -146,8 +147,8 @@ def _init_app( return app -def _step(session: object, step_index: int = 0) -> StepResult: - model_loop = session.model_loop # type: ignore[attr-defined] +def _step(session: Any, step_index: int = 0) -> StepResult: + model_loop = session.model_loop results = model_loop.step(step_index, UserInputEvents([])) assert isinstance(results, list) return results[0] @@ -193,10 +194,10 @@ def test_one_step_returns_video_actions_and_metrics(tmp_path: Path) -> None: result = _step(session) - assert result.output.shape == (2, 3, 256, 320) + assert result.output.shape == (5, 3, 256, 320) assert result.output_layout is VideoTensorLayout.tchw - assert result.frame_count == 2 - assert result.metrics == {"total_seconds": 1.25} + assert result.frame_count == 5 + assert result.metrics == {"total_s": 1.25} assert len(result.tensor_artifacts) == 1 actions = result.tensor_artifacts[0] assert actions.schema is ACTIONS_SCHEMA From d54f3b8eda976dd7afb6f798f2665b7956b00c06 Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Tue, 25 Aug 2026 15:39:43 -0700 Subject: [PATCH 07/18] Add reproducible LingBot GPU evidence Signed-off-by: Jonathan McCaffrey --- .../lingbot-va/robotwin/README.md | 17 + .../robotwin/observation.images.cam_high.png | Bin 0 -> 50260 bytes .../observation.images.cam_left_wrist.png | Bin 0 -> 24723 bytes .../observation.images.cam_right_wrist.png | Bin 0 -> 50408 bytes integrations/lingbot_va/DESIGN.md | 96 ++++ integrations/lingbot_va/GPU_EVIDENCE.md | 99 ++++ integrations/lingbot_va/README.md | 165 ++++--- .../lingbot_va/tools/compare_upstream.py | 421 ++++++++++++++++++ integrations_v2/lingbot_va/README.md | 35 +- .../lingbot_va_v2/tests/test_real_model.py | 172 +++++++ 10 files changed, 940 insertions(+), 65 deletions(-) create mode 100644 assets/example_data/lingbot-va/robotwin/README.md create mode 100644 assets/example_data/lingbot-va/robotwin/observation.images.cam_high.png create mode 100644 assets/example_data/lingbot-va/robotwin/observation.images.cam_left_wrist.png create mode 100644 assets/example_data/lingbot-va/robotwin/observation.images.cam_right_wrist.png create mode 100644 integrations/lingbot_va/DESIGN.md create mode 100644 integrations/lingbot_va/GPU_EVIDENCE.md create mode 100644 integrations/lingbot_va/tools/compare_upstream.py create mode 100644 integrations_v2/lingbot_va/lingbot_va_v2/tests/test_real_model.py diff --git a/assets/example_data/lingbot-va/robotwin/README.md b/assets/example_data/lingbot-va/robotwin/README.md new file mode 100644 index 000000000..3d356077c --- /dev/null +++ b/assets/example_data/lingbot-va/robotwin/README.md @@ -0,0 +1,17 @@ + + +# Robotwin I2AV example inputs + +These three PNGs are copied without modification from +`robbyant/lingbot-va@7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb`, where they are +published under Apache-2.0 in `example/robotwin/`. Their upstream introduction +commit is `5ed0eb32046b34fe5c14f929d81e87ab6ebe02ef`. + +| file | SHA-256 | +| --- | --- | +| `observation.images.cam_high.png` | `78cab76d394114ba912f882ac9b00ddc017f98c946482cb9267c87de82486b72` | +| `observation.images.cam_left_wrist.png` | `fbe55b713e1b3d4505fda6be3b00132213ee1725a78cb357ed2bbd5b3a3a7a93` | +| `observation.images.cam_right_wrist.png` | `b9e6821b38073567232f6dd8f6389b123d09d3d1a70758c27d388e547e228249` | diff --git a/assets/example_data/lingbot-va/robotwin/observation.images.cam_high.png b/assets/example_data/lingbot-va/robotwin/observation.images.cam_high.png new file mode 100644 index 0000000000000000000000000000000000000000..7546ec3aa2064e4e32d03250439bff582b9d6042 GIT binary patch literal 50260 zcmXtf3p~^N|9^FyW2@ABiloG5)7%m#gmRg=Y{bZh2`9NuI7O6er{*%F+$E$FW+~xXzuk-sqn$gVmdB0!J*Yov!zTU4fNLFTIyQO#U*s(*5V2-!l zv16wp`1v2$E^xn@g)Q8%Lv9BFZ)A6UXUhx5edkdlOuN8Z?7qrtEzjLZWc%#4(m8YM zKcF}5A2SMUeKNQ+n<06Dvqd(xZpJHSW$CY7H{VJXNXz`LcgZ|Wk$MrqP1fj$tF|p5 zCj15QE7h5CXgU=?aLQPfDVmtn+_tDWx1D)?$FwcgJWzJ zd_=A|`F2nuNZm?tDyt{FBZ~^Br(K{XzXTo^7^ptzi(8M#Ena#+362s(? zG?6PF&*nLDCW$J#R6(Z%K0 zgkz8yCP@x>tc-x;D3(~s!eP!MWIqsfO(V>MOCTD>1TNU^hw*EjIE#r)B0u!`Sf-+ znGeaBXr?)M+)uJ(IO$TaIWrv5=Ql1cUA$s3g*jnDqu^;2W;LRpif6JLRnWpc znX~^eUJu4y{!QK2Y%uB3^r>^fx6l>zN{)#WL_|II(V)fGLRCQz(J&1C8AP0Vg|ani zu@%FBTiP>QW(Mi}$!JTM5nCp`%FAmgHz#~ytfuh2)zE{y+`NFwOLBYi&*_~S+A53c zV)yRlqin;(1eqizp#{e=F;kSGdl%&rKX}H|=#U=yk&Gwnz6j6JGfr9ah$`xo+Gw|n(@K^@1;?e!avA=g?(mpao z=4|F_qkGtr;`o!VQ0f+4W7n4_xJJ~y$-RgO{Mh2xfczE#J>!q92!-~datA&b;Ls*MDGLlA1faCKmqV0y##A+S}r%q)_8XIHgdTY zlIj*oaYu+8MP!%CDci)4MN|xxfo0jsbnkog__hyZ38yjNH<;p8hIo+Xuq?s8#c?AXS}(`nU7BnHJOT88*|hGPj%)y?J* z@j`s9lwXCWZ%6e(emh$dTwXu;9jZ&Is{}tu9Kd5F#Z&KLJg*c;(X_fa-qYO$+}t>a zv{lX;$AO$;%AwMgSdc1)ZmS?(+z#X6Jz_}3^k)V16ida6=e@i}13uT@SXmwnUYwbm zyB)T2_wJp?ik#*Z(*zP_})RCPD+Oz6hfSCjUm*$b^wm-I|UAwt)fclOZcOSM*7 zp>cOQ1Ofm!F^z=dHT=Pb`1hhSGALLgb|MB1-L}h;O8=-#Wm4QEi6z7mDg(;IIk7Em z+4K~eZ;v|2-bQvKJD4a+EA$kYw-C|h_;#%hVhgPs09T<%goWivnmDoO#OGzDir`xb zAfNluN*&Z!Dix6XWAH+P-M^TOdn@ovY-|fylImF;Wz0cdZ9k(61@NQR2$Hdy1rSaE zWc&&(DWmb@CL}n|sFhRVw@L*J<(`e$ni}uy%nWe3mJqxDSmci%hYUhjzqiem=UO89 z{MH|xqwkNMy>azw=-TpNU!NymWaOn;&r9FvjRE)RDC5)K^xVNt#5MxU|tY z$3&SThOvVu5>$vJH%d2&T|hTyfG0r{G1ayVC6@HKWw%nwY7&>VOeD4tRpRgc3bNrA zshEVhf~063k>-{vv+$3&k}L{KvYhM4riknLIUiy|=u$lP$~iKla(Ut|45V#W37v+$ z=uMZ4+m9`eJ2DaTk?Dx)j=Klx3F0UqBz86;Ei6c)q5J99D!aWE70xB)6r1=YxHd=0XgjmHwl?!3aE zgR@H0>^f{SiD@7AXcjw8+g*}8gyYc)AIsqgWC2F9fUaS~aQr_E!7*+&C`oc6nNbaa zOznxL5j491#vq~!paCC(7|w|ihi8!C7)kK5q!A*A4)Np2jVdhs)GIE%C%szX@Z8MQ z#@D{_-l52a#qovKVB=$v^Z&e4tKZXg?>iH=HD8nZ+S=OMwKk%v z&%aXUl3xEE^^@n23`fuUA7%9jB5pdp`d~a%Ol>T{Pp8qjEiE`WSJ#*~vY$!z71yqi z6yMe_hY3K5;r@O|BOk9qWpW)cMO+gstC20q=qaJ!PZx$>mSo~|rNEy_hF^0+2ru6c zFkm`XojHJWR#c?&AttE%7zzgDHTy(bJJ1ghk)w=EBf)XXHee&l;~iTdfPO5C4rQXG zXSK2cF1M$%10Fnh;IEZ!dE?LR7nxD>$6X`0ehlh`+}*b?`DM}4^58&fVd33l29aCy z{|@l^(^C1Dg1pkKIuB+&PQyp8uWTEN-^YW>F=ET5n{vqBaEvFbF`WdW)rImDrz_FK zjc`sAcYj{x72`%LiEB<+ngo$!z`^^f9ZZIpwCOG-PnN!1mz53sT0x^GP)s<61xqqH zuA0Sr23YRID;I3J66+C{R!HwxWHqXES@FP1OS`yu6mBY7I>x94QfK7SE@)>pqHM)@ zm7a|2t|};uoqR=U+P!Fz?BRd2Gl$NEF0M8=*YNq4onUo!s~Zby`s-6$wd=8>qMd0z zx~GS7>aWku&!rv*Ozm195mwoES@*ph!+5j9^^za49eS7!OcFlnpuL=O2= z`-h%r(k+ql^KNBWIN9&jwW3<6QHfghiVdnlH;P4dlbkRfA*#BOfqEf@neb3?${76< zUFT%z+!Z<$&oWiTF;+0B#&aG0OcT|SbtTPWwscWHV>OAyr$%R(<1Z*8&9UWJDn`<1 zzsu;;@=X|>}9J-W(AAe+gEP^J^0#xIi32CI>iV8uTVgAQJzd?!cd+7q&z%$3$Eb2J&p%5SNN*Mq!FCf+S*d~>-Sc95A-9}mj_?G zcrjQLVi2}EetiGTP)*p%KYyUgTyppu7N3F&fa_tIDIDwZqRxRFQh_xoS~RoNDk zaGHBjWIp8r48y`dFT|_jY#15s6cF2=msUY1R^Ti^txSOdrlC=QTz4ZEllUnlKG-J5 zViEN{m|#E!U4J^^vKk8dIHnwNBt#P<@qN*{%G{ zvO2vR8yl|{DnstfhF{%&QJ8mT<7?`AY;3IVgZk@BV>M>St*wW0@^S;JfQ+xWwAYXD zSaIL}{e|QIeE;X4vF~GJGgl(FnXWyu5EJbQ1P_*lSH5E0LFrzx!3`p|o2ggMx!H_F z@N#>UPP~!@yosUswSw$?ETPBJFQLo%bMD5QOsA7Z!a(MkrIWf)REFd8~LvQh_Zx5=fTU3C($D}!Q#|oe-08cE-(+`j) z25^#4()Sc(kVb4txNa8!rh6vXqWfZ??nXmgPo9Yi3h-?5 z2x8NMQ>8N5{_$zvCLFv|F~r!za12-bsaAj@W00Ng6CKip+?(c*_E4s6pm+AOzP`?l z@cGs0>=02i(UqCcn^VJH!iBGX9~lm+W1O96Kn}-KbzUaWq&0#Jn$Il3IO`$)+8*b0eGhiifcpxf)Nw z19%HAp(P&D!{|ShEI+)0G$$s63N~Rh_W4-~gm3+A#OIWsp)xaJOV>3M&|u*8YLm zLJeb7`?E{!+{%K998lC81OV|kGQu+Gq$w~bXK#eg1K9@vs_&tzufH|bX)R39;o%p+ za9TZha3*SNqO~T(YoOIruX4ao58}62it=CH{_^EZO~~x$s6V%vYIQ9OvvtL_ABaH+ zra3&F1my4~Nd|@E$fzaZTB$6E|1ld_(>N1MQHuE#9c9PqJ6UYg>p?S{LU)zGB#5|Z zlwUNl0?68YNlnbieRrA}HW~GSUFrijnXY&=fpZeSV$5+=W_bgD0vBM%IdSF}U`sm* z-W(HOwGPOq(W>H6Of5c}#He&jmh<BLXggHlOaVryGuP9bn?k#SI zl#JXY4M^T}3L8^_o5U5>GHj`%0}H90UiIsnvvm=R)1LgZ`e%SJ9KL%jQsbPcnrqz| zowE^}6H7mW!z#1<%*+NX;hkyjy1J*&oY6f~7dSeTTa%|BzIw=Dt2n1VVxR&N6Z*SE zp#33Q55!0N$x}OsQ{+^V?QJ{x?t9oGBkxDjED^$m-2!h6l^PEO9FRJ9nRfCAc_zvi z%}a=!72p{%iMZB@`@j=I7;=Lg`7tQkQpAM#Two?LNNkgIIV3C*{D~@%7@Eoe)p9t7 zB>5IT-GL4@x`5q*li3n!-XJD!z#=JtBx8BnuoR6zjaDyKf-qP(0MJx4*s6Hw0LT(I zvRk2ujC@53BSrl;m%8vD14G-(z4bqO52?9WJr-S=jhg?S${1C1ITN)!H4XH!UYl07 zdjFk_dnrEm_dULypZ_vH|4ZL((Utm*dF#meUW3+;{;^~>zrBe||4;O!orHO~A(+)g z^k+rV2QZX$pm8W*3viP}h6#em)_moC!0$2kB7*3BghtVJPf6)P`6=R5-{Rq#d)Ppd zdbruKHBI38kEH<8(!IgO13;}*FnU~4DC~4jE(Oav0cZsf&oY$>G38PZ>{UClqERIcuWaKc{u>KiM!GjfX&EUg2S=D6$j>T84M;JvK$po-<+fWpU?b4(C?$WAJm_6Y9{RBU5O0iK4?aMZkYU-Ony zHw`GZaEO>@w+Ee0gUq!Fho660ze$nwDx zIL$LC^rgtG|`TriL%R*w#TUg>8UdE%$(r}#{g$8TGS6e7niW_z3Z9v(WtHN z`i*z>&m%Y2zJLGz;`aLaeE=v|)~5Ofs%D#jy-%_13<|2Q1}QN-+_4En=f?Z`jUPiB z8%(ur6~E~&b{KFvKs<=k#Nj|xSTvpko5IO`mIf2-1{~qcLD_j*3?3{>iK{sGL5n5H zAQ6cbSiyNW30o$hE~Y~lN*~ZhT3Q8#Wzt;Y2g&vjIS}6I77(1YjFk2ip{dSKBMqGg zi_d-2QqqXpJ5mXt4gNAVy;?Pa9S@zyf!~3DSR1=|IG~mzG;uv~B0atbg_2}Rs`5k_ zAH%|yXSC}#TkW@3t&dyhnFwf|&tGtoooHaD&P=!1Q<6 zOb}s-aR9pDaR*QgES%wDMnx82!D=Uj-H}OZ#YBu8%9go$6XKB|o}A>hB8t%kMV6-~ z4ATwck#3QIj{uoZ7!eh2;qIlE6%zLw3xEOxLWn`}PBzDa9adtMq9wr|928G+Xryt( ziH(ha;Isx>wOykYy6v|&ma1F>w6X!G167)rmj}Efpnf<0V5MjB(XW33p3^<^0Ki&C z21vN=vdE3hsP!taT&5aWcxTUk8(pD3HaOuA%z8A$B(}Jz1?>fPjcj8-@{9z?3J%h- zx63c99e4;aRrS6-!u2Nf6hNk`6VLEKk7@&$0&l{&3yePm>_ezE{F;Rg3Y4+9?b!eG zLPPc-Ep0K6NmXgpUAFKm=MYQ{hNB9uNC~J=i4^tF(6uSo?WNxBwSl4MHx|DABUF#a zE(zJHdAxIhzW^!4b_QL)emy)qC_H?0b!=>Pwe7C{_SEdwa{b2FvaR{8&GpV*8TZGk!!nNLVz!SY2tvaZUDhJaad(D2{+#1?iDT`VvE-ZYnTPX?*(W%|DC{Gz^ho{AEBsw`D$8nCHoaOq=m z!chGus(yYG-U(wZC2V>C|GReczu%9zWwL3NA6;;fG0Aa6@ZojCaJQl2rm@pEuFiLtZT%Q98+uS5W)JE~Rlv~M8}nbgC+#oQr1u2{1ziVm8*Q7* z{qp5y{+IUve*uxp{Q9jv*Ql%8sXq5D&-bsbV4FCkS;aYVmgL;dYZ&nE+9CpM2T_GB zaY)Gj5Dz#Nb>*mFpw03OIt0QJLj@%Zd~Ih&uWg&C zOb|Ioa6B$LE{zcDn68x7g7bjNcMIbC9xYAv@m;sqdR?Q|r?a)ow6hn`1Fg$n$D5gl z)U>nDrQ9C>XAFq@+%LI$D>Fmu@9I~XdD|NS+l$uQi>s?~&1k<(v#zo4c^tc`Mxoqw zNC%|}SgL6|(0gG%l;n|^lvRVN7mSPxy%#YbBmD9Ec6i0H3sOp6NMBgm-nN~lh4ejf zPBVebQ+C85fB$@T6Qu!@Ga}`XKTto{i2J2_Gsv9Vx}1Dy0^4Q7$tR6i01;gKu+ahp z=0S0pq;}eBDK0`FER&#xR+mUJW7TDx=GF#R>UuG1`z_u$U{XFZ4@KYOA3Ag>YV~FP zU~S0C+?Ut(_L<#--7{bA8r)d@{-WDoTcdJddc3*0udi>suX()r$3Q^+QZd+@*{J0O z?a$$XmwyA$^CTWkZ`YaJPH88Cw^vlW0`lu8MqHpSBOb9rLYj*w9>ps!vpcyil4h7L zn{el)TaeoBYX{4O-4!BsF3OQnMjGKXuBiEAJ=*%71_ie%b_vu?r3U;^m(0EQzNcSb z9Ba5tV=CUa8IifR0w>xt; zrp^wIabeMQcQ5S*C=;UPXI_Ag6*1g-&XRK*JU03hfy7V$mp^b!-Cr7K4uJi5*sqg$ zqpPh)@X;cnH8>%;JoA_L2`z(HuUwf6*xs6zja+^4<+!z4*@*_q+;Q70il~r4t)%&(K*RW{;2-ETUwO49U8=u3xMn?}D3-JAG z$X1;KM4uLA!GpnA{y(?NFY$oLIR}bzS9`h2zlp!B5>j$+V*^rdQV2^`?c5mCWqGjClEA8`P zb7E+;LO!i5xt}?zoB6b#moa$ZF@kCWKiA}0bM~x`%)q(jnC`!8K1F;l+`s)}dwtZP zZhPzY`(tgjCM|8ngZ_>gZ3=ytwMV_Y3cW@F)ozW01jruRdKorb7ZMt*V5I!U+{xgP zPGSASE*FL`4@hNjjgYWVTX1gssJUdER=VAvoy$rNb>xs`?4GME<7A?R>D_JfACY80 z+7Z%UaQ~!H)^OhWbSKoYF`1*;oFk|RnS#-2hW_G+Sr&{Y_0md1<>=O z*mZNh6@dP-wnYD1L1m<2E`O^e;hyFP$06qKpFuYR-9X!J5<@ceW3fF9P@fvN7PPJ# zxsUJ@RM<6R#Dp)L{JvXkZZj@FG+M5lHW_o8o0-1aL<~P-grJcD;}}&4v_~%l7lZxF z^2EkdyJ?rT3Xcy=F9-1lJ(JWkZ>+q$yEUG-)eADLap98jswWC&2>-F@`zBiY_$O|!YS>Te==mNdJX4Ms9j}p+p36K!4LYD;S z@4f>x;|fBbmiXjPk4WuXu<|CU8AWxH;_bPA|F+jO#oN@#+8 z=yCmz`6S{iQ`*-2H9#_+RuUemH`*|KF%_I6!dU3WXLj3ExU@s8W$7X^Y3>QyHt`5e zLYjpY?m9%=h)!D#bekJ%Fa7_s0A()7x(r=zEnuZ7Ob1vOsaQAu_9Nxo8jq?0=jqaK z7utyj3nbiY4ucvLTq83e_Ty`+_11XQ*8A;M15km+=KMpRIt6#0@g6OC`|)xUs8T=r zek_0eI$briwK*HP0<=2-)F-KnH1~vIIloQ8#9y;)J8&fV(=0+l>|!UvdUuo6_Dqbn-`+mF!6?JHHCS>;cX4|s6FCdFfE$+3pwH{hJ^6kFUfHCV4yW z!PLoh2|`bR-Z9bp5l~EE5p19hfRL(iQ&T5s;*>!viTJz()UCm*f|?7#do-IK9t@oM z_WaqWnrHof+5_63uU~S^FxVL0zr8%Wz4d)KzxZET6*V}p`zaBnbtxz3!DroiL7)Fn z=p+xc8GxI3H8iAOpRPWEO%WIMnJjZC&aGp{s|lO7Le|g)^`GdCap~#->{3V6Y@tR! zq(g7brwZtyrHwZ5YsOfxAi_X9?C0~q%Dgoy@Lc-Z1qN9;t=(L61aQBZpUkg}QDV3r zFPoSIx-8x@%0q*%VYfY71=YOMcFJ$KA z=j|`+pzl@l?D_NiHQgGSI+-3B@qZ6K$ax^V$gFcEWLC6967zzD#fU6ihED)>~Vb> zTv>NW!qozf;jG$%y+2Ck0mH+q%c5b6OzD7`$}>aLNow&uE{HIUfE$1UB7p~Zm&@kh zaqtnUH^Z0Ft-`B~`&e9SClC5EY2a_@kZ3ur*ad+GmBP07F^zGAs%u}PGx{X^oqyHT zF@w-~aJh$MT`yb+pa1@1^Ic)4#em11*6M4M%fU53NMeJ05s9IO zu4+kzIZK&Gs&h|Cpzhi*%G#44>0VIiC~UeT3r(?^m{ewXreJ-PJCrZVK+>ckLpGftj;*HJ#IR>U-?@Opsqd;mC%i`;g_+(#eAx% z-u~wWIHb(*7|-D=XM^@x)urD-fvl?OR}DPzuFK=ZyBu#lpf{R7k52|X-vZpbR#m?_ zJy}+--t;Ce-@Wlg4&SeT&9_%)GR)41ap-5a`%ztDkA;wc1t}adx6Te*Zf{m@FN)kp zy>}qJ_atJ7bAYo>=2CSnh&ko*EpEAi#mY`1OfOW7y z)zTZ!S=!`2gq8}3<`K$da5D1|oF4GVaF6O9XvuQ=m2wx5OQ&H*p!#uYdji!zP`b1c ztrgtbHb>pjsZYMG8Ew-0M&y>kIua7x6Ot3qPv+THhpFH&j5IHzrt=533g=MFJJD1URBXaV|!EKI&{6_b;G)?K%Nltfsq)A(IuWLQ04W#n#u{x;^zit z9SDr498K^J$~gUICs>pCrM()aQ_=kvrf#4kiXM>zy??0q`iZ%2T5#z}Uo?0jhPmE= z&S>XxYgzs9Kf$TRe*KN_BO_m4-<6#X-s22p9YO2T`ENNuitz*6?(;lO1X?8|Pk#ig zmNWEh>2ugDs5SN5nWq>8^5A5!vJURRY2ptfr zZ%zRlBZulY1_Pqjdz=F9;>~rB8$nli-GB=OlQ~Pjz?8c705ZZUU*Umn^*&(Rv_T2n zhkO1|IA(_D*}m^8j&?ikeStaZSyT%n6s_%M-GR;(kAW!Vn!U^MD;&e8#w8Z6T0 zDV6&)Vu&RW{wVs61{;lRHrFuKFpcrjz{inGa3dHKiQJ$zkF1)(ZhuEzi>AL1he6Zb z{wu3&5DflV2b!gf z_SU@nxB1^x(Y(*~D|4>~_s>R#Zg;-6KK9-tCWRhqiS$>slW;nwz(|;bpMd92K^?)LV6YK>b#wI{o_NigQj;OW#zMUc!i+Ody=^ z;3AA7WLV;6OHV@;8u!HO#*<5EWfK-29I`y${%PAsQG_y5SJ3%ALfR6SQU#)mj=JfkIfsbsa_q+ zzY8*J<)0U~i;HD2a8+^Fc8aEk)@`5^Kt0YV5cT#p217PEIhWrDR;ErvX3R|G^j2^aB8K^pa1!M6)zIq!0El?%XNtI|Xuh9XYH7RB^%Rc|%^ z&Z79)2$U{DVQ|zNLjKebXa(VZg47t{MXKJIiUx3ry@A>=nS64gNGp+L~JB^^Q zZgYlLyrlTy!~BJ-eX2EsIfHzf4I{a{?{b@!_4~H>pjV+3*hc`}E#I=i$WhQ{iCWAL z`Fvwk(Cez#_rSddeyxk>6!p8V*XFNXN-jV>2U~J~e0GZNdl;mx3lpe$8 zxhHf`zk#9)x-~k20>7!mWOc3M#|QYpho*LxA6_t2Zu$sxYu#Mak&wus(LveG)m*)R zp~&fF&l(v*M_R>_c7Z5;?XQ9d!mhk4_@d%4KgFwdS|b-2PDWCrOOO&bnR}a5{AhV1 zeL@1~Y%$`YN>eN6R%pghjD1FPx>PjJX)+yKbOe0zd9t96v-}~wwp|KGaOyGwJ%C~p zQZG&I5}x;smR0jKY;A13aK)guYHf97_;#$l{T}iyxvq0$oO3Y+@~O#w=i=p6oU1DT zM(R?{AUK)>n!D*Pf#)llf{V13w1tzJOgk8eMI{4l>9qu{sM>Vw($ zEho|(%>97gkC>gj(3OH7P@^DJxWod@y<@E&uVdbGP20RVNyjtnr=fBi9!TxgqH zurJ=0A1K-VOZP7=4BMdHd~8PwUZ#Q$`f0+A@xK>1b3#q_Y$On)x0%-Zb=B3^!DvtM zyW$Kkd+CB(kJ5wU+ih&#$yS}3AeTv+XBUQHkDbGSgCpLE+#Ona zsH3k$v7!qP%slPG5rl3IKbZxE0qom((;Y?q1f1JC$2IlTy?ngvx&3`?>ice+Ne&%6MZq0>FSK9lP< zBWnI_kEC6Np7IlEL_&D?)La7Ox0_m;xhBxw7J7$^0#q0fHyOC*JH!&VvVAC! zMxD%f3azP~5&Af=2h^~$C(rRyOWP^5j0|=K2;M6fxGKwENhz!5#`f^${;kDAaQ0|0 zSRb`LLF*+x=UhWnTx-{2ycd`Z=oHclAE^X7e$(*Ee0mS>o^V+!o6qMjEI1uMemqdv z*z4EPK+7jP2vU;pXq~eOl&d^_xf{LQ zfbOEvIArJI+P81I{X;;{S9W_&RyO>5;qkl2-a97K9mr=7a47;MBMWZm$y^BXY~v~h1BlsS;=p7{gl(W zUvZy;urCC5ofKt-nfMjZQ#7cGSe&ym@e!VBJCbbm8zr3Tz ziz@^)U}zW=c5z>uPV3Ww);6*|*Pl}hGPsN`mvUjc^u&o1TdP~^QEQ!|c@fKhRyn00 z#C>W@h~xu!Cp$7wDqT9hnn3YAaiZ{Zb>s$GzfDao3VK}ti?*~4`u9?j)t}&&5j+tc zsgz_OsY65W|F_xSZNZtm&?r~48he4zw}%Oot{96GIiQzrQv1+Fb$Q}fLU<{_)R|0g z9<_`@kiX?1r~~FdqLeol)CC;P(;(`%(!k(gt`3G8rq0Gsnb5hW^qx^L*K=D~DFpnVrcuvkq!XN@3HR0?F4@E5V&*w=LTlk_~3!R0ES$$E#S|5ar?Lw1R=qz&T0A(@O_Oq=b9 zuf+%xP>gB^%>uOP? z$8*1MPb(G00LM3on26DUTpu^uOg>y5$%?D(67>5P7sR(qP=hLI!PNA7fx4W&BG2P2 ztkD}FAoeDx<0-fHf}RbBOz8eJIrRj5bYSkK=>Az3;BePFUV+)T8yoIzA7F$e5ueG2 zVU&x(lE4g?1_x#Zx;Q(#@Pe5$iLK_s=EM5(G&rJZl`04%nwpxTV5Ud$2vPMXRehJ6 zJ|=*pvqNTqRzpU>#R7BwMbvKYLPY7k#$MYn`8k@$NqD#2C zzy}VziYKV>M9?9vm2;OfPEdaXoP-D*v`~`S^kxpZm{O2SYkN9S6S~^xwIUn7HXJJ( z@qIXUWp-%x@*63~8w%bX6g_5T|NUIrcup?xA2owBfMZVZl3Gs-^R4Du=+%@*d{l1} z41Nnu!MLR-b-(dUNbde`hYRX>Nn9Y@B<5a%9R@wcBg7Gl3QjFF@vwkK?x%I=Sd6NC z%NfhLn9-G1aZQ!g97j%3c&Q|=p7K^o%4iwP(jHC&!v%KW3{Qfp>E)$Y12XS{{*BP> z7XZPlo!dXY$j(NE*7Gm9|4S9`CkAOcIk&P+l(KmD!8wN@(CU6dt0#Z(vIb2HkX*F>%TV-0kF6gWv~15IvVym{OiB&}P)+&0mhEaAwIJM{#2=O~@B+@6s{N24wY zBdy({6X8?@=H*7WqL%+zdRNMS-nq0HHcBnF;@T22Jih#4#rg~J8)5&L`|S31=dr_v zmv%|v2KN1vYNu;)Eob5BN=|Wxoswe;=F-^Cv1hk7vIvH-H^#f>ucrK|ASeH;`=gr= z9WYA>>VsV(X~{^0X`-o>CS>kw_-;2rjxKp@w*?%`Pg|%Sr2l@=WGBYxS9byo_6Chr zwJWzfz4(%}Id`Q#>hCDf1F1ct9WXUL_%v)~b?hT1qXE0~u;Tt`LXr(^C*~qbgq{Gq zs0c?UG{__AHgP59DJQbjzS-P}4&=tGDZL^hAS*@qyw^b^SRwu9u3vW|pWj5ohT*$* zqvTB{;5Spg-`nBi;{zvHc>5Sy>_J~lqyG+1d6@J^bR5CZ5DBNpA< zZ~YpVa??;k@Th>2WeBApaJ$fG)x@NXle^6=enHC7&%Z6YiAah~6Y08%K%|_6$H8_c z7Z@5D9+Q_Vq)VpVe29UQG0U6nE1Mm2!@Plz(un!-=i63-bnfc+w7a z#IfDxhUN{}Gih-1|A`3=f7DnG&)Lg{G3JvbcwCZYk4fhUQWUq^$VGg5XS(X0;7g&#fNnwF!dbEo$lI&nF` z-_PV!lxU0dfH}Q zG4=GEoDNlL_0+slQ7Zc`Qv8*<8!JMjVq+$(@2s=d{&%_jUI;(vukQu%3A=XfM9N*Y zYDBb&Cy5}1Yf=1F1eue1Dk*vQPH0yQ1MA}e9`%XKIgpfmCe^Y54?A!pAqBt{Zn5() z1N8)Y_$$)w-W&8EVid7kT}v@eplz3oVzrD)nPjou+GHI*LE1^;A6=3C`6Mch@~F4kXY z@1eY-GqT6P$kWj&S=p6ST0Ppql_6SMy-t%bR|Rk>NI7|(<8mVI1Sk!l{mQNG5UkMyN{;<+*Aa+aPauGk60BE)xe-zhJ$!|DYxiHbq<0_P zNp#Gwn_ZEe8hU`35e`o63AnVUGP*oBY}>7gN7qg3>z*Nj}eHZ0>d3h(yprtatC%IZ68H1 ze9gIelbz5Ik5GU=K^H=R1|QNT4#QC%JK4X&C`-msSA(vuY~}j)F6*y-&v&W{05f5Q z1-{NyQx(j`gKdO+=%nA!mJLNW&8b>iTES{FE_FdZAuA)_zV$S%g;w2bedf5Ug3fu~ zkg8(xzyy{jv5Sr#pkQ(bjUU3A%A9@U~v_uL6cIhYah zXBOEsnBTay3wIADc?!#T%i%Vreo0+lU)L_3-JE-A5V_V|C>m>~8swarAxFYYL`oc!VD!#hoM(svLXAh5=qxY4p7`pV6b7u zuUQQ@lE23xV6dIyB!#!e8YvY!aA38=(Qk-`|7$?B+3eCt3EZV&19N!W-rLbUR|`HM zvborK*I;Y)8*kwC>)50o4b*V*O+-pY5&}{{`uLUK3-I%I?yn8{F<&q^XmKQaa-lEh zkw6Zu%F>ieN`9-H(_w3cLI3seVG917@0k%1m^vOA5%pyIkL|I4{^Ss}vs=By`d@5q zhONxncVE^Ts13gX_M_6_J4wFcWI3^JZ!!Zx0ccC3C%=^^&`-kOC`!0v_Wox5pt?XI0BZ{>ghY_dnn|D%hNM`{huy4xz2(psM5`(6Ftp$F~A~Y4K?jp0kd`3vVQ+LUn!qg1Js=oDW`;mB!16P` zL;oe%X)~BI&M<=uLnyd4-jque)l#N_fEVZVf%DXuZ2 zqDM}?VA##?%5cMSfkmb;X}v}pzJ2?4S!>dh|48s#%x@C+ktzk6a!O@?HeoWTzwFw1 zSfb)4;+QxBrBKc_m@#|!^3U)fyXv|+{p?om1ejX{lLa&DEQ}r)JFmOyQRje0UMvzQ zNf#{s|5hgmUpFfRZnLj)EMO8k!DEc*ALCs12* z`pujJ2oeG*SLhCMrx(XE1}BC}>Trki`8$Xt4O3_&`}jf?kNYmGUQ% zaQXlOiPA{U1kuufTl}@-*PX1tZocKe(b>O43Az3!IK>o%t<3B@z9|P_z0s5%psLm5MAdyLZhRe1 zX&z-*(!ZYu3{Ed38P=VGn;zS}GgHp;XBs_$fMkQIUPOB}C}LEi2{=cX2&k-xq(D%@ zK;YytNF$iH&(MB=mRL}P0*3kr25Ra;Ho?bvIhh`_q-} zKd-)ZVkQD71f?{+EB`N=YH~(4I`Y=Lg}_q=g6qAvdGz0p<_lm+4@KxVf$_q^U}7+H z!YxQxr(poUqi@WGy8&6jFWOl|3<06_DeC(l0RiBxmcR#yfb<2i>mM8({0JDZ1UOQy z@m)wC$DPLX9Y`7^cjjQay+eoXL$2)F*4G!HH97NXu;$8_Z@gOH-qNA!D__87#hQs8 ze|?<%^5nt%b^&@Y{PQ(_UwGX|2iN+)h03YmT}@_XRn^e-FB9OXGD}rbSn*f`Fdb^Wk%13T|TPMfvS4ikRm7Im?8p_>t`fIb{}p|b%(Y@#IQTF-k{~^ zf&cO3_1z##2{%EqCZvH}0p>dhPa0(DYcoyZ%M+Bm7cC zMESku<9<(#{fm3(Wm7G4beiN#3p(+U)PXx2Ilf0@E&d*NewSo`zgU=$w zr0VmWpN1~Yr}ujK27WMwBhk-+WM2>gQ8FR`6Vq^BQO+8vywed58&1f9d$~hzzh9+) z2cAuiZYB(%vXb0?k`bngMZrM90D(UfZAoYC=Eieb++Dl^fh1KrxgY}}S8>W%_*+vL z+VG2=oy$dca_-T*Gu2Tm-&&nyBUXo_qP}mL$wq?r=)TJj07I#qxbU$q{NK0P0WP&ye@wLXe7mfl@9iA&L(m%>toB~=_kOgGVS`1H zlD;+IaS(|I*1sa3SKJIQ)hPK%h`=KWA67uRNkzvY4X@*c#o6+fB^{Aoa5L$qN^=$h ziOi6DFPW5XIA^}wNCAa~@4l#*uuDM|DS`ujSqYwhk|>PE(Y4{RvG2>d19_2)9kWr} zL$c3TwmvUS3VEF%=iK|3Bkm~c-?$xAJBf!u{8B+i1&bSVvCd&?YU=rqrX#MK69e`x zVSa&umo7yt015xCZ5d1o3NkWw5ca6OQ2KF392hRb^#>=Pi2PTSl>xse3;98TwRiMQ z)^~K45MM&oJC8=n;a|{?!!MdpcFiATn0|f5A^dL#@Kp-72$I5!5A=czL#gOLiD+?e zcU+o4+4?EdAImF3c?#o<^m0{uBwc58S!ap3jw*EYESA+TcNzr;WMjc%x@gs$kh z#UUdrJs*E zuRuS45lXN3>X{=%UH?03y3_jew!v(~_ZMM#;q&9eFSgqE+h$P2{~uFt1JCsS{*Qm_ zIGyPRTX#6!7@OQhbcc$}Xd7u5GL2L2?>~-4DRi{5u};ig4$_q4o;0Q`e5)lA5+)W- z>qg8?lAGM^ckO(BzyIU^c$_*;I`e+LU$5)Bp4anrUC+yaNXhgm&|X9F!GK3ls}*tn z$8|O~_JI(kK$<%Gz3*vk=jP0R!^mx70Qnxb z__77M{Qn+a4SnuLNRc5^e42FB#tyL=-s;9RV72Lm>%FdDL2`5;L0TtN2+}g}5X9J4|1_l-u`x>WD4X+FxZu~KuoMqVA*C!51t-(~R zdGAY6{B{P(NZI#5<7lD@=_f}EYYVMADk%!svlzJ@qBT-9PW|gx6OQU0M-URq8CxOg zy#jy+tC98A~!{7H@$y}>PI-Z(}HgQtZ)v>anJk?1PQ8TGe%b`SvN3hH-uL{3B zB^X}mi~o^T*ElzKF7|G_h?DsmwU)8{GwLSs_(wF6siN`QRt6yfcOiG}+CVGB7KXE4 ze%Jr_`p6x_9R-G*1+e`i;lD{7LhI^LO-7d7#U4F%+>>$~F(S9#5eBQA(dmS8<)~c5 z@3Yrr%oKc^$JgLlE7Q}p(!RKrHm{ZC5!uo}QGUf~fELUzS55+fKZoTk42zTDl900A*fF&5NI}ou^SZps#=ReZ3xu zO2^zO#I|4zuBkYbfXh``gH=SF&Q*L1$s*#tvE1*;ACU;n|0Js8qcE6UrkbN*SYo9z9>ZkhO| zyonZqeAJ9tjp*t2J&dioc!!_Yxa)%7;Qkz-QIxLmw1pFi2Y%&=iqC(+LJLO)-Ypp( zk6^kvh0I?2%?4ihWhL}u74&1qSEr{X_qV7#i^P8lX9l!-o2-TZE8$t|^bKj>@XADZ z+>d0h$FkK+UoKm0PW3+q%?{(!b8l1k-8-;pX9gPc%|3TgPmaH6ntPin2{mS-U2|WPJ<%B2JyDLe5tBXt!DW+jdRHc1?wk_7`?>p#wO-5Khlg zbS?_fiOgu)>8h6EA4EO~n0usQeznsGY8(cmhbNXkR8;K!@LGB0KX2x(Hyw0B+C9N3 z>_8WMBP5_woN*e&U?NpIP;3X-hFT8Dr}9kf#PI3_$Ov5B4HS@hPE}f_#v$AV?W|Ir zmk)Mu3pKDrjlCF^J34o^5Z=&f(t7*<-N$jf0@*es*K3F=4B~N$D`OudZjW>KAt7;y z2X+=5IXH{kz{vf}a%O%lkXz)_mamr`fBfp?ax^1%eb4LRN0Bp!ry7@d`KupS=7%8N z&++h}J|pjev~2$1#)IiaBpW0diA3XlG#MutfQGbcMDT*cN)DE*Ur#G@t6CK|$)Iud z$1j4cs*Y{a*Kdxa7+()uOLyt*Yr)$7jKl>%NZN+9}YB42iSupJxoL}Je@XQXnj zsM?+_xUx)y_v;AY3XbXqD+zHy`Coa*9OVp>{SYJxEskfzp9vOgjB9EP&T)K$>;2EtT*dJ6q8rp{ zow8W?rvWmN#%i={zjAE+Ws{B8o1T%WzUT)ZQFgfFXicrHYu5(>Awli)0~miu`We>s zTD0)la`l63(aUlbf`0@3Y>`JfmiAQFW+U;|LyCy(V!i1(i3)jnMd6L77{=D;Nb6zc zSj&wKZ|D_t9w5y(8`u2ola55+`OOO9PtjiUqe)AirKL0hc47E)GBQEbZH?S@f(Jr% zCbDBvFbF3?2V)%70~0Bl(geC&H&AMmsC2L;18Ym@o3 zE$O9Kt=@4*H7o*nHKO`mjB4?RyU$KptSnf{X0}f)kH-(!U-Y)fDXJ}1hh%<{d)?8+ zJ##HQ@Y>w32VUx&#oS@vqXWm5+;ur>S~3^ixRe*a`b4?1F^(Sl;pw^9$U`3cf;J&q zIGO?O+o;41_iBO6sn`I%ea+Uw>p*tE!*b#5kf^w|NW_CJcw^LB6BJO*csV)d`qb=q zZ8$->n{VR~M*sd1;uKmQ0N@D0<;#6dqdgF_L)I8;fTLAp0%B|s>aAIG@k{~+0jb_W z&6qdyaD%Zp5k9;E2kP4k%Vo5vJZG!Y-MBA_XTL0QUpgvoE52fvtGJo?8l$%vedl+* zv!eEG!O0U770|1)INu~|sES_+-p-HSmfZe)AV%s3?B^EFwldYN)|(8mfp!@5brJ5V zvm%qZ?cnznqSv!KQM6zPe|?-9hV5}gf;}E>|CFG87M0?rswj6;58j~u&7C05COj_Q z=eQO}S&iu>ptgLphpW#m39k`F=QhzVJd|}MLLN6_DVwc|yErx30Ey3`jn9bjCf}_s zzaM;4x|W1Qsyuc2UkpiQ?fT-Q+V*eW&pw?26dQF(95L@wcz>Oj&TSb=yst9aH4b}FSt!;XVrYZCEgNe~?#{}sxdAJ5D{(0cOz{n;zu7Vho7o))L-Vy07A z%oZ|J6c8z2nO6!il=im-@Y06~YCjpGmr9L0?W>bs^SQ7Sd_KfqS`y(`wJ1ezLMc06PqRb1jGb zbBmYjEd?^-?iCdupx`e5@Ij^QM=Ju6`(GT&*!ss;9L9dF39{f@d_F{ww);sa74j~{ zw3=-Wx(ZCthata6d~rC;Gkyx-Bow4iB<_xT7%%$`j3te`+hMU0OuX<46Rn9^vler- z8G9U$AmD=gfpe02U0U)OUY(QkA|sNHa~9#F;ENXrK3@*)dUNF-QUkLr>v%R>=IoZRxqGzRvz$*}vRW|)7B;U4^{p0T9l7Av~{Q)#_ z?|+_lWcO!BT2u|q&tD#Sq1GAZH*qjs`hY_Gd_OepR&~zwsOW*6YjA}?Kwny(o1An# z!fW^O%jx#nlcr*MXNxYEgc|#UEO{;ZY4&*WFF4k2k*PjuoRquuWRRO=FX>GB!{TIl zcWqBmiw+lRWeZR@qSMXOA7=4ab9x1ZHeE5g9e&LXnHn9afm%^_PR^;3A< zjQ{FD=j!;W<&UQ>|G27LSv8!~-|`B2A675*-So@R@XjLo;QX{BTi7(s>5*cW^vcP} zFhIwXqCr|zT6aix2rt{dxna?&5be3X0`F*5gLAAZ*s)S9CXxVs!T<`N3H8wfd-<`8(ShgC!eAjXrdaj07psQ+l zRWP;EG<*rlZ)Qe1UEQVzYRZaH{$@P88qGIRwe?grn&0QehB9d53$~nTm#?SsM^@AH z4Wqc}t7X!%?kEz@WM*jgE_@w&h?CmSdBunQXbZ`%wo~Aeh{7BZZj^eujMm^0JUCc7 z7{Uupdliyu-(1=AQ-v_B93{9=c#rh>Z(+l>Hnh?#OVZtgR>FF!_CADW{4q}!>1sVQ z+C^FT)2wT@yd;w>`2%J1xC|P@)1g*oqTYi>7r7Q&6EGmXV0ex*kby6YuB$s0zDqWm zA1k{Qzr08G{^sfzW!XQVhsdHinO{jf}o zXHabvOW07b_>}s5Ah56Aqqfc7FDE&4hh(Vqa7}>|gXsp?6LC!@Gad%?P4N5= z?~eVxFn!`C^f6e*$}Csqg@F0Xt@)RyE)6er%6@c)g@swfEslo`Rtj8O?Frh(`>E;7 zP`-4&a##>IW#=^%`?a?!ZsJ#H3vz3k>xC-kf4XMxb_XU~`_;8On24hW{2cb1O;lF# zCu;H>aLtAhOzVn)z3Y&PnUyY5^09XnLmn+5VtYTaAG{Eyh$@%u+%M?r9JaM0-4*Mp z8=<3ncSABcU&q1`w&(&He{d4YoHI~@zqjU}Y|u@AsKXsfA<8u~utNprLqu#32qD9P z<(CH870cTVb*7#4d`kvP`xrt)OjIzk-|^gbcXcoc%eZeJTiZW~1tX`zZ(shta2N6< z(9g+_lbvdktyr$`p0&5PgW})qV}b^cXe+jHzn|TMQ1R&bn5nL%srdI@S&a)%wl^+Z zGm0N=+kJWTs{G$x7i23Z;J2F4qEM;RrtHB(`%|++kA(6<_ea*(*EiPx1$(S$n`@&V zQEbDa36fmuMPt;mvfkQ(AiJ#WS0Ms;7beD=F-dw@g=!C46rD+U46T*&Iz+Ohz$SKR z9UV%ASt9Eji_;~)KwmCMdeorcMM-EbN4ErSpY}Q-7$8%;`&^w-^Pn$DIzypMlqzAP zSQj+X85rEoq@bzWLq{(ze*8Q+7gZtf-yOgFeYCxs2bweQ-n~282=W)7r_+jd4_m~? z)s@=$ar*h)d>XB3{!4@OgnZd|=ZE5Dkmm>*8{?+BW-s;i^%ZpoDzyYTKhQ3EWv^+L z=MZU{SLskx);)SYqu+!@A>tH!$%(c1MYu9reSNp##Dn080T)5`L``U;0TP(E0GAo_Jkz*=jC#PBFIQhntihnux$Su({ec=GM=ckC` zQydl#`VgVLUB3%8r87&qM;fqM&T^^G8SBOX8_n|?HOLNO}bCYv({L>lFULD%+R#Y}o zL!;3`#Zl&YexGYX_lraJE7f}J-JADJF|FT0JbAu;fZsj37$iQld7k956Xlx2kE%`j z6yh>a;39CEuXjE8Ahf98kDun_XOc=hLvg&(Q#skI{1-!{kB#CQI>FvnuA^@&(KH-` zW~!~0yJr6;>O?@drG!r3HdQJ)Dc@THu`{=#cARSCQYx&}L4qA9fJw9Z>}?y;N#pC( zNar&wAdtXi=-gb?&al48u!^Bs_(gYjbt@BAX7AhIipj?0FkjgV$=hjVr>W+=c6?fP zXtX&kFRHId+Wl%@+nY*$aDN5Ksa79?z+sOB9zh*h#V zMwTRQO<1Hv6!W%P9>OYNIb=yWDfb#`K51cNrtsI_qoWVAGzNn7k6Gb->Msu3)x?1>_6ONV+3gGa=gRTwKM%9iFWSHjI zYqt7mw}yC_tkdeTrt4^OJ<~4G4njG5I-yrn37o`Ha^eC7P1l(=bRGL+VMjW-stx+8 z$LOrgw+Dx4W8h8j7OM*%%D^F2@V6Kfu@;t)(ne*7DzQ|5F53k}hIdL1cEL+yk|ogN zPfj5C?7*F&2qr}@5DSD<_a#<5g)s*1Qq_WNp{t8dCaZx z`-fh74Iw5-n;HG`J7viGzoQ1n1iF;Q#z|EFijS1sH^J) zuYP%X`bBhf-9!**?4t%y+_J6qw(r`GB?`yh)J}@1rL?^XMDhTAfDf1p;`))iN;wiy zqkVBe`BzZ?;)7)bxCCZh!WxeKaY0v2JrzP)PPToNAt~2JvR6GE`h0DqJ9~~KTR%R@ z;sxxiM_F!zWu`-561*n$aG9NVCV$2brN}0{fQek{^;+qG7Gi+2zWq8x40AWFj8@@! z8h2(3mWomrki=vmS0})UHD?QtqW?lfswFLguN4W=pOi1>^bZ3_A-M$VJ)_lC$p?qhp{+y)hAkK5_! zMQp=Opcs3__I$IGc!n)$TrUe^xP2KQ0%tnCA}p^^0ZxbI&YCICY^Fq4IqCkyrtIL(uaX^OpheZ9Yu za-XBqHIO`Cz8RCqmcTKzn{B#58)S{6+$KQ+2M-*Dn4qTu(hT2gB75dDOUp6BUpuGfSO z#POthH-VFr(Q`XTf9|C32w7k&Da0#g)L?JNFOPq2ZRh!S4)pF9=>?k|NK<`w4!;BC zpK9;-uB?|{wIP(1?Am^)4?M#ZEx)qTfv1MSrPZK<+=ARU7Di!}3NumLxV!X7kF~4y zuM)Mh`}yPAn^FFOkd7(bkSi=kI2M3g`lEDlnT{_LLW72cEaB-!5MbNnl3Qqtw11U> zFF5O@g4RZr?1t?9c3Cg%(W*$cP}lhVlTj)nPjMPnMJwygq~hKsM^#LYqcGLKg4=g3 zn02*6CX(k-x{eahf`fn>2S`r`CuVMjh21`Oi7$Fnn^swyE>hJ(A+4y2bmMG%!IK!P zUTy_>;P-*gV`GaAC;G&rHSM}A45T9+^^Yln`eI*Qu=QT}&>%E!9G898B<+#EiKY{7 zyrB4!V$#2okf6T;|AYrQLYI#~ZOe1gzh8rSBD{p|cCDgq7IJp0!mws3v{SMlL_YMnkXJh9p9G)r@2*Nb8)qiMWgo$Y;Id`-$87v} zW!W3AA1vv_fT_@DSw}=YO{>T(rRf>gmxE*qq)aTzP85w%@3tNC@CK$zx+&xNTN^Sc z*{DBBD>?-^jA#&Cy3|2X-)yL6ykD>QdggsZM)q$7JyaSkDA29Tuy$O%pZ_RsYH@LF z@w|C^l)9D@x5tj_IzbhbCJ(sm#3A*%1~OQ#uWO5UFnJhE_AB&$-bu{@1}UI8nW-sv z?;z>~jY?o)TV^Y9inB_ZfjddA%Kt9+{sOcIS+aR!lAbp#X~7XQt@Qtyn|#lsZd_idP#(VY{mYbW z&Odu?khrOQ>P`d4pU&FQkB3TS6L zQbE&H^ZR@rz*9YB*84YW-NlRVt6E7?dfzw^J;F)j38;$e=J?X!3e;wQF2?7uI7Iv^ zBpZ(+o*(f{d5aCW{1?io5awPuN+P#A866ypOcx4tB>UxU(DR@LGn5hz)g`@wOH#}E z!C_fL{Ez6kPvP;Og1wG~dT1(6TL-$e1=VgS)UnzblvOAPh66SxXIqcqPiIo?bhsOD z_~1#!9P$$|o1C3MuiVq@iurwJp*3w!bhI>VFp#d;iqX_g(n~@Gp;#JGZPYTruhc7Gz<5Y%oaTEyUNzQf`s%C(jSoJ`UVU)31XyjWz6O#|* zDhy^U#Q*^59na0@+>*&C(wUwdxi~A)ZUUf`Iei;Reb($H9x`osGZI4@!W+4%8F0r*Wv7 znqYTYMWxFiQ4`^tlk4CHo`&BYQq)S=QCgNxFwshXK*sGPZ3HBmlsu5}TZPMwJBBz+ zTBsjCG)jCuqS!5gb)A%nf-G?}<4!S3efl^t<2AD``P1f;4V4EhP7q1USAv8#$b75k z8~!!FU*wBBJ8Km8;aAymzAQO@t`mgFR(ei(d3k+$<=nr@NmNHmoooi#FNUt(XfN!b_&lQh9u3-;O@WeSeu)<>=(he$?*^1yAQqzpH zO@sBTldjbU*OnEQNeVIIkfQD=b4YnlI+%Ft&@z)1jnhfB5|wS6ahlcor3y%YJC>_Q z;x>9Fj|k*RgAW8U|NVybSjn6K^-?5|A<$wB zD3W=&S9}e_FsE4SvywIcZvZ4qPNph1&4QgZEJG_!Ny!5DPH4=vGsJ9CJ zyM7Al%iHB@=cezkSxjrA(rT&Ly%Ed+C5}Ng2)to?(3t8C5_*NtnRNTdC^ql4i+;91 zFbe`{MbP6e=f2*I15{2YlkA&?4ZTkOg%xl>eBDd^TQO$9nJm_{^Yi#`IjK5NeorNA zY&+Yl&$2d_lt*EUUy-T6*A((`zAqrVJQ51oXp3pGtB#@~w@^}G(R;^`YPI~If*xoK zy<B3nv>F=t>uNp#gc;h<&%N!_hv@G6*z+No?nJb|UD_M_=>_LOi zq(wz+OH98H>MXBeU+Z+Hqh_OEOR-VJGY2fLqTCs%d`yo`%OFEB^$})z4(|nhgfLr0e#szZ~ZWTtq;Q zN+;*|uAk4pbr2u&&+g}>s%bGqhAe(R{os~rkjUvJZ`x~xv{FNR>Tqjnu5&iZ$yt7T z2oDQ>zz~=B{B%@A@!?t}q$_y@+}bhv4Q)@wSr%^PD+l~6i-&^5lq}iAa7s6;yEi^&-9JJ{S~@ly#)uRk1DX z?6@C;p?Ocq#fF8~XPw$?U;YRhcp66&(7PHc6km(69-x`B{65& zDdcjO!i)gl^BI+JH$OEAA?KMq8HY$xlufj6eua5(!?=U=xaO!z)jg`fkSj!2-x3-I z3M-Xr%igTdzTXZ@_2Gx2vaM<)1)jRvkc1EDkr?C5pvM$o4|F27k>2Tptbv}D0+w{T z;+G^9|LNm2p+^r8DN5WSLk`)0`cIr=J%U(_w>y*m7F*A}JaS~WMSRrKa{o-#I^?1W|N%+Uej?Ck_!qnGL&E39~0=H@4dQu`gY9 z!PzNh=vX~Isq-PEpvDikuM0!kfy*nTHNb2lV2bYM$iYKhwwwsaE>=*~CY^Ju-{$IA zN~)-&*G-Co`a;CLqhkPy(3t%!pIbyG3>G-szqJD6skay07bztGZ{mr77!s`b;Mt-5 z5KwvTh8zu0)6RrAx4A0VrGe9W$HH-^gr`%z0ZZ}l2S4n;IsYn3Io&ip1y6iT2}g!g zqs#UZF;^kY0?uylUx3&Jows9`$46gG#ZPZ&`qtL;P24oU6hE_F*8I!v%lGrNqo%_| zRL`+v13`UCHcCom!3U4s*+2t`llllKcv4<%DI*5!iD03S{xHicTg84{9a+J)-Wl`o z(&~7B`(*1$0YBR;R7)G_PprXZoU7<9I8t64R9D|}hzDGJ?&kxqlvo(XzU+Izn+%rd zaTOJl25N(Vo_xtPKENSU3OYn6yq%wkQ69e)!FgDGJw}&#NG)AwAch7>BGxkq2un$) z+x7@YoQGv`NAm{w0xTO=I0h`=ALG#2o_PUlphm;%TeB*yw`-Yfe&EcCDYVf|^IG^J zoBZJ43AE8MTHb22(W+!>d2VX;Th(%2Rotgwx4xq6=?)QfGUGnUt*erMz9N$`&|CWE znxQY0HxCUO^1Ba-`Pluqdz5hzjt0s21uOw0+fD!alunFENu*nb_nIAJ8=le9QuGtr z7gVV@C=A^l_oySiOi4YV71y*(biZ7 zDLu0QJG|W7mmenn1@1LMnih{Vt;EY_m1X0nWaD;Ci)B*!PBn{Kv!a`~D){F?xyxUd ztJGcMT+;t2bcsKx?S1TqvFHh==R}V+t4~5fMH4C41$Yw`R8{)JWcN`6KmYmI;jp^U zrirDYel!2}4p-zT`jwgSh(R9jJPID8E#_gR)Vx`RLpZk4T8GQNwH3`(;-zQq%hNo0 zRR8abiSAAtr~gJ7eL9Qbq{7l|HPxQcJQDZmN9d()Y({IfAree$CdX8NV+>1N$szaKBCIWI#0_Rfy{?uZc<+nAPlom}K)_-JcY#V$JDE@bY)z8Zs|9 zBb74(T=FFDM@AYa#JSJw4ZPi(nTe{7EbCEx*^lPFCgYLSCpY0f6Rxq|`CBYC#ZO&| zUpg%N5WjMD>hc^fKM*@T3O(WP4Du>=o$u%`A&~-R@4O0aItH3VMg5P?7-ntnqc^b8 z&i2;@sIgj95sH+X@Gu#ppydOy`L?+Q^Z>iAe!Yp1WV5oF`x9bNpS#V!ctW~`NC15q z8mvY7z)#JXToki+7YZ*ed~bN5*VTh!jVpF;?=YKK;)jdJbJO;D*x1?ar+Uy*f* zpwvtdF4wEQ#gooH@T|&`($$QAg>zh2?0i&#Yn-6M#;ahItW@y&J%bp(B7svkm`V51 z^S4QXjzp%_4MvM1eGJ2+2kEBja2;&4rb9SEMI0_qQO{xXAU{?-S2S^`F4Uo@#cHD; zb^^~f-q(z4CN%4|5;WCRy)P6yvz&_)F+TX_(=X~hMZRjyz(g$6BnYi{m`l-`TnnO0 zto^ZcJZAL=F7}MXdMv#-5noiA_`0Lx(J((WOmJE`t4-Bf$LvDkty|dPRne6Dx9?AW ziJy;-Kg<~h)!P3`29M@17a7HkCVOv`ZR8e$p2J_Iyu=bxQjmYUcWrau z_W2Qb3v3^C(qjq=wwbAbR%JTd>S!UOBevr`h{bwipo~_+iTXPTOj#>pJ=}TiJF;_e zV;=9-D9k8fb{|lrAAUYg;h{nUix{_fc{5RBn9>Hba>V`x2Te?bEh@;w9WAOvu$2^p zwgk4Zjuj5Clca}Mtj$K_YtuJpgVXGvHcqSR8iquh;T(ugjnGz1;ig?#)9am8$1^)?<5g&lmVT4>@w_ zb>?4O=aF}H6Qk#I@}!?0cp6D(>)F*zv;DYt*EnQUq5`}DMG2Z^Eh!}L-a4>84Hh0;|@uy#pkUUy82-}9#~iK`9!q;#5_cR*UY z)wONTwSP@rUOM8v6HfkoyllQsIm}XK*Yx$p@Y1Ss{K8>*Xd1pa{gj#SR4hQ1SSOFV zw|8`Gn+U1(n-|B)cQh=rqZ-Z&SWMgLOu}=tP^Cx%B7=s*G-)GAMY)%0P1r%*-Nzsy z3Y89ippbH@90a^1BYLSd96qIPT?Mp4Z$Y}l!99O>=jJ8G&g~O)!p^~vQE|8H4Id^N zg;WFqB@FmL!bi}1uuT_oXUm1w(tcGvCA^AKwVp(l5kN&h;=VE*Lbjjrr@$jy; z2ZQ|hg)Ym>U!VLk_4vZbeA7za?o|NxOR@>&_|fh05eC__qt8nqmRY%I75H|P)NGy@ zKN_+WB{(+7H#_Y8??)O)>;E)nT4N+CxMMiUHWhPzdld0V|wA|XSlD{%)u3D+GAlx5xp>}@P{I&kYO zC}2x!GYL9L0s0Y1Zn!+$5oK}IrTU4bLt^91wofq^Bhf*v5l`rvE zpj74=pxBfPQ+>lwOtA7ywwxvVVdn+&mml!s3h*c#%fG@}y3Zwz@Wo(n-qIGo5&qQD zyLrG{heiy&&on>qG?dmGN(s2!v2-izV`YhQ-i^}CtW)a(Wh*5t(uHG{VOkqhAKU;1 z5!vd>*nt>pWD$a^{`WaPJ_FJ$UeCMsj@TS=;~;pCM>fp_6}(hMIH;Oxj8j9`@PHlD z*iu0TDz%@JVrq|*Zb|h~^FDUQ@I#8J!nDBw&DCqd7jW4ipGG48;dD6zs_B9w=9Xd@ zdIKpn_RniBTE;VWu zw&!0I$VC=0HF|<0pzKMDl;VVn2pX(avXP>lW}SM^ub(9ygr(ueC|F9#%@T8$RI~O5 zI0;JP3Lfa%U|^2;)}k1K&Q)6-E|D=B=1_%Eiuh}C z_OB}0WZw&u9)75N6&@yyU$4c_ZjWCbiJu)_m4?ga!YjO639KHjg`WmGotcR83)Hv5 zn0Jkm&Hd|Q^Oxpo01QnyryF>ywI#!j^ci1&GpYg?tMW+<=?nKckXm6#mnkWPzY~$A zyGoTR0vc>w8cKLNtokx8#-h?8G_7VA?quShqQ>@E0B_>%i9_O4wl5nm$tWO|l8l3e zJXe&a?W?4ICzK|t(|iw_B`kkvCsEP1ZmvHA!K}}BBg^^Tm?10PSpLF`orQM2c;7)cQn1`$bPJi8__E8nv zOkV(=eQmz%_mm2hdI^UtCvm5N?ya#+#|&vsa%_*;5sIJTi@-Wg!rs&SoNxz~&_EWZ zU$AC56&08inT-I52sqAjsIGLSSCo{s$Y0p=3O?W17x(aFH0MLWJH~ zcElr(W)&fiT!e{BFzxMxGqOuUk;tQBA+pLwzxM#8cbq>Z?hfsU?O@&aG=jJYIA14X z%x9nPEyiUS(9+9kMyR!^0)jXk_n~gJ&1)q~_Mvn2d;E_KuMSmA7pYi?--#M?mv9w; zYsk#(NJ2W-C>;c#08V>i$zmj=hHZR~)+hZ0$)%*nsc7LLs>k{=o>+{L->^6rtCpY` z4872uPFQLYsmIQxP*`%y88qf7xwvDD7zEZcn@ghdK7(D(egFO>R6OwKzUCA*-XsAv zu%1_ledSK26xgLfznMqt}vKu0g~}?%|P7lIQJT6HYElz%m7mhLdA`pv601WgSOurJhR$@HbxLK z3h)IP!wWZt@4O@gxn~+@MqAEIHZ4BgzWQOlaXt$=;F^BqG+m^*6%h{hpsn>4pUUrI zu<1{F2<>LIsKh5e``RB()WK&j@jD$%uKJvAj%lb&G`{ZKz8(ux3wFr~ecDld?m^Ua ztBKRt-8YB23U=XiLB6!aGbnGY(DvPPYsJ~?3-{G3IHaQ39Kd4tLp}77pj(})%A#nw z6gclB^%?*ofCSYWQ-TWGMh`kNGFh3CN=Pupq_d4h^af9(w$lj=6c;mn6HdSE=FdMp zIsSgB)b91D1j`{+3a7*+##_wpq39!ScH!mh;I? z-|e7K^6=`Ikqo#V*`@rISN^b&^*2)VPF~-4jG_$nPF3eSVpO%%B8ne%#vU1mNN&ER zi`slz6ya{vR=?5ui$Mev4s{|#=?FbE>lB4ydQho?T=l%1QEGh*rXxD-c7~}~atpmI zyw^*3UK2XQKL+if-TYASOKN6d|0%Z$>0{+j@K}`db$=p3qJlYghfxs8&qFjLo-jE& z#vFRX86)X~N-H405=^fx;-R=n5W{o^V6Z5=LRbfnq-sPk z$yiDty`$re`BodaeRSPv?o<9DY*Q~Re|Ro5pBD2-7+xyGe@!=ZSl z;2wv0iX1&njm4M5=x`C-ZtjpKX*vCzYgua2!2FLVznn^(;Ir_}WN)&Xl~3uja`LPS zMhl4o^u)7$4B5#cmNQOmS|us9+EXO`WRi(Rb)Ex*v~&ZOLX8D;xAIFo)h7#FN=F+f zBE!V|&e(h&Rg08kYzU~yGfSFu@$E?C6pwe%HiZrSa!ctN;9=c|E>24#^JUjM<5y>z zW{xz?#IN?}$4Nbgm&g4rj$~d_BHh|bP5*6Jo3{%$*lQOl()R{-L{#mw@s-(pXntgV z_t_3zzh5~3%REJG5RI>y){sb#-^lOAcRQny;ch^%J;UYwZGNpj*}5A-QG60g8LPDU zpw`y*#!hi}_sx8ZTT2vgyooIn0fX{9gB14?eHa_Hxjg5L3)>O^09jxhYqspoqa(89 zNl7N1g-ORt3@!A)_F^fl!!@4C3;w#(&ie9zGfUnW!Q6HJHa|4SXfy=tD7g=BYqs9g z@Lb2E!FK0XgxW<=R#j2(W|y}uO7gbnFTEQXzBuxH#A5zO{QNKRUstAN{~V5+(Ko7e zyN6E2MnmGSkq7KI;GupcY%@;vZ%D?!w$kZ1a z8t+pj}#&IppKgL5swOC zVz0GMX$C=L+zeS^aer*n#4kNOzQbO%f2qYpNM`?opc9rOG1J>-w_QbPQ#+{MEKB-Y-5qJN#@26Kv(Sm(Rb6{@_=I-a3GGJ&JMt8YYn9b zC_c~Sfmg`^&|)FJ?n^VulGZD!MI>dRd@(}>41m)vDA}!lJ%|{pPxg|Q6|feiyVMi+ zfjm(-5Hykbzb%4de z``|_L&%zUbdSGbR{Pk}#$5ak^PGRBsVhhL;ifr?i2pu`kcc{+n$cTk<=w`ngc$GwU z>MS^dD({_k7|zU`PcvioK@z+=OzVRv)mtqJ}>=hDUaoD{!gyl6@v71|RO1 zOHD-dhjzzRB z;q3f3scAL8)^((!uj(PK-hKKL>Th2%ozL^f4vd@+@;~A+5#mw~S?w8-ud(gpZKM>_ zYp4?2fIK-dz$HmkrXBwaycsc2ek6Co)~xuXm|J-;#dv&4^jD<0i*$?Cb?at$Mu5WJ z?m| z_Cac>HS2WBLc7O0g^Bi7>+F#qgpphwBvrY>n!q+TY4$yK!Tp@5yJ2!B78?TSH1RbKhMv|9RQ~;X-9$Ce4>}s~$+x8NKISH=`fC|#dvRkO#sXxuC6vbHQj&ZRr(Zn9zxJ=yLiQJjd zQT~MZo&S*|MZ%CgNf`SN<8|`)X5_u|A#gEup&k7}T`BW%D-8^clCgm|ywUA_+f4?P z)Y=yNI~$h-UQ2_PKcG*oB0P3x#B%A8uK`$~8`WIu{+NbpMYIw!eRq1F?+CS|cc7+U zzdbRurms9)`h z63>C$?*QMdNW=};NlgqAqIqjs&B;FA+Eo=J%@0`kCmEMd6^9AVzQ z^L)so0tM}%M%?p|Gio8Vw6x&dRAp5g@CL(x78&?FzJ<|})%3k^7Xg^3ENt5Y!>n18 z%q~}7wJhI9KnwN3_u#lU$?nR&o`AS1wTzBME530^+?Js;ZJQ3tAr-difr;;Ex*LThfV=kp|1r?9VZZ5* z9S*M(Tg=Dud9Pv}5;HLg#U@%yRK-0AAX7De9Ok>qs9Sm zA6i!1Y`uYg7%+q1YuagfxKGd{bk2p<@_?Nh0YEV;m@`yT&8E8#$ByI;FHTg&O%6K|Z)4&6WFk8jbn>~e}rx4dwrI8FIagE5#)Bo3b7l{w^%IU60HXa7#qLZ;?b zmwT&eeae5o)Wk>1Ml>r0@5kxKKNYO}CmLJ=X0xsbh8K8ptTXhklSei-Dkw{SqX!V3*|}jhjQCi)rxJ#mgFj`+liP<$b#SK+ zleJN&0?P6tZyy@z{$x`27cwVE|5pL~4>`V6mi~S^G1n+-W_0qhMQ7l4*5!G@aPI)# zj?I2{_E`A~U)q1nyh!H^q;C1gmYNS+u1aA-6M*keow84z7OSEcWsZiC12r!Z*9jq- z3M?OX^`AehwZ8Rv@yX;=QTLDYe?<;l^9dL&r{M|r*%3!~_0egWRv|C%AZAJDx))XU za+|-Ne#fZSr9uu;RZ*0F;U+x&G~w2*8I$O&wxeDhITs%V><4M-Uy0+Tk^D%0uR1Ga zK%486X#BWxR)f+t(A!&{&SnGq>*wC==YG9Rs2w#6LmhVMPCwy5A6qdkImTJ1TG3`u zU$2@d3A7na*U|cygD5ND9UC4T%Bnd#(rGte7&ng}&F?7c9?a!o zaE(i&MUo6Y+@3|8zM1%;v?KB_n#kB7(f`HG#>I^A_>m*=GgYgfvt->yvggnP4S(u* zFNm+s4zS&~aUQ2;AT_UidipxXY36cc?o>zC#nIeJUrdr&a28S$#NN^OEas?zH|u1v zP`^S0l_UBUcT9(S?+}PS=TLF_OqkMG8+fm@(SRG3$BQxRocVO#B5x!Yt7VNT;JKXZ z>)`WTB!7pk^WNzFwuaJB85Nq5P8cV(5=77HD`7FV&9*AoUb1`;G9iPUu<)214g;vu zO5u!z$x}N!Y$XtL0~AFRu7vh(3(x75d46Swe_ku0xL(l~vCdXhUH;;kYVz6JMk6z$ zmm6zr4&}!!>Y_ON{8K;v$Fw~0nTc(eVcUaUN^n+0v;A(jq4RH8)(2NUvK84^%Q*P2 zr7^eLE}_&MjL*i}4Lnp{bC{a9nw_nRFgGvjqJ`JzTzdflQW9Zh21EYPAL{A>&DYbsh>b*rK7uht073Mzv->#-RK_j0Nktl>#<*p3%k{gmB+<<+k}ruiw`f~R z@jqj~zvsWZdG>k2yf5eSI_G`PdC>`+*DKcw4xYZ2-ufl%dio(cj71_unU%3Ef4uy5 z@ZOO)YI2aIOJYgf zGp`(B_Wj)Z_HB-Q%jm5fQKiiB_k?#RP1U*#1vm}3C211`)Qz=36E6nAZM=X~2Uj_< zIl%P?Qr+3*>$4`JfKrsLQrLd;tN`_FZ_P++4Rs$W9fJ|&*NS!=J;qorKzoDsIhCz9 zqieuxGH|X~O4GL*wW_W)(m*X^t}^8mbx;KE7e%rVIdQpC17tcCB!baVo!XPTj$@i7 zV;E&Mf|$MGO?)|32j{VWG!N&mOb;eK@3Lw#NG2VD$LHL58zVRnj-ic;FlBS=??q#S zCm*REXxhFZ;c4d1Um)zRdHsFz&#HpmUytsV&p1l+SQztw!BV$23(zhb?VGv5{>m-U z`FLQVy!ylxSsZqn?Se{KbNN18X9j>7bG@-hB z&FITe$02v$H4il{pPV`D#p^Y(z6&-rY^Fua+qc^>`SU%$t0i&z_cpftJ7th(K3KdISvP8uK^5Xd?)tuv;IZFkdmvFLBq3(l2^rI4Z8+}Z<=G}||%9k!x@SrL0PxZW)tJo#_wM`gwgNOSgF z)SS-A{+z|FDT-^<__F{iC@Xc6aG(#`H$R=)0lEP8@+Kr$&*%tnS-}LA^w7+*WAeu_ zunWfw$$T5QvIAES1M%5G5WIqj6abPp9fG%?@8@XOUNk8{wvr{EdI|DzIo-9_Nnto}m{Z>a^+5LdWbJ z?$9ST$>(Jk5I4VGf}rq4w8Cu2Ob905DmuSCbk{BLKsBy++FzO6edV(g4|Ew~;;>KL zJ+J{kj#a7s*!p`2ND&g_SnB}y$~oR^SIrwTYt(#028RhZzoxMk`VC>MZK!Yo~0*S@zj&2MhrBK%(y8)z2rlpZo6n z^j7%0{J_%-FkNinW&Ofm_1{7CKWIgKe*WposT-QhMK&&6E6sd!}2}n2Juhu+5L?gLCTL=+RzIj z_iIG6!e=Y(EO(jPalmSJgfb31Xa$*}NRkZU%I(YD>gtzY!XEs3r;1Rf z<8O$hqc2SERyntDuFaqcijz8Qj)PP2x3`Puv47baosMv$;foTO`qb>t z7xUe`$C444&St{seaazV@`_)EPI+Y$)lnTQ2fCK0LpBt`6z-dbW@D zdx5~~VAs2QlbO30zXBQcX#MwBKnDBr*7yIy9D?-4;q~e}7roumtbQ>RG-}#*XqIq@ z)gipZ3_d76I7{}JRl>eOe^E9BO?P|1-E z(ra9{9!fNzZC(C7McJ!v);?+Np8_ep+rHKdf3v|Mrt41GT9G6s$sY`H^TXdaw?w;% z(vWn38(_Z(@N;|ariXy?xWVs=&8KI+kb8qOY-vTlmbr`18e=n0W#)S0+=+}dI8BpV zPt#&;#$=_rG9!k+z08?m)bi+&Y70EmKMY2aZrosMK%^3M5Ylkn;L8UVx=x?_dcS_* z&w+)ZE$6-g`P$p{@@kuFfnBvEge@kCLvkDSjIyL|$L-?X2;$0aQQi@nytbe%Ky}RE zxTkXUGY~uJU2)H!nnRG+X;kK#;#Zdu7-V6%Y{e z4KYgyt8Ipu8b`DO+dqw*$_y+A_k5}yc+C^_*`6to?^cdmr9XUY;iWqN<<+g5mSd(=hp0o?S7w-r)+nB`R6qFz9XBBZWh`Q8#Rj{#1!Y;M9e*%dTFBk z5FHItb#|}GTK%R2MuN3t1_&W*U0(JF&5k>7WQ`X>z^|zhK8Jojiw{yhvBnr`PZY~^ zvgq6o?t*8TPe%u8EBj8h$C1El%6e7GICSSi)wN^ObK@bdU5uf?A?CKBoj?tk-&l>y zLJhog^uzYa1&C)1^7Da^RRovIDE-W>AtULU)0B>kPXPeNYM#7;tkn4Np!QdGVtqEi zCn~utw$WNJtOf5Yfx(R(`i|>bAgfe#`CX-2T2R<`Uk{*d`~Khhd8dd^FK@xPjosfT zo$6+eAN+~2?N>I;ieP&JE2mc+r9%(NI#rG|)>1^#=8c-A_d#+rTCAdNXr4|^7D6`X z$-(*xIsVtnT3`X*Cn-6nqewbr64SKJM97s6y@z|K?MCK{HnHyzpJo&r5Y zYv0~0YChT=v@_1jF}v{sW%_Jh9#jj$Yp$(bd=^#VVs8sy`=LUl8tk#t+G&Zm&U7fKY(AKP&nU&Y@T>-kG%EMD zAG`^LPi2jgwf-kO+a1|ISRjp>9sGrd5p(rjCGQU#jaeB|1#aDO{s$m$Uc576A^ya- zl;ZFS-{#Nv3RXIEWwm~PTzgSk!Gia5S_qZ zr9#}+I(^b~gHo$1teo0$r_rShoYH4Gbu?5iR-c|!!gg_shBE6MjcTAR9xWWLTh>q|t$r&r^t<|ME8 z+p;^xQ+rA^t~VAjN`px;v=-I3_a?6%Zc~MLKq%(HY1nWKJ2amw%I)B`B?-^3^6B>- zbK}B~>~NnR(hfmFY~sg&Q5L8DTUYd}ugxUm&*z|lT;hs!Q}!%aND%a($i@u3HcyX z!TH-iF_~a*^`>_maU^m~@oWvDaOTy(;t#pfMop2q=#S0J=&n2RAPY^1JG}Yw!>YBd zG?AP3pI*WwH`cQe~c74RYh@!h-SVJGiV>I$4@|$hr&2kw5vz zlamVIc>@CfCIhckTy2XncMd$k4Z*je%;_7@_|8}2etV0DvxcUL;3YNCToR7LMw?O7 zr>*_4=F`TbarwMla2tZnf(;4s*miS%#Y#3I-u5N^&F#h&IwdPDQ+dwn1rRiEDPjxg z9UA`jjFD6PA?9Qm@i zG4hw6@u>N6T2)DGh2q-khtnah-G|+3E!GZ+drrO8^y5nRvPHAdtp_uL##+fFsKW=0 z_`%1qw`vU_9lV7Qp#|UXJQP=X%(iU3zxe67?=FQl*@oHDWbb%0jm~EWvy5Kexf4gH z(>DtWoZddRYP8nP13wFhSs5S+EYn?i*1021-c=}DI9axG`T$(ljYjqCz|8-}3VMOG zqU1^X8>-j%kC!1HC>r}P0U2=jd_T6xSUgv&Aadh}i=v*X(mva5q80V_q6eMIV!PwC zWn4t$COjf+icSxf^)gC0Gp~vx@)@bod%5&!J6pT^%7Kb&HO!14tC4otElvMwQcoNS z_GYT>lj3cAf=lZt!ZF?uxi0MRB)J;*K%!e7Cr>+mwi+J_^mI!3AEuNCKqgNEGpB;n zOi1bY3Hlp`ymEZ-9wc;v+>~XRL{c@9?q~Wxs3o^A!T$D(#37$mv(D*`!9jexGT;7{ zU{jG&i?yoMtZRmeJt~1+Hi_rM{uvvPu7IkP$9S`GS<#NZgLwu92!}2Y>fiys>36RS ztA2CrjmhVwpCNKzci0fufqAGQ&o&g~_lnP(kIg#mx28e3`>abHdP=#`cLB;2p}l{e zB?r2XnD$S|PZyDK$3NKop}JhKq3d&q+*R_xlYjy1IHkN~VitFUt8paC%ne-mDtt)3 zLWei6x>Awmuc9P7Vq1+`4&V1!bd2SUM*$xs7e@%x*b05KR+}CRz zyr(JHtdUG%mwnzF?Y0w-6jJZ$e6VgdnbO{1d?kI_-aq1A2QOa z<$2Lby);n>WDAEaA)udrb^l?NW@<<^kIk>;iz2+UGKn7wJ(YF*w{&QV@@3hDPTbcJ zXcjeEn>FE`(`AmGdXs-9K0Ws-!WSs*(s@nKxLeum;}D|05;;n3f(4u3hHQu0>ZYxh z(N@p}#1-v@4=+U^%r)PnZ|w`KOn20#F?O+BQbY0)b*LtoN;2n&aX@#Nv?RMBypaq5 z`tmc?ThqwTw%AEc^`aa6&;`l&TN}F!MQ);5A1%3U1Fb|nB9SS`21B3^kJ5#Wn|^y7 zouADo=Nd7bTUeeiv+2*Xae`HM^1JS+_!zUj7j6xFEB|G}fxY%u%SU(`Oorh4$jYS# zO@|7`mJE&8&#nxTtZ=upG@X*#ctK{9XDR=nCF4bzy!h(t3voye8lp2k^{yDoB^Ef6c~MYF;~e#s(W&ngbzOD z&O>Il>E<O2WH(WW@=WfDKMpExLCdQEORTe5kflJBD4NZn5?Cw}Rj+t0+lCZTFbm8$Yu` zlAjLy4$dYUv*%)8>Eh2X6DNKoZl&B^h7qUKD4$#N$K?F?h1TzB?w93*Oe-H1Kh_e& zrKRq8mnWeF}?YhfqV6pby8^rP@Y{~KY)TDZB4a7$r08ApEa=fS`rFljoO zoG1y)jf%{Gk;9zG$PC=f%PZ>MX8SN~mKt{EAd9yTL!AZ^hdh}{Q*d>>2~&(lkYLij zC4Ndi#1@TZSyL$VL5>>vA~67qlv2Yk5Z?e}Ob9T>3_e|n$2`#P)grZsa*}0O1aT(P zX5d3_%4b=#=WeiOzn@zSsh{0p{wBpb1WB~UqdXm{iC;xppD-WSFhz$!;xJzmK5k2C zU-|PcmYu*s){sIxCnheO!65F88lSwAzt7*B-(F?moV=toOdXs!cAQW&|P zuD}<|$lF#AZj|q*&I})+|JSanD}L#hO*>RAe8@V9jyz--7R;*qiP-SV%%OpZ2a5sq z>SIJW6Zv_FGt8~9gdD6OuCEnpKg2_1BBDo=ZzfZ|p)WVWXt=8M$=fc%{oe8G1de=J z!{Qwr%8GG{;r;cpq-#gjm0e%Wr5&n)PyD$G2>!&d4vRur6RHLCk#%-||GwM^wGXkA zd|C52g_ue%Ph6I#Kax>l;@hs66Z!Gz(s<$FtX#i!B{DlhSyKl8&%EkT#YexuM)t!WR4ep-U2)Ljod3ql zFzPqdy9GD^Bt*8qqRJtM+Sl%WTU_i0$qiH4u54DcD(}ssJ$>N?le>Q0Qo7o~r*B01 z((2;k_E2TLJS_Hc{@Vk0Z(k`>@6W&T6T4|c-|KQ=mueo0LviMH)EW|&4hN6|x1~g5 zvsdD=^1;>e<19uc8CbeulhLTr(YAvug_Mt%@q_WiOdNTbPVWD^B;oe{z3TnKm2tAr z>Nb-RRn9ae?PE3k=(ocS7J7S6tLMX6>QEdyr83*VC;CTa_ryXnGJhbl-=JyDiOWw zBw;D0R=|-J#bb3A=&F3BE@hhD*WmMm>{9(%v5>qGhc_I2wFntIAZ`=AfR zfq)5Bgyk-&0(&k6{isR7txE_p`ReJhEGwDo(|+)6aG%dAq#Cw?`&VJe#D;G2$%*;q zE9%|PdqL!g#76`ihJnFiWRbQ(XRY{it3eC~h8elrY@<{}Fz!}Lo;tN#%9`9uyKs2!M^uS*aFZR(w&XvXQ6KY)U*CkG8-{cq4> zFdV}=!}aIpY*`QRW?!O+iKpQ&^tcxtgn{i7Mn@T9Mfx}W0FWsTe$Dxxa0!#)g+T*O z#=S;b7^#=m>C(R zdupJ&ZgPIdk19l-JA?mfq`V*cB4yI zWWG-q6EI|kPormG+4oes!WZYJN6W5;U051aPd)!J(cxY2*$1^X4*a=Gk>8k9S1{(- zn2cWcv7XuQ+Y=@&fs^rtEAzVGNuMym_BMOYT4C6j?uuhOxHJBKK3i5iU-$L>9`77+ ziej&LU7+7m@%^t%D!URjj@M>0=w>DgXxLoe6%bG&4LbwNU5~>t^Yl;j8J!61k}!#E zhjS^w3x=}nC?|&LXExMM`3`{SmwRp}3_4(BKJB6Z-sc-R+mET7vlhGuSg(&-1kCx# z`C}f!%UT!<1+l%sXwOJ_+rdCb-Y#XV;v0qq14fOmUn(6v33Yt<#{hSne$+lzG_IV%U#)@9neoa<%9}goyja%cc&X z6@3|fMn1s4LO~aGKv5|t&|L}h=R9WJ-b6pLV3{1`2eu34HfuUF!~DT;;~Fg86+LP9 z+rTVrx27xk1d+FoP>d4w2JQG2FI!txFD#aSOpNR=dV2uWuk$j6tANqZ?q~7apd(|n zK=u*H*=Dp9h6hNb%D$)vT?9}YUkEJDTCKX_D`IR_ySyrbbE+&VG zNwVG*ws^Y1BKi?WPmx;*m*k4L2Yur4<-!2J)g&hIq@pU!3$M|_;mDxL$*|kf`SQm3)>5203uMWfA1Nsx06sM1>yBmpuF42`ybm^hA4JqRD|ZO z%LArKwbT?l1sod5V>RTXKOH~d8~$x{cegXx5L?zhf(IUrcvZNk^b!X)i< zFq?FidUN>Ae*_z`x*$zG?25z=)gb7Y$`Cw3pJHda!4=Ms8|+#5ZKq~?xA^M15WO)} z+zs*|IeDF7bB!?Y$w59#lEvcnT2jS4oqbg1myh3-Kt8n2UyTC(VM&c5-pu5pz09Cu zBDim)yi;e8yw!Y%IqQpN(j{ytw}J62vb+{7BuKf-v3E8sd}5+Qr_pG6+XLb4OL$B} zy#~5ywrhO$d)>giH`CReg_=f6;&XgDuJvtDGCA*xJC4VjOkZ4vX*jAlKL7R0+uhxD zA&Id>J**Lm#;P{xnC5;c7jD_IMX(A_{boe{CZqqiw;PhO9))-kMqhHsxR~8}ZK;;a zQZ{!bF-4U|uTl0MzujHZ1#(@6c)z33=zrSpF_V=)fI6}op~W&YwJ?^InGB)Od*r)? z6UJ*7qmhvSaFi)|>p+t$`ZDgzBI(~&iua=@N#4=KhWw9Y?|Ww#b6irRF0oiVWFJQk zi{ysaNCujjiGED@A1we7PHCXKEhi@Dfj_re7t>&Y;`xJOiq8>Z2wM5O`T$d6=+9u&r<_g#bSF9U}Z9cF`% z#uJnV03ZW=eS;s|qVX(kNfV3->|t>Z zvxd~cZORbDne5G#qRVJeL@IsICK7eRSfFK7yBB9#d)F8}3!|AkQ&BA7dOF47aC+bA zf!a8eJ?O`OVM_oZ&Efc!U0vY=-8&MNgSFH;CP{Y^(**585J~W4`*~nYkxXm=>FcS2 zvIM70XCkqb!^TEE=H$x_Za=g*JG5A=Cz(s|L4-#P^}0o~x;v(LGeWetHD!?M%9cJc zIRP5$vJ-Wpn&$nVS=M~!TGcoCc6(RdK=+5YEGzsP4vz}k!~?blaFt+^&jpV=_i$KWu7I+Z8x{PEL1KS8 zUMG@Lrm^SlI}n+J^|!092>K%)sE79i5C)ONj7mJ4Ldh|J z{aebK0KU~IZp(vE5j|~=h7Ux1d6x!^3^cs9l23^Akemz(wLbKw917e0+Sh_CXkZ<@ z^C%zuHvrrzOW5U2Al8w+*B=Yzr-9U!x9KPU8)^*5gS=``VE}S#LXl%vCIy8-1pxn) zfD%$t2uF(}GcZl6f$b0)`YA^lNmz^w*f=XPY;NH)Sh3_4@5qU3V?|?At|U6kZgbGo%NWU6>Kc*X}b_9 zC9261t+!vaX5pU*=O^D)KWGC+rE?dT0zP~q>)w9fLOH<91I}^$pg6KWN}Isr3*v-+ zpFsRW6d&2M&hg>Wixty;NOi(Ipu^e6vOE$d%K+5v_%T(6{lh6NJ0q3Z>mc(<@0%!( zH+iUqVPkQ2f9?TqccJDsG3hk65S%R`u~m9QZq+h==AZva1im@c5u#4lhLlLG&+dY< zf6COycqt$(HKYc6e!K}d^}wzT7V8>}!Af}8At-TS8TA{p?25nxF0K4nG_fufjfKdl z@_<9z4a24kjtuD5Gg1}lAlByiXd=0~+Zmk0KevxLo9$z>`abe5CR5OlWIed8sk*-M z>#3>@>Ld!~?ztYo^qpf8Ir|ETd}(}X4VTI3%#a67ma-81BZeyk)w}o0;r30ssi;&q z`+V$xuSdR;vn(d|PzB}13{A5fuxG;$dgh+~T@A|;V@ZQO7U zm>?<(!dX86E5ZaCA44@G5L0oacR)j#!di*>qYJ{hE8totbk7s9T%P+yEr4)#&|nC$ zL4p2q0KJK^ifo=8#e>_>jRmcK)V&mRInQqgyYNq~f;s*K5LasHQwZ*CpM-;kqXn%J z2<#|?WI1Wab;X!}O1i$E^JspqTW-qGW}?twKZw*NJ~6338+twP4j2V6%wQP6&1hwH zJ2&sxxOXMqDpCTM)bAO{jm%gIuuP7|+ft@*!|@=L>JW9)W`u-dvKxf7zH&3VD^kW) z36AZbdA1sKl+Q|nXymnDg^AW(xEdrCwLkZDP~F`8o`MT26+nOoUKPuc$SgbI>>hM# zC(~y3MMhnE78TDkvjVDOvvr{O8+{AI3wQmw@G29!i+E7m< zVRgmCI=Ckt)0Pt}Rf!X2U3JANiZi)#GPqQn0$Jb7A^xS&(Rhb<7$S+bF(#3`jl2!@ zi!P1#^_Z_y_ceT`yB}kwK*g=nNB1|pd_B_Idb-aj1)&Zq;qlmhP?FxW8>38nOfQ}! z(Oz(^SyoJH-s9Khp=-0|Akh_(CbuzcRwn$h;(4kEtHZB#Zg!5hkj3C8{QmHZ3b5lS zN@XWy9iZ10C2h0}1dF9YT5Djpgx(WXyv*XAO^6E8kXPAJd0Y+#ZKE_|Caf}_L9Kkmlz4fg}+wiN^Zm*dmh4_5DQcAhPp{jH(pCvw9t0k&T@6L254H3RlO%Ew^7 zxM64d9NZO#si-k%#7W3aL1YKfpZXX+&!71p$SYChGU3la#O!Q+`Q6XbmkHk!)Dw18o&5%yjC8G;4y{Z#XH@BPdFD%zVmQj*rZbd}4(p~MqojhpM*94ZWWFxsN zUpmI~m#__->MLal;R9EKNS6Hg>xVpsbFj1-vV@tj!Ej^`KNrhUkfILfzRs^;|7Y@t zDtSZ$x6VtH={IcZ*N8T_}KfhuIEAx)P9<0rU$7R za-u!BVSZmtoQeI#WE?Dj$0IfXZ)Pbv3$Sx_OrkPdp5LrIRUwZZ&dy>mzb0tjOdtPWld-dD<57ODtv%Y+damp0`-@>&Kg z*9rjabSY`9J(B7PaO>fJJL{s>6%x}&M>*#u$*6TCOO_nEF0egG2BE-iOE6dgwk-VJ zk3wt!A@Ssf@FkC29JJXm&m@N2)4%H^k_e8JN}(HC3IiQGukEoD)AG2C#i?ecNUOeR zn%0*~CzE1T0kq@`6ZF~Uc!?&aG$}0}?hF4Lge755NxGsP0W#G|#^)n3 zq-0r~{^*9lcH0Y)IWb$bdm!i!LFCq-=x(tpIs@#817(myjUS{-BoJqNoIM?Ahoq+9 zgOw?Kj$-N|4WS16&=V8ch*7gDhbdOX0cwGD7`*^f0cyHEjAf!QOb1*10Ohe*!$|pq z+Sl^L*L3l5FUwo2qK-1Yl8GNSri)U+OYls zZrB(!%F8aiM zWdv@d1k&&lKr`}#hs>Kw8gJ0rea<|yfdM(`eev2_V(SnoIZKi1TC)s;NK_y&0t{D* zyw{{e#RNRwqp+v4bV!-pvEg=8zJp4hV(79jM%C-bcO{ffjUCv#$%w#KxaXpW!9ep2 z7LDZ^jD~^7_v15^bhV{)D>kB6E}@HA_na#FLF_if2lw?KGWNhL@y|y}Af5n-XlFQL zw1uC8Ebv*7N5lfpJqc1fGC?+8EXzXTO?Ty=Vi)jm@~Z((p> z@a*&EJn_{P4{%`rF(xALfLa_z-a5hYz72*x zNZcGD9^KE*pYY!>w*BjYn7dt8dzP7Sm>dkJktiIddkbU+Bhqz(1Q}V(wc?W!Q));U z7vV%%!uLA$ewywrzPAfPbwzP_fc}bc!Z&~;?%UC+FSZSdaFDOYRVf4v(bV?N`u~05 z(CNOIemDdrs;l<%hs%~Nf5|a7bI?&UM^zgbG`2y0lWDwt?ov_!Qmv7zF;vPWZitra zurKrO(qn)t7Mqy>$&WLg{qSf4NvA}LQXwP-#6+f8#tU?8GiiQwd1}$pGAiK3PmpIj z^@)rn^wU|L*KUt!@hw#Pqja_Lr;H(W&eDITyFq?~d)1o0!E7M9Jn-rjjEE_B`PeS|zk z4-!CiF^CqGast2vBnqx_JraTdr}=K2$jX4&u;EWXcOVgE;uYxPlQK5`wolxRz>&*~ zpCa;OWJ53TS?T+S5j8AsI2Gisr0ufMQH5MCiQh((ntoZ#k6Nx#v6_NE)T1F@uzo^6 ze6416ZbY+@WL>YUf5N@H_N+(uWS7eiC2prg(R45N&Ms&<`qhdasCHuX%Z*RhR#|KB z3D?n?E{u~G#B}n?y0XV?qY|xtXt`Hdu9lVL6I@(aI}luvJAa3(&0KqZZ3@hOPDI|N-Jjo91y}gE48Q}f$cRHA4BZxL{@mTa z`a|fm)XOrUMmIZ#`JS$*?h8I$UES~4>HPV<48f70G)uZywAbKz=*>}owdM^!E9_f*RU@Aph5oZxRLa$ zC2H`_Dxv#&#Nxte5ZzL{M`tdUc;$eXN&SOY~kIzrJPp&&@T5k9cYRyjN2< zOPU?~LU%T3>do#V-_tG=Uv18m!R^a)NBy!NrlCv}7W=z4o>ideA>{^3~cGcRHviu!+C CgXVAm literal 0 HcmV?d00001 diff --git a/assets/example_data/lingbot-va/robotwin/observation.images.cam_left_wrist.png b/assets/example_data/lingbot-va/robotwin/observation.images.cam_left_wrist.png new file mode 100644 index 0000000000000000000000000000000000000000..68323f37c47863b981821ab7c54ee6281008eec3 GIT binary patch literal 24723 zcmY(r30zZW_C8*_474r~TnLLI1`rSfmnDc&sHhQT69XcOB3BYX6VOCqGNsU|hyfBY z$Px%@6#*3m0STs186{{4iB_me2+KHzSnGn5w6#%$|9OL*@9+N^%7o$iUN(X6lF^}o$OhTrciUF$#kXz53r)~}8E`z-Cvx$NrC zzbn6){d_9Uz9?xsFSXUOZ`QA8bAo!^+mm;WWKMV`zY3muH@)T8gavn>Cd9@}Ri27m zo*PMbuj_Qx^ghVO^xsIU)}<@N3Mxfczk*8X;kk*uOID|FFnj@xg3EXW4wf-n>e|i2 zu052;QLA>OQ`OXY93H_S5jEA`H5zo+=4)i3HP!c0f;}wUvq;BO(o@NU)5eVHMbRnN z0a=-}i<9$hxCd6B%8sVcX_T=o@}hm(4KK9&8mUT0DxVm@4t0 zW4@$PvX-IsLiZaydCG^@9swsSyhsu92loAZB6dtPwyD*(h$EM0y|dRk|YasC1Lwjoh#!EOs&^3B8JHh9Um?g5Yv*i8!FwB4P~Mt0!e2{ zqNcYL1Zek-5=a6zfkvV4rFHD7KZzffY$Cf`bIOXo!1QL!!uwN?a^K(CCcQ9xeztKs z^fHr8%Y6JbmCsZpCDX{a)%!8FH7WlT$d4y2>~w2Fl0>qFqv5jDaz??->ndm6{X;>= z&_@qtKCQuPc!?~@9<5o+IQ3& zExFV0#(uPF;20k*tYIp6Do=0y2K6OY*_a&#A1y8NHQiMVzl8!(i`%hiib>RHA8KBN zI) zz9ik+%lIO(ciYNlY0*Vkn2l9c#%J!3_PU4-(%DSeHk)QQ5<}Im+~2!l311X%rI~s< zKx6LHE*_89Ovbu14fU~Pg)V*3PW7aF89{R?!ni{@(KE@ZjH43ayF1$=oG&Jzg$Wu^ z3a9Nwbyid5F%O_wE%d(AN@C|y1coOC>G8!OBwgBPno*P_d} z5-36ic1n=UQybT@&$!|7@zgNbCg|1ZSCY&_AgWC= zFd1a3CWbP8;?PYn5GoU2B6AhD6cE4|k-X@#P?99cj>=Cs+)V}hB<%DH+bz2=dBLXb zUR2a*oDE0|oW^kB7S*RmPE}fLE>cC%VY_s3Rt%Mb&u@U3MMMUZmRwVrd@dzoR%K_B&YU^j*>uZ;TIEdT%Z8OIWbdX^ zi6&f0lm~T^g}6AYhZiexqrHN&)iE33mBdXfGQ$+xSo3Sb1Nh8BVU}r^I!gntT|*u9 z(}$I=F50af@^XNx7PK z*Mv9*52Tv*WGl55bnY*wDtEM^XfhL4n$JhDXw*4|?*4w$8(=W~l4}pac;KKrzz8w) z{@7OJ#`O6$%vTB*T|Go0-C90u7{77$?zp?IEot!CwGbsw*x=+}7E!0{zwF0uE{OG6 zAx6tsiM{(hOy5^faqT`*9T5v7`8dVoRDP555B+H;$Y{a{8?G0g4w`TN%LcaNG%A0v zR8_oUC!J=Owup*8bR$tPh{0eGC=_9MA%Qdjc@Ed^n)%z5kAz~;8CFAUQG!z)G2q4R z6{On%3knGg__FCcGRSFl=*yh@f-Gf^vZs#17U43bT-ut*==$cZc9{+rcX*GRmIN!p z@M~9wwqY$7S!q;{E48dL0VAz^F#D1DLK0PJ3P)fZsj%ouII9QTBCVV{kCd^gRmMxY zv6TW&2!ZT4k-`>=y&)7qRCG0og~wf&*E zhl@c}*3A*HhYX%6xkF}XmVqbWGI@L?w=3@p8ie9_ zzHiu!wSbk~HyQp0{~Kv`lCmo;SB6UC8{PFQUbaV_0oQRJLs>)E4f&yzK$6r(QzZpL zstx9S%R^OVyl}U&NNR)icu-rxh+pA6FjkONvvf1a32D<;4{^+dL$tQAqPjF%|qyY7@ZEj#l+tV2y6s)LJMU(}cDXRaW8_H!yE0k#bB_ zl%72ndGh{Mw?*#8qxmZ(V^e?h7Wi=daAc$MIwqJdM6i7j*dY1WNn$mZShCPCP#EZ@ zMRn&WXN_v2r*j2W8G($zl=wkXZg!9X{l{-6jFa;LZHUCeB75BlFf5KW%7@?!1Pp%# zLaVW1`{6N-cv*zBkYH#dkg~*V$lU}u^ycw9FTP7)Z61F#=mn_l-Q+VG^IC#-qad4g znx&9$P^x$lHX!UezCDG0KAw#4t2Z7jP;D$jR)mA~dc$WB2xg4P6GJD`zEe<@4hu-S zBmI|$N;mdy;0M}aMM?9pCKjzA;QV37;C*%)0D8fXDAuhaYiu7lT3(V9G8vF12FaP2 zRzPTFl)5pns%UCePbi7e`yioBWl?1O1kg;32I8DYnl>F7P1+)osw4$d=-MC@czNQl zsH0wN(K@hie~vkZPkj&hV_3NhmL*wZxTsdx;I{LqVnVOmp1KEVAk;lL z-L7PEeTUmlo?qDh-kzY`e|TFT_43y5fxHKn49+|#b47QAV}s}@bZ{In3W*T|yNNR; zE=X746^g$;qUOkvHS=Z?6f!=DYbLLNcSpVSZQeLom##?E!j0oQP1s58)RD<~gr|2~ zu7DNTO{*p9ZI!hwHjRg3W7#$BT+%UQD8npY&|Fyr3pO9tirgE_OtXMQQYy2`t_ZC0;IIe~!3)R<_aIk+Ecpysf>L#eL8g>ELf|m50BQz^uQ}V5b~rdIC97Fx z(s?5VV=oNzxddNV{^uH08gO&#rv|kL)yMzTJ>xwISQv>KZu4&PH|>hv4MBKvI?6!k zZeX_{a3|{S%AN(SgOmFptJ|sp^~iY9RNLk&%rdxW)Vei7s>zOV1-Ale;M%An<8eqM zbb}fACQ~6%!1_;H!`% zv^RN43QF!>N?FFH{i;iZR#A&71c8vKwpfvH7z6-CfYIhxW%fGdi2R7K9kgj!iuBrb z5@HC6%DfTEAMD)gi!XxHVfu6IvIgU_L*p8W%~}2YQu=8im{rReFF%K$S?_=x3$?ppAO9s1j>!sQ_qLH!co^-T*ij z*R6Z%PU0p)xdvEHnT5=?oFTJ^U=Kiz054cbvK*G;>S-dUf-X&H}WlI6qS+~ZD z1jsy$M4clJuGe2??uLu?5tt`@$yhG3a25Llm@zh*tOQI}QY*Z2cRXJCI6*PN>ZX8W zr}V~7m9ky|01%}h(^@+|7m7IHo$1kDgbUX=NtBxbl6h> z3Rm^$ES^F$_5gn1Zw`t8 z6uNd91s64lyp@R=GJCx%=pQsy>M1c^4;l4mgCu4_knNWPFb2zegwMmq{PgRP74QUs zeRkBR`ejs)l-|~)<+$#Wh(MBsmLPbbWwrK3%H+G|b@N}eM@5xW@lgus4*W)m3PB>m z>{d1LGw2^-1e(SjLZ`7vz_onp-2!$x-WI{_mggI1^;QxLvzmPkjwA@s<={W4j_wKd z@yGg=VgUF_i+sYo-L;MJ%2;K@j#pc`hgP2|to3WKXKYjLV4k`clv;n%o4&KQ(>rr@ zY{#FI&n`;82;g@n<;_G$li!0JAcCOcO|;Giz?zCp5HqKC?7fk>P2lBYwG%2pgft|^ z(~7Pigl1rXM6Ae-g4xlqvf)@tRS{$<(zai2t?2l#2Xzj;D4GQk07^z&drq%l)=SkH zXM!r7gTWX_kWV91nlVUJE2Nf-z;Qt$tbO9HhrM=p=?eI`_rY*D2%txqy>lE4hN0bG zBlN_vX(z~zVzID+3p)niqNr~#s9BMfmPTfbE?Sedh-BUI-wjpHphMoXgrGff*B+W* z?DW>8H4bIuzDu4u5uvUNM}We_j0^?>83VXmybl5g-}l3bS%m#ENQ+@^jE8Lpnb8E_zbD(rkb~bhA)yyuV#r@Zd@7n+4%T0?0Ki<3apW1Cu zP|(xsn73WUcT1qK26W^fq+y6NVgUz=L#%U#nrkXsqRGGhmb!-05*6j(LAfROgy<5X z4*ieVn*?9fX@W687|DcaytfV7MoqB!Jk_j%5g=Dlg# zuv5~a+8!R)7mv9hYFjMRZ8!}9iF4Yxu;X03$I2A$A^VMwhwCjIaI(%lx6cFO(SlY4 z(NS=uZ=*Lz$Wf1s6hu zL~Q!IZoA0bo#X8`Mvcy$QqsLU!vVN->_7-7*nLKj>_$)@GnOOw?sJv zSsK3lVAUwR|GoQ}gWH+$N3bJgM*?3xjRD?SqbfB}y6y)<1(-2rZD5kiCtbuiAf&`Ouz+}NX(De zPn}~tSNZ7pldsy`%Dk(3I^7m=2t`7$moIuZH0s}`%!38ROo{_0-OAt&#WQNnQ zA1w6YK7>e(j9<6n&cW*i;~x4(qDg9(1lKM`@ByCJ07gk!*xEV}H3JhAIoFhlBQwU#P27a<(Fh$HWw&WQBqwM(@6B_^pxn?zhK#SIZv&ZOn8^)XaD* zY~Fi@41gF-5VwFOG=be`HzY7~I=bzabw#m-9T8GHE+6+pY|2aE)VJbvQ?<4SfUzvf$8S>?4q6Nc{rO~JHxV8rzz_=`{hRHrD z5TzJdl<}LWGROGUuGVE3lTKry&_PK=Ov}a(oGL1K{&yQuWVtrw8hB;@ix)A1t4Y>D z;tl}ZNI(!m3A8$*dK(Mw#YM!Bv`AWn3bQ4$(q5gzn$G3=l!baAYiIThF~Q`ZPLz94 zlEF>%nwk4ZMb7>t>!bKK3Q&LO&cK#xtdQZD&>jMOut{rdrmm-py*G@#ONU3X9kK7@p{aen~o?m6H)9!St0yztQlq?Kw&vMB4d`U1fOv?H!fhjKMc zANEI=g~PzA*=6ENwGMa~C}W`{QuAsn#lB%Ra2=VdY@P9J3tTolzD$J^vD|x)zZB-i zRXLjg6v_prP@u<^n%ilIhVtUc`sZ*k^Z{QY{e1DOka72N2C4y7t)dQ52f%(HqY#w> zH6Okt4@6+PMT&95yRl=%+b-s@{d_cmcE-l|zwqOXF3|CS6GO!ce3uZ$_w5_usm9oU zzxOP7%UooK0-S{D+k|EupFY@qDI94rHX()(l@Ot(@~J-iVWH}j7_dzUl9)0M2Nvjm z%o@Mj2tDtCDZ(3UyanI5nx*gIlGkk*I! zB5(ENx$du*O-uHP6J*pI$T=X^00UB#bh4d4;7ZEklx3ML9hkfocGQ3c24>y`?J~GA zD$aCkk%qyBT=GTUAL4Ds{_zj?Ntg$$*7rR|lBuL1w8?zO#~ zSPGI^its#v-8Q>pDL;GvB5?|xCfR3)S%mzb4#C};3&!{DUIFXaVfPfJ_bI_IUw`s{ z7eK=OT94-qL&Q!8U7OfqMV+XD3x<1>H~E7=dxv^j#bdeqgM3TAXf}xq6hI9JK_Chm zanY!P7PI|lf*Ue;rAqzmCZ%=o!Z&YU3tJOnoOJ3;rr~%z6Hh}ffvm6&*b@RN1O4TK zyZc&HRFw2r4=z1_vA9wU z;acqi-3Yn?nJtcrarFPmG`KCrCi_naYaegW;ua8dSDa z#e(qc0L>t2%QuKXjNKP}Xv!o?(~GA1deL2<;@ii|idcg#I`vgf6Yk#X3tH3ORA!Go?kXt~dHVx1>Bu-QOi;xtV_$&`xzN+> z@uX62@Ugz&hC2jI2$@3HB+08i9^#AAl7U5N|k22AX2{8sGF7WLYQ4ZFQ@C`09+aSm{D(mEP8U1VBu7Ln%2{MVgqq4-Gbi~= zxF&Q6^?Qq37UJM-O6LR@uzE_DO2u1ymYf*r2Vxa{Wk?kGGy2gvQ1$Ntn!|F z-FfJE&YTRmAtXC29J_Oz^^N2&eIntRZVA#Iky}6UXi{dno^j@0?K*vWI3{NF+LP@{ zCBM}&3^>FHb%MQEwX4mI3KTrg4K+L=DM7@du-j3cO-hsjwE+tuUR<|x@6-lJ_}fyH z51K(5$OXj1AGQlgCKxEqoAMEO*Deh?KI~%E><5y4i=aOYz!3^0pl#vw;-Llaa_U@Eb?4*3W>`QXF7~{~Zx( z%>6VSBY7ts?(9 zFHPnFSbg{W*~OuUf{DTWU3;E0cJf`nL3xl{Sz+z$e6b^E2fq)~#Q zj*Eq)CPV+UO|?^@b133&Q~xX}1Yw~{8boP9%>kQ5l+j>m(!1#wH)mHR97{TN^Xdl2DF*8Qz&KC(SS|XGWC|xm^r~e$7{^{z9#L`1-e<-{Gya+oQ zkwWD<#~J?(P_0w1=LPh@zzDqYAsVVnuRyU4Wx5@70r}f)x!nNX9042=3^zfVMTC>b zxTwoCQMcC7pgKXC={akUv2?= zAfHr?Iq81RoqQI>X;Z}=-YX`SRR9%v@{1mG%v4M$sObtS>fA-rdw>9s#Oj3rOT0?0294qlAGbw>x4A(XfHz8vV(3lOotgS=xi~}N*mi|k zSWAGWMLE^A_d~%5KmydF?u?-jAO=FU0^5L$mI`m0DwouURiL}=w#GWZj?0a^eLxEF z;rwb!eR74{8OWH63=1*+RX1xZsVRLDo5>x(1VPxLcp_vTh0upxnoPMI!_%oXEp8p* z{4EGfAUXw^J4l|vvhJ!PKAXNVJ)q3`MX@L>vJwh2YS6_kjf^r23Iqw^F~m^Fc;i1G z8v5BqyidFj%F2BR@?DEaSETN=;Zl5QP%QBS0pFqaUbp;>jT%vca7m+!<^r+KbSULI zB_7nCl)kL2n0gwTGB4Hx2})J#HUavAS14$x5#25egvzp;;u+J39K9?~(i7u@NA4Am zQG!fP0|4qBVCXQs zw^dq*C|!D#ogyu$CHSz1oH|ZKfi8eczkk^Vv+54?_1UY5o*Rbz-1IZ28*u_U zUmWx|Xqw_c@)H@IsLHUK>&(-KlTSl#C?8hBRg6KMza{6_`Zp8r?VmYiktFSL?c;j@>==|J1njnSc;Jt zidwb>Dn4qYCQ#gv8j0FJL^veVU5t271ZbBaX+VgD)F&_%&{XBRIhyQ7pck34nBCAb zfiacllVjl^%844*!TbO0bY49#4qVnBtFMi>TKu7PH&Zjb!WV$=-!KsnOJ@4N&%Q!z z3<17CmD>TWTsq;kUsg?p0TK(%)d!DQh9g0=OgJuyjf6X@36Vk^$reAPfCrl2{*66EEtC zk@N&I4L3V$LjvZKbTxCW{q58nq=1CS-Rotj;Y6psg8bKA4Yl_xMJPoLW(aG5)x!Vy ztliI?>LfjKxB3JD|EgOJOOaNg%D8Z5v@0cuSVx3SGDDQEVmuNWL5tf==fLWtkq4aC zKK4G^qlKyp{+NkkA~eM@a&?_8bn&5Q&$lFH0PTp#Jbvf30$~J11ip9C+_RF4yIkOy zzKhl6yL2`2J{C}vrp-*?K;u;Ti2M7=&!u>e&?XZ#r7XB)vf0#MQ)32}v+e;pFtOq0P z<_Qy1X+#wAeZzc;xX4p+T2Sb28!l0uT%p?UgZgalsJELhTrOswM!1uAk%=tuqGTo} zVJz#WpCQPj@(gV(rZr#JVto1AK=FiMSY+#9wR=Xuaf&88{_OnYB)9~RMAy-s>TM1> zvW_c5_luqV4gGS6<|Lha94Q03x5(Q$?J~Jao@HzNj?gM|u^^fFrX+DO9^h0SC?0TS zT31zi+U2`Jp;BVam~q*fVO9%N9IRfh{v`dfo-FB_9*C! zjkT-K1~L`h9s)^EwKv^ejO*O%R_~MT)6fkgFzC7Nf+c(CIME770OWkeTI!ATWSV4? zj|0ce2a!PmCYgZj0%YwH(e4T+3i! zJ+2lc-8IH?|HTO2P@t-y8_J)^mHqw7*_e-kNtD_+&XnXl8)0q;D*?KaojX*k~RsjXbMR) ztn$$98+8AfpuaEiZB;$U=U&wNFpG@I;;7Z$(BsvV`YY+yU518do)e?ovsz2>_Mzl; zm_q($)n*gH1!dg0qwdmA3n5;qT4oDimF@RTLf3#{1>-X>mxK* zRkOuIr4y2fW|@xhfXCVmZ3+J#p7av5|Li2jQ(eVVZ=V#8!|dvAIp7rheDsaWP6)j? zxh#w0_`6fjdN00g^Pd(>e_xI|FBT-odtUXBsVpFC`?3c%;qxI}DEl)pjvuK!&_-2q zOc=pRuY)-;souG#aeVc94hdH8J+Js3W=&+G8FpG{F{f<8okM>NXqU08gqY zZGiB%i;U~y%2%609EAH2ezxV}WA{Cp-ISS?p0;2(dI?cS6~RCB-=%QkEB}> zV(#Ogv_3!}s)X^ZG2<~9SmF;umH^{opKPcss)VW6m`Kh-aO7g2uI)!*$|&#<+pkBN zwthTb+f^WYDrq6sDtv?PWPO8V+@kJPuYDG{B|*5JpIP6 ztClf*!;}v~mnTy6VbEG?Q>_DZ1-(t+kNW9BznF98a|(^$iE@`>HHh?w34CAuY(ZdU z8_a2M()8+R#QEwdgu4=9Hv&a1oZHX3Ca~C ze@_#!Ie$9oQdWpNs&V&i+`(kUz4ij<%$wzu#V`n|aVG_L%T++$)AB9&lV#k+x7GV7aQtRl8t17 zC}hZV^oqXhSUL?LovZf9)IEo(f3NB3ym)I>%jejyul?+Fb>6Lt;1U?Rf;`wS*;fwp z&+Rfjv@fIZMx8^SX(5QC%>p3(8mUWQIMQ0cAJkblXcvGZiH?Z|6^O8wWv9ymwJfPE zq~@Hn!ZWOEEB(i7jj2!}1Zo#>eCk>0AV{by09Qh5SY*LSHuR>J2l4~MH%GFqdGogfw&=? zp{;dj}|)n z*Nl{^Hafz1pHvegcjMAYN#Gt~8vc>l7qcr^@ccvsr9UBJR6?{Vf+tr<0RPCl@YB17 z&vT&yXqLGw$mjMYtOPr>QStojKqMS@2LKyB?9D#)J&|ziGJzK)TTGKQODu_axZxD-_P;Y&At{`m9y%BSG{PS{f#|4<@rJjd`ZX> zm`5sU5GWFQvR~nK^SevMVD~kAB7*^ZX+Y1sIF1jlt&tXR9jVHVbi;ijm0vy2JFa&l zL|}H?aTXvrKTdKaRL2ie^RZhz*7kYFhC(B2gaenI{ksoTcHO)juSA8Vh2hk%d$dVV zWddJJ)HyQpM1UnilqDiSc23xis>%m9dBOpYq~yG*M_p6TSNVF6&&Ed=NekV?QNOw} z^7mB$Ifi^%?=0*f*2eqc%nB$Q{6(pw-g>WF;eIV{7KRfD9Qy#4O51p2r*FNeUHcym z8A7{#Bfh&328Sd`F;wQLuRfV!cpjjIsRV!ISKxGPKMUYbY>jnu2k@z>0NNFhIgC zu*16gRCKZ{E^;unT&RRHn29 z4nbuQMBr5PO_qof35Mnb-9oG$V-9sq{dO{HIDN0Ia~VXMwDLFkB3w)APF+NTQISi8 zy<*kHz*X1Psw3GasJCx+y0640;Hjk+E(F0joJ~uh%(FCH3J~a1=ffryci8?F(9jnX zx!zeoc(`ZLs7nAh3Pj6E#rQ#zj;tdiY>I1-5=gginqw8H{YRB^ngdkwuj%mA&|Xo8 z)Cn4pSQZzR3D#7lvja@}O6|BkfrAn17m2xVd(BAoU%qzLMbA+KmZ&lVs{f%!9rJvA zIQHDhb8d%#SRkOE*2Y9)B~K~p3)Q-tJgN3aB2c2Q^jTG5WKw>>Vs3=8iAMlNX0%9X z!7f1bGJed0`UPeEqIWhUKm)&J<%cbs^?~Cdnc)m^`edh zot2@AdreE0dDK_q+D%I4$-d$7BSe@~R?G(Ui+O5ca?`usco?(5q1UPNJu`^Z{FFMV zy9UM4z)C2?RqV=*&Axg>z@F093gZNw{)>#CjF$|Grb~&~87#{vup-H_-6{3LJ{YVa z7l1CvSQO|_1}jPlxOQ#8czaa}l}xJ7KG8ui?AQ5FBaLgcrDi_UTvGi=ems5kb(YD# zHn#e*{fBhkEz(MSo5~8egGLY;26ApDgp_pcOy?%$0sgByVP+hh6EOZG28oI2sJoA} zzZ-NACYe%{rbrq_#kfMMVafff88ypif2(s@Ufv$nHNJv{oVHa{4ukgV@Td6jh zUD2VcA*@p_rw+?UObSy6gRKG>z*{bqNd+dx)Bk5ztgYLbj)#p^clMb_{L1+0{!ZILQx8<(2!z9B@JVMXFzWLHHg zcM;V?ig_XAZhJEvY$vH80tPd{n(g;4k!5e=|FWtY)U8kMUzF}yY}3vl+r5*2~d{^nv{W}u-!21 zS{lJkGF(^(8cZx7(5L2X;CpdhbiQ9mHQJlt;{$R!8S7dG5jw0sLLFE!wvV-JE2WtX z{=`?_p=?aomH=eGC{8!*9RKC@8YRpBjQsvh`B+-=Ua_<^+R+%1Za>UB=#|C2W~ah? zAEQvF?d92MM2}uP&6>zQcfm{XwSPyFsO#SF_|Ka}!VbsY5qa9~?Ll&S&L*XEmLbF` zzxV4hGrN2kg$@qqAM*CGP`Gq$SkfjA-H4MM9V)|gKL_^eqM;Z6Gi$^&>fb?l;yT_g z)})^{g!7xsPa_+%T|@B=HY!x$5=kW;1j zntQR73p{{bns$MqP&D@ZCl4wQhGwAgo=9O=76Xx=z>eRliv{IzU8o%es}`^;ij_6Z z%YfzWJpzE?S4q0)QICvQdwi1d^}qc#x;5FtoYrOOOWoSl0pr#(S+)m+Hlwpt>8vxP zx9`*^%Wmi7yt%tvp^H|Y)CgHH={yAUBg+At+eZR7n?$c3}!rGSU7k*gm$ocuG_#*2me&Lux<3IG*?Z8-#$(^#AQA) zx`1F9tqU9)X63B?oK9sLb~`$rY1hoHaToOja195#@8~*9540|(}`XA z23anA=@qW?4Z+xj(Mz=6$SZ+}wP5!i={_P%q{tT7alOiEkt=j-2bHxjgriNL+Cp-W zC9o@K3Qq4xpy(wft6=ryB_g*6LXTI(vg zF-yjOEaiGd>7pzRW_?H8iYZbhWUGwXSixcOwZLoVzR8gPuq)AFSyL#QYE}qH}<(-gMg@ICnp@8e{sXdaUJ%Vc=DV;6Y z?Y{Ng2vH!<5hemlRh$jS`bsMF;pyFs?!eWB=Y_P$-ZTiy>@U*tD~;cu?DMGa^v9Pl z%a&uUS84>njnT!u$kb{?WB7~AgKOp%Sd+*Cm+!hxFkE8MYbp!v!}3A%LAygGJ!Gc#Alt_Eq#Df>0k-bqK!XQj#99o!g>n_9Iu1Ah<* z!syi{*Vxpa2s_`)kzYmX>r#O+czH?KiwnXOMrog%9!b(2COIv@vd(r>hwF&FCxvn0@JzeySs3R* zR0V37xJcaUt0C@Gt(NS(lSBjfNhi(kPQX`%2b2jnn)$Ogj49bZ@1n@8d7bhIJlFyma-E0DZDPY}?C5p%St0HC3BkH^!};-<;xK3E2ohob z)(hXvz6w~KU4<7O-Y=?=9 zbQsMRa;yz2APSPD%3aD8_|!*qk}B;LyzvS@l)At%3u1izpnT+K*Y+#kL)GU1vk5z* zL8l9z9PqE7D;^f$_WYKl(v9F>>X_S)*HXiifQzk9Iv=m!bGqZ;-@OVCdMR9B5#Mhz=xWzeth%gAG()fhiUy2 z%|zGqTa{hA-Rc9S8iv2vbMnjrPKDXUZXfKPX~JS22YP#z9o=pf!-9m{@{^Dacw8?^ z5A}(BHs=R|2Fx*ww18tvH(U@6Twrn_diH>&tqbBg0<3L$fIUB?5!F_1#AL^ZTn6ak z>-Qew zL>o+dA6gUAHTKGkvW?h;%CmRGs~dV+`rQcS&vYAwPeatJh7lhE7kJkUt2|7Css*w${*afxGyLnu1TjNTcJf1GJChLDiYZ5r7!MnUp(+E= z3UTMmd|ITm!x7<)U^KNmqeQCm4)qlhSLDXD^v4oST;h+$S1&E5f9q}^`{vE>V*iej z1f6%{pA(LUwZ#XZC1H9R6#!8op=MeAV1lkBPU)fYV!N+bI@I8cDiblV*y%b* z(nLn?WM6Sc*YwnL|BT^uzVTLYi$b?b|8zUQldJ=p-wILxGjsUxWaJOR!USEb{#F7` zZg#SR)Ol1Y8>|}!ABXYNRa>Sav99#och3{m2M^ibNq_gpOO3-+Q&;wz_gRTu+$WiP zLD9m)_QsQQjiJCl0j7(41)M8HF*S&ZM8!gKc99J>3(E(HQZzhomkBM1a>gsR|M9KL z6nNrjgVIwyHZ|VFI`wx0gor5!5e~VNI+b`No$R0Z=bc9fwfC+C0-9VSs}A8aVe9}s zl1&X)I#b8VCe#PYl+$DK`NDfuQuq|6MqF#2$(uxGB;yDY%*)IoaR0Z*x+ebw;pe>n zLzU?>oc*{<>=ZM#y|My7ZF<5C_8$F8=c~}|?RCq&hONImq}xk3q{@IfmRaq*^Vq<+ zT{mao6Hkb%&N?ep;_Nr*H2%Sp1I1(0uiyJ;bRF(ci~3;QrsJgC?}^f657QQjPfu9X(t^OlpTK@>*!~4gc z!8z0K-+-+59#_Al>;-ez9JHSbtc2GV7N1V zo}sYXJzsV4kKE}=10=wKtg+lT55yV2e8p_n2#mLKGwvMxb6}2fJNNS*Ci9(wfgy);`2LCf3`2fWH0B6n9f5It;)Wo~nT4b#sA;F{_ip62JZ7M0wUy6a z2lS5U`rj5$zy4kgp&(cRvI#GOAYe$~i{emsz95>Py$sKRPHS$butVB6@=IoE$K%ZT zhVwv`FOHP*MPYgDAzlCKsqVn*##__AAQPXJ6;MlnULaDz%z~*i)GDHFAREGf2r?07 zS=e;^2)jTvGrtHMN!1?JrW3C`3059~q!z`oKNdUytOpU&n&}@qfzfctT3CgC>9ij_ zoaL1zwHx_qxCg@CJts*wJbsX6rnrh7fAAQc|H8QzR2)&29`0SHaTKkT>jra)h>Q}! zzx_j?K_CFiWD}sUPOsc+X}}!JJWIq`ogE4P07u0(Gs98OYS2Y}AsUc{!V&^HP|$Jm z+)81hZ$43l2f0EmHGU1rfXF>@b&z;vo^E#Hs>OF6N0kkHm^uXDfM%4xFf)iDlrINt zk#j78QVvfqfqh{3vNI=rIaOf|M~t_uBSCPjbYzPlI&`lZVD9zFV$*2kD@EOjt~ZLGxYxPxWDh{He>bp54Wy{kfC;_~)+uCG3xqtFK!Wq*C!SpH~yI>RDz?B%8Bfpde;w7s$Ugb>|) z{*?K)g4q9j<+$BD4}O;in&OGGDvY``uDx{$z_%gFI;Yrn_1rCTW&F@&2p^X$oM$q z`bUmSKh3*Jy|wy8SF-ui7MJyN2%m&}|M#e`$V-=gnp}4Ii~Z-k_x#JOb@M~9)6#~# z1JNO0nddG#F?;^4&&(eu2pcw@aFZtJQyzbu>~gt4@$D=JrC-8lv$Q372WEw=Dz>~! z`n2>@xW$PB3qSXEe(rhM+x?TjWqaK(X>j{KFY9x3s#%VmiC;~HoBP1Z@W;|T`2!M9 z39pp$4WA{uEX`fS_;@}2U*B5jY=i#B-nQzDB_j{^d|(OM`L`wLm!;)0IM7lb_|Nng zoXgd5E%}J$U8|2xtL7ajw|tm)@xmP9fq?1|?GqAQ_9;kdi&fr%p#Oad?*BLtT|e{t zuhnjG;SLP=Z`2bc;6=)do#lu&)OSAhzmMjBHCp3@whb#ql2No5#^W_sAm5E8uh3{%rU| zMEDNg2>m!@dldQ_h&d1Tb@j&j*(VwzK8Ev{G-29}>FB2KyMB&@W4etOlA!5myl*W} zaGcH27B6jymR?$hEYb3oGhBz1m{0=WKj8OS-nYs;o+E53fz|lfUHB2~;8u@mEjZ2h zPBTlJjNHofeQw?Xn~&M2zeR#BJ{HS99150d+WLnr9R1b_j$_&boNR-+!>+H)<>6nM zHnMko!LsLJw>ps8L2)xSvgL!^K73{79RJTBdBslQ8_?YNN zE&bGaExfnjuLVl}y77!$&;kb{E?Q~7=#7CGU>^FN<*0LBnB{jR-&>xT`D)&`Az#5x zK!fPN1IPc8!Q(HH602%HrMrM#!r?-`g^$+dedV^r((;oQ_q<@A|C2ZpwVaENC6Aw# z_c~{m)O0HJcTCLR=3K`X(r3*Yznu4m*tq)h|Hp7(#ajXJHO9wQWwSm>Ulk&9UpxD| z5*~=L`+@T;6gIvx-+b#R7z{EmBu)R8kj()rzP0@HR_wwpCTBDWv1z*@aaDf z{QjwB-hnhYGUz@8ZNH#&*)q$F-x+W$*d-<8RxwJy>7q?ed$EjXLDu^9A26?>rwfo|NO8J zR<^iuY1J{yrE8ysY~sl;{Zu>u*0=k^Vh?^<1Rtr~1EM0@-0ku#L;J7T9{zlm>9hY5 zdM`mh$ekN)^ZIO>+r-67wB=DZrgY+CV?izbO^2LzrFXoXpEZ2&&TWC+of<6ZUv2f5 zuAc^{IF?fOY2J;Y`|ZVnxqGI4I?lWwSkY9yn%ffFz#gcJ2%dawF)@1R&E#bEYR5U( zB{2&fj#`?|dh5=z%Y~*5t+RGHIVBg8zJBvk)5$e(-u>b5W#+ZFH`;P)AAFhkUUO&3 z`;i|Lr;izXY8z*N(ijli^Sj&U#IjiqCFOX;j@;i~lJ316Yd_p4`SRW??(?5C+m39% z5&)VE19mN)xw%l8SDg3@3O7LuK5K|P70CVNC(XMvV_kdi9RKp` zKb8b#Uw?bU>Ej1)SOX(j!BZV~y0~NQ*6H=4|lBFP}mYL z+4o>U+X(mR)BmrfYYj^(-NI^9TFqpane1qJnXybQ?bIlh7FL>>m8a272r5}a1{UTm zub|Cj=5!28(J@S#3@_leo8T?MvDC!UGR4Llo)k?<2@yfA=X0907)Xlt zD&&*}oj%SXLNF&ircIPO-^W)go|_}JlU=FFp4MwX?XE0eB*u-k<+sgy;vXNeaQPlo z@OsbK+?WC}oH$ic#=Y>@<&h@+y0!rH(k+|5XQJV#UJMS8j&F;Qs#;tr3p#0<;ikvT zhAyftpzA8OQC&_>-2ayN;ElH4E0ute$iJ{?k8aB9p)@t759~t2c7JYe&vjtC(d;W@ zj&p=HJkC!Iaen(1g#o$`6NdTc;P*3689KU5Yu21jzp_|LFEfjCG~*au%nb5$?FC_}va$AD=)3J-8{4^o9AUMPmHw7BGe3Y-$Qim% zrUq>nLqqS%o@fJ!MuVsY+nttNI-c%ghneyy3*g6n7CRuKM33d|znU82)N_+Wc$@c9 z?+zA`EUH1}NZQ>gK&!0;6p$ zE_TL(y-C`!m9D(nivz+?kA%kgDd(~P#@i=^>LL>s2L(sL{Xv8Shf7G{@GNdn2(c9H zC__6}hSX^%3>}G-Rn6(flC4fl?%C%AaefJOIAOi}ks-^0`-!5SM1-Y{zWVvT8{Y9g zgn}aBD6#?)e*1e|iR^qhiSnc6>+P>}BO0Vi?3%V9Zg79yMfGd@5)Bnh#1eF}vkvpq zDI^US?(&p>`lDv}&ao|+e0=Z`$LW2&7kXJn97%nhERhlpVmXSEMy|A3r+kW3rp8mE z*VuvCg>*lL36@-S_C#l8(O&Y@8+zvaw5UPe9%{O3dfuVaaUtjpUD*z~?R|WT z)JFeVDpIvJ-tb>}C(dibj?}|Jo4McJpqgC|UDr>=+K_Olf#U zEFz~>sJ=sl^6*e_e#V22^FLzHXf!5dWA@=(TD!=gLuxe!)pG=2_PT}Ha@)&fIS_`m zcB!B4$HcOQH*JvmHVeJ##rXDy%lnf)!iO$0AF|Ftkz}_H6dM!S%YxWWFbbc9f^IpE zx5mola({pS*x1os5nFJY{yx11kZC%K92uRIl@K2aiggM=@t7y2Zt|Gd@^thX5{1}S z^+P5#$c>GGMCxvLXfY-cT%0T1I`huhh9az~7VFpN7v#5%zAbDYlk$?gMybfl2C~vs z^rZ5(0Hu(9m%LASGGB#ssoK=R+qT+Jy0o98%Bg?1}Xj7WoD4|7{%t z2|Fhkrc0>%W}6bMYw@FSJHNnp=;N!gR|V4kxdC5_{7r&QzbXssXA*byAEb|+F`Hnqc9n@1 zfQ8Sz=aD7cO{O@CY_1}3J(`&hE1b#vZnY4^2P8iJh!s>Fva~#1aL0ZWaE}5|k4_>~ zSY>a?Z7<-@F-6+>Tbb6$=SMGEdF%y{qY)?T)&y_9x?1?Ya#0>URkQPOz)XLgL|lL? zE-Ji-55$T(r?4Un4F$1L5C*3Gwxa5QamKy$(7x=p(Ym??CbNaf6kmOpyDNb_S>oZ9kPlV_#L=&di=LEE9&*Y%H@@n$8K&yQbw9q29O50l8T|s;pC@`N;E#$y@N(! z0-QKR7^;9-7mh-N3g~phh{qW&TbK$NC_n~-YlugRqA5w}^ zn9Vq{C`yh_h;o(*i`X5os!=8!CZ_=7nb+!dZB*-rLRKX}N2r83JeVbXf!akRKzo+F zboy?Moo(RaB4TgcSn2QI;qQL~aw{yV(>SLoKc405%1n7F9j#bD!2L}=;$=|2y0aqi zVdK2Is;s!58!xvpsAk9JCTm%6HBSMvq^B?lJ$lP;6>(1!Oi6J_@r?f%(kz^y{_D{*zn& z)mY_Mcw$fXxWzc)!uv7B{l_VH??Bsn!a%z>$KJG+e1eY|R zo~tUmxp5B$hTJ$9z`95>n}vcEy+@0P&~HIB6M^Nw@8zej2YbXjZqFh;ExTM@{$5XY zUL34osKzcL)qhffKd0$GSnzijGk)E1x@qrk0>!H*JKn@kYG?>iYKn5EtcEmHpCBzR z$)s|LK^zp&W>A<49>jUVN(~@DSugWem>Kt7vd+kHEH&A=DAyK<3+S*AwEbK4tB4s@ zJC!o4vq?2{dPp~RQUA5$G}%X-T&qhKt_NM`$hH+Gg0g(Jwzs#c)pHx^wBo( zL;%cz-ez`Aw%6?K?{2v55<6O#(UQSrej}HUx*XuCTDlC9fHeJQrFFfI8xSA+1E3_1 zfby+9;rFfpm%M<~Po#O)CCTZT<~8jPI?7p6jj%>oGc3gHfgmCj49+@Mk5iMQaBJk+ zJjczoMXpZ7VRCz^KLasz%7e%-ot z8-B1dvt74NgaWTW;4tuc<*ruVx^;wgKbRf13lf>E9|}q7u6OLHX*ZI??QSvnv@)pV z`(M}RWN}`PRm-nqleK#{{l+9R-cNPX6S{e$rmLD_M^$!5v)K_|A|)}T7}2N)Y!%^2 z0?QWH5{1E&ZSmU((pXHK^6NUe&_<&qn&1~qkEG`|-VcpNhj)*UC@P$>@%{-PV&P*v8ZGL^Zsp zroGr3ny(nq?60E|Mn$v>aU%IFj86IfyauOyJ>lx2o%D?uy!(m0IkYh;?cIp1Lh}8k zK#U=7B%Z3hPfj}pLrlT+Xw#e|XHPMZmg5@xj>MQ+5Xz(33T@s59~91Ry0 z##D|*sNy%|*tiD`hWt-$88a?n!sRT}EZ2Lk0>5k5uLWN-F7Le?c|YgIk+Zjcc@$w3 znd6~yB<)PxkeAbejL~yt!B_ldW*ri;Oz2_4rGXCR&j^jeqG&&(itslOO`?7hh%_k+ zENIC~STT8GHdQ-?LUYn);T6*)Q|dgdgq`+;6^# zh`}pQzS(ZD{)$z8gu6f|aglq3F=HM)h?a60M3YP50+nvSlQ- zaI-E1f?0_Y?(qItV-W>eOy#k-iX$9PH^Ls(H_PKTro!U!!Mc!@_OiJ$*MMEczxPfQ zOgQx#*2ip7I5Vouz2KzS?0B2pRNAv6qirmxOm)kaej^k5*N^p|k~M8?)CLDt^*$=! zD7;FrePUOLO_5=%%&Ke;$Erx9W;`wJbWHh9w5plA=XR#3%Io4>V{VO?bUj&P_m#I! zT4wH#QfBP3Bl>=dE5+wnAA_&sm47?_c5~Wnsjx98!PFXK{7hL zjfACYiBSoS$CKL?C1z(Nbr_`^8HkpG?~gRQ2y3zif=RMTN|g?_bd#!|x>p>&{HaQ< zHhg|LYuu+@Z`Z)8UvTjCn*O&_3n50jB#Z*h-b1P*C%$Ayt7DVnfs8X{m*V3OG&fr= z#wTnw4*U9bqBAeSW=8F@xTrFIDDeb=Du&(H^kE}ceZ=9HLJV=$Uq>gfE&J6L9YVHX z^B&c(qa`s~(sqAY$F<-{a^QMo3*RGrimH6Apm~t-K04jbN|)d*C4ECHz;hd;CB9k- zchsK6`-Y{u5zI)!;{+qJD278}FLHMg?PtZX%@H`}d&wPzOAb2TTH7q#y}U>>w|;qo zw8V=lm)@VN8>r^HnjQ*YS)T9V^LxvE^on;C7ngVUPE=m7r?N1wt)q;NG?(u1*x?b^ zHqhbaKeP04X=$mTcq@Hw9$i_2zQ=AXgUCo!#>wIs5lFP)$E21I1h(nT&_0Fz!-`Q% z3xX-Db#LF+hR+JROobBy!Q@n9Z$SY-OurFNi(xTD1RyQN_|eh|lY)zy)MD9I zZ`8NL;m(}Uf~AH&MJm-1c$2$AbP|q;wY?yJcT$^Wtzk=fBz|F=1+2ENL>+rn$6ky| z$#&ByAS5Q-{TatxC-O6e0Kb=Lgi)vIJg0fM}Gg7KgaJ`)N`p0Sy>iLff<$3930Xd(t4-*-d=7zuK)gLyDM?# zTRc=eAPUd)TsqL&(J{8P^l_kOxY?$5xwSLTM(zGqy+}z?c328x#6Z{)2rfw(-Ry7! zBE@{vf>6L-I$@xZxH;8UM;>n=AFPMqMt*He}ctGY@P8D1_mFq zlt@5+2-8Be@XU2Y%Fhc}_f#Cq?;#D(q5!(GafK;dVFg*7R+zF`jAL!Zu8H6-uFu7Y z5T#S7#Kg^FrJJPP-nEaPZ4VdBc1<{(9UoUxQc8n`IXLuo^}WShz?ZR8AB%2@uQ?NU z#_2$7sp{qU1D3<+%2pm~CS(cbKk4TsY*kCBkALlhlVl)j+c=qsHRH9?&rZ=DF$o*} zWVBM;^-^qme(sUg$);Aekw@YwCL{!tL&HC!5E6qlDcY%2_6dRYz-;5->h0*mdkO5# zj7iDTjU3Caqtmv?dNGVTSq?7ZZaLj-u24)T8+FwZO$#q^g>W_n&lY_|QwP}msUqr) zVYrE@aHv_$wU&OJPG4P^>TNB!KQZ-IgG74Z#r{BxGOz~}@dBiD1Q5x1I0`FS{cfX) z&bmV@pI=#sG0kvhY|1p-3^!s-l=jr5@xw?@*>B?P!_uYC=pm$}6P~+OrN9{H5~@^V z068_#@Uhcj$lr-pfIHKd2N&b&b8wAnnZ`dd-4!@n7!0X^8}B7Y9L(G5i#h8y$YZpz zBLsJa0yc9gJUs$0LZz~(W`dtE+FZVL*tDm~@phYXtJloObCrQ%S7%mdlbaKS&HCZSE1zeZe%WYd zPYwGO!TF+&1F_5NaXy?cPkqK6X4=stnG`Lo=Oa_mMsjL7E4Z< zVN2=Dk;mmA0PEH{h)MXuQKFpt{+{UE#1r}Ii3EB_Dd{5K5r{!~PYXqb=41z}>_#a42Oa&M@n2WV#+lmaT0R9;?c^RI( zJ^JzExiY8eexn!R!huhphU-jLT0cE(&LH3WfDN`}QwG+LJY*Um7_pJETV$=2`z?hk zStM@EODic)8`MQXob<#agWSw^spgvj#K&waERf z)73Q<)wM5PRKLjd53RZXb*m8zb5QlOGNdYx%90+{EowU#r?+I?Q_afET%FJG*=o9U zVK@=zxs7AZ9)v$E4a;!194&j2?Wd0XJ*+OT{Q589GV29rZ5kf-9G!yMao0t=Q zIA(=8j!BEDVj}7(U-)HbalRgMMzSqBSTRv2F>I+4f)m-h;JNMfYaG*T?4Dq8&%OYi zRmXooT^?VezB|SJz*=yOCg{1=9W*tq3kvlGce-}{dd>abiK$$tXQF1z&?6n>+h+=I zmjH_SKkK;^fBV~Fg3a;itW1;DiB6x)LrWJ_4mMWEwj4$`RRaC{VH8*an^Ede@}Iw6 zT6sR=onqPMe@vvZIaWnxclIvad;sqZl($~XGAtl*vp>1_u}?7*jkU>HrlCzclTY-? zVZdGVQB2t8CU-a@;)pbcb?4Y>sI15tagN%yM5-i{TKSU}3(+!Qt}P>cbuBFJLs(o7 z^}Tj2IQSZ@Vru^0{`L8iX3QC>X2+&0ac9cnN_sBcKI4D&SzlGS;KRu3KmL>~k#L)Q zujE%5usdpc)K>`{>(RN&W(iE?KdVEmaw2po(MLTYsCSPSI_DfQ*@@r*sU;Y+Nihqa zE*T+t%VNm3){jt&^zDfr)KUN#a@%f^El>5r{+GjGAO*CHy{o-HMYB!q7)i>Kj7`x9 zPBwa1q~qoBFn{{$N`jtC4gGomo$l-V&oWN8-Os*(&!=6@2U;MNRuXsSl0AyMx4&sN z%dA;~50w2g zR>q{59}=CU99^Q$NK(m>x1JRtk`uW%Z3*JG1SWUxY^E{Gm1ph;sEBPuc>lKG2zu4o#ZoSfuxxoHIjKt~~x-%kCW&t{)sg&x_WfBVc1 z57>8bqK9o25XF%ai*+W~d1-lrVe=n4yrf=JR(BGo5$+M4x-<_+f~qJ1Vc9YZ!J=yiuEi*`r>^Ek(IC$|AUR%Xk z-D%8P+Coa|y{ctS9v6HP2>7tSD|1~vJ^aSra!6_p-Q8%)?mCd4P0_M*iyKmIb==wN zc>7G*4i6{~+Wh?dzPrgM*V^FarI8M=7Bf*I+(cpme#E#Z+VzRRyMciyM^Bo&YgiC8 zAb;WjBSs0?`HoMr(R~M^gdzTkCnp*XZ($I9qY$>N5q{0fbl1GSESkwX?w8T01qm45 zIj}yLhOXm8A&F}f#-PG$r)VwNH$66MNqMMiq8pO3&tPrM$6REi+_QhOL%9 zP$x)HdI|>eqn#MyZVb!O+>T;4sr4ad5r}Ptxi*V}xBFwfJ=3?rEVAZ`w?qR z#9HTIsYEk(cR+R~wQ0vH=WI+IZT;w$!wX~gOz&qI3uhY(x{KT2zTE{M#ZR9;EiU&T z4>8Iy)TBL0%8w&oErgwpYr7B!2wCQI{P?tS&=tRs<&k2WEIN2b(-3y}cgi&E0e~cy zTCLQwQ%5Mf<+UZ2C(E#7!$Mj@S+*Z(Opa9?1&8%iP=;?AX2{!WL1Ke?lnK1m7D|so zY%YRgeEFO0a<4b^=q7G1{Taug=JeUZPH4gJq$2rd&tSiAQ!9TYOOH+F&6SOt)&|Wl zgQxT~Ha3oSb#=|p2TWfYpEQ=k`0l@X5JJ$Y$!q`3eF&7?v&|0%id(Dmi62Fvvg z5eQf25f-cCdM-}3MQwZHW`=%+tdXBO$~E_%EyFNrY|r*B6pB?YcO!G%bHQ}aUjB)r z2qsu>BPQt-Y6K#gM3w_8gV#nd`{jt)jwb}sHZg!cEt!QWj%~}ICMToRT&u%YK7Wrq z-sjID`+fFhv^|Kx^lkq9zGuhgX0PY4v%}}gDuo}GhF(P$+o%bFWGASZY?9WDW4ns^ zSrV{3K9))0Q^AW{(qd1bz-LC)A_#NXu<4pKN)#6Irn?4*fl#v%_MmZS)0V@NEiT$v zYk~ww5D=@d$^X2Dw1vruuopy!TW)YBduV#Cgas@JMq`1n)CJFt0X(2LS(V;vc(&}^ z+jLpS%E)-Ks)rMZ4hISV*y0{OYQq zh5Pl0qe_mbfA?G?^dB@?Bt}wnn07#RwQ~Q$<N>e8D5dU*VaUEUor6Vaa!pT5m zAlghCy;nX-$JHu|N)Xy=$hF*}V*|Zz=husJ>3f2&U1gy1&{RJ@$06vvqN5I+%q7ydCl4Tr;NSP|^@V?3gO5Dd z@RhDeOUu9SnUP#6IF1#1XEIO1a)yAJ;!qR7uEX~Lz_e2~GiX)}Wn5B=wa#-4@i|J< z-5-_OR+`6sH$Vy%D_Heuk3h*zP~M)TErvL$ks3BawX*vD4zLycF+Ar@95W)u)Hki= zrJ1*U{XKzTp>0kHmN5UhttTTkHntV=4A8|z_qR@EhLKj-zRe(EDer16g^Fu(N3*4+ zrp&Z)$d@m#J`LyD)X~FuS00{jxALS2bQyh!997#>a~escj;`Ne3FVVJq;OTpl@gHP z7|2~lB94)I2d&2b1xpc&QqAlmAZ#UWhG>|Kh95-p1wP@iL7IetjWC*5AMxF?YQPq|NmkprhYd(TQGHRf5zT? z`7ce5O+Gxw|4ofFf)Z=YZOP2FO~N>?P0@vdp!;f3{*$?D6pPyZc39XN)j>_WPu8J-V8# zLm&Hl73o?X#OHNol$Dj&RoA?zt{w0$_kTNfugxB8hC_Vf&g_UkMH}ib zVsB{P5xM zFd*Hlhfy+_^p%mXU%z&|irn`1amwhAP+x(2Nitr~?1kLC=0xC4ey0doc3S0Peo%=a zP>kVNiH;0BE+3(W#htVgp%QZCb&0Nm6}9)$$o|-NJFQfJaRPxwdVM-63C)V%t+(dT zp%!IJY4Wl}$S|lk$f?8XXffP5ZJ@lz386_X@d&Q$Rg`KBdLua5WcH$26d|B+VQX_>r3+T_q_?V0ho8B&ZHNN6++uovUC>!dZ_Hb+*_=1BW^cA?|^Io7RR$*TCul? z<|weJ8jwJoBz*}?+x&lm+KLc5)nX~eEOpS*J!8op#A8$bNS`JCB$-Op1^pMB>)x)= zblf1r^nMM0nm^~D=UQuMT2)a|QT?Jev)3>KC{|heR7tY*wYZWvD3VkfNtDGk$HJWGSmHP_4i1+tco9@Xxo-T!>3X)mBhMacG!VbfJ@M_z@FgU=xsrb zt%U{S%1fD_*=COewy49hxArBF^ut!C%Bvk(?#*t~VR6`^#H7vC$(G}0sJDKKQD)Pk6)VJcdE^vqtmw;6PF~U9N5jzi|!wK$ypYJ}>nj;la?k{>66J z+7SL26f9}6kVhbqybCaTh$cfx23tC<b)|F?kVq^{Q}WVAxV7%VcHw z>#I*IZL^p+%#rnyB@a_xyAXtnx5)3;ufY%$E0Gj5!{bLqQxEP#Fwf<<(mlzxPoT0- z+Keu2Aog6NlB8hpuln_Y&Sx3tdY{*xKJn{lpr_GGW1-OD`7d$P8uV9*Q5vUgyIca( zat#jn68;=V?j)a*6~QxjhO$o(DCR59U_9S6wpXrWazGNGZQr;74x3+S_W_bVaJstg z#oyk6Rl(jafAzUPjlg~Rd*5Sn)4sH`Y>SYoF6) z@v<_o9Mt`p2)l(ul7-r6@Q*8NFiIs5)IcW3Qc2~&654T|#C$7IkchEVdhS{jmgEUq z9E9_SWO2C{1$y!rYEIbkoK)1XaP?Vf|7E>1g^GV}i{>`C9??wJk`VlNkO$V&Ews>X z(dJUb{3KvicB#_qLlU!lT_6pkLRJbY?vCEx8x|h2+}5h(BNQwEQ|@p4R!}e%P@Zvd zpv}K+=NO*7?%GG?PGyfxoqv?vIWRHrRTDV=b_}4a+4T@TNLW8~!T48d&XN?-TQWyp zM`+)q3|Sp3*CLFuCX{2G*fg}afr{W%Ee(6E#+|8Z{Lg%Uwi)|QY( z)XAvhljW#r^$56|h#`8T{NR59jHT|t5OwJF@Yzg@iQmJF$bY|4PMye23)%pc03TwV zxA*2?Ru+AZH`<$Ao_98HX4fvLseZ5OKcugB;c0Ltk${Op*AjpJs7Lxuk97Nq`3q%^ zMbZ$ZP@aq|#YQbos~w$?#A}$ywYY8+)mGFoeJEcSO%{&@xol(xKLULj_J>4Hg+1IF zhZ;v?zXzQh-Fr3%gvg7v%f;i-6vFA@ED~-ud8yjnCXT|k0pR+6`5wbThK2XTo4@}G zu#1s&J8Q|EOJbs2O)(4V5#N`poDEJ|jU<9e^F@YW3nT0(L9i;AoJi9f$gDOtHmX4( z7QC=G*jvB-bZ=Xke7XFV&V8yL20L$hoGf}*&{gkM=2R>#qc5Y@7w#7(_%g5(+4cAP z*tohQu(zo;(UcurS8pi0ObLmuBN*zaMGjD8th)xf)z;c(>%F0USCTh5S|ALGPilE^ zQ4opcSKn!Ay z`J8v=I2s}6_cL0GIFuOIvHgk+!4Y>d-N1C^^Jr_ire@yRvom_%4HaCHF}`LfMQYIH zO%3G7=Qnw5ia-DS@4|N(Z6ReYYN}7sLh|d<(ooH}sLo9ou&5K(PdAa}ypcrXvyZs{ zG4y>S%zRfZK^wp^R!v5C75V( zZzJVvt65tFtZc0*0N+7D!wNU5PVM{kqHHMCLftwh?*{4--y z<;(M%4nA(YgL-2I#FoqD0~3*v^9@mpSzFais~109IVnPAG!AaZvKhRMsizd`SY&ZV zm<6FOmu7-&v3-Kzz-%n|2G*8l6pI4W@5wJyxpkGBA9t^eC1Xh*&M-kH5^ z{`<@CJTb$GMRh74Y}&Wo!6UBg!gHs~U{J4`zf}vjnpC6q?5O#cao+fZcXr*svj9Kg zN*r3wit!i`yd<*51Fk+}C03N3acIBW!i#C38QDaMD-6^@YZW?dII4=GU zy;F^&qcSLjylo$>>75GjpIhKv@HTnjKi;YdjsB3JpeqJgqN1;^$&o1F5ds}C5=aRB z{BAuLh*Cpi<4{drv(46l z(BGVjEG8A?)Bq>49`QDc6fI4~k(ctsN|mJ*OQCXng5Iwc2)p2%Png*k-8q&i=8V^w z#hIoFQp`BTu(uzwWI2OB<5U>HkXTdoZ{1rP``RG$C=8AH|Bu0ayFAaIUy+;2_= z1g@_3FXo*c%*w2-t9ajbN@cDZytDd6^_?aIx}b3;(*$@+eMrr(r(lr{V(y+^q&kb7 z4+hr*!JpH72G;yJT)4{ne4#4Cr>oC>`yQ&8>T!>cjz>K-5TKVBTQ(xBkU(rJ3>mxC4u;+?A@>#okukp@I7k2aUh+WL!%ET3g;KRl*XpX&EG%!vBd%U}qq{=ZDkl}y-LzPE2D|_}i7A+F zY>~K-4fc$(eq=jlXkwWd?ugV`Ka$Tvix934H<5 zi|n2%ydG~%pw`be@WF9i@4-jr{hCnWDwwqY@_1#iw|C~LH+B(hbfH&|b}Ao7gSW<| zNJd&3%`-!vJ7BF@s-^}n>b4$9+R( zI0fPY*#e(yXtG`WNTnjXI7>)&mks@|Ge)#1Bl~WVk4-ezkGa*{m6k?Tx1bQg$ZCJpMa}iHqAW^JY}QZw!2L}hvT`5r zo=~4~(0udsha8}3si>CS*vwoa*l7(4W^KqysqFn5XBS&C1zYwCv+=kQO=}{+QAG~a zprDh>YC#L)yR;HblO)n`4CiFZf@fxM__b>x0jpDigS9Kuy=ruW>xT~~W}xH3$urw{PP=yRl?5Zz{qWMb*3?Ltr8HP=XlQ6; zd_COP_j&(xz?^rjz5g7BZDw}W;MNcKn+Cv_PZl|3L_!t{xpw93%2G!|UURcd{Pa^* z#eA&C`+J+R@_5CX7S0<*QParE@_&L>6#6oeh0hmgApet8K(Z zliD{-aNEUoNHzvq5$@=_lc!1r!;`pQ>4jENjQM`Bg5j zcc$*!=TZG)n}K`x@B5TjDIUBEsk7_z75^(0hA@it^=oW1d@RmvIVSBXPNX-H2?*K? z?@5+ny4$3%q-%}-*@oaXlcU=(wZFY;Y^-uDSYoFgCF%y6{Er{6ujrf4M!J57CXsamcs2P(ofG;{Th=U|pc_)vzVV2!;m_ z-fQ|`nPV@dEsLn9V$=~^l`xXGBs*U1ZTsrC3?+2uGBZy*yEtDC>>jI0&rqV}MChYb z>!`;`sb=NQkeY!V?+O}>GMW?AuJgYx4GeW~exrE1+?~ z*-x|sjRkFBfy03M2EcBM6KR``P5JHbwzRzS z->O?x4^@ON_9PE$!i1A$)DgT{+(qlKY>s9O@kSpW)02F0ZQ0|EvE7bKbfXEfU7Pd))I1u_K;AKB>)O^k=fMYLGmiC#Lp!Gt_l zXGdh(_94I4tba7f^5UVs0zv$M!w5tpoYfKIu4B{Z)hwmVvnLvsQtpp7; zWWesAB7ct^0_TC4sXA0~#9#Zp>4CGWQ&~a*x^N$17;w}>@pFxs3yo7 z6s`u^WL-L-%W}l}9TUS#a58sjBpMSJFisk;2mW35)T>W2XQ6U|*{rI5?<&;b?fdoj zIyg>7$Csox?JH_h?sUA>`0S(&0EtV#;R}Dk($GvM=#yJFFewy%5<(!=?G#a4!jmwrO$%ZHN{Tf~pv$-@) zUXp?2vl8zn0jHE(a}Go~>7;hf)m!aJ305Nhl@|k7r&kK}ed_3ZE?|4>=;#?ng+8-u z;A+qn=%^11=b}sxfq&PeJId>nkEBQ7eA_q_&*GO9MY1PK-JpF~a}Z5t^kA6lxc0nv z>i6E@%)q(6L+wAo|9()l^Em{&eLVSCXUIvYFWW+(oA9Oap=F(sNp0v=4QjTi*KRqY z@|_kRbBYM7VjEgGP_x0CQ4?XH{$WvSiH&T>G4eOM!DTp`qIObckKSy6DWAH3g)M^Yi1)d4hgy~^n9 zoP#Otf+_bvkIeCYRr#GMtWXrnA*gSW4V(5I1Awb>QM+`&GEllMNboVxHOzPMAmR)=lc3oMUr-0HMn{*4)81PhvsML_61#DWhRb3@$snDeT{8sz+8rSDQ@H@W>>Z?D#MXUZ8JP0B< zNMsCjiloI7jcSamX}C*H+hM;W>~M!hs`d?v_WyuCh2pJX4LJ}bB$1s<%$EccOI(xk zUKf|s*T3}#3Rl{?i(TqWj6=)smmkDNHK1kiri}Tqw;lKMTvdfbJ!PSb@sAl&=iCU` zT?o6z3455pO{aWtoB1jbz>I{;{c8TuSHWi<@bg~pVh~WO`rpQ~v#oG>L++aZ#Z;3$ z3Kct_ITgzs$S|^uve|09%zKqw6&QbWUmD=H0&*cc<;dzN6X=#swmhUQ7;;du+U9qJ zZmR{HB$VY>0DT@gbrTX$FBG7Fh31-R>`pW%gg17#XLTQ?3N)i}H=qPdc?rBn3#iZk z79VHDjS4B_E1NR}Bjg{oKLr;RPW}y_+<6DBMIzWH#$D+g5R#xZg39l(8t@SSrjv z0ZoNjO$Zp_4HHFj!gMOe^ zE8x9)*eoG2!6?-X)zNwrfrWp~b0=B(_treRt2J3aFK^tT`}f}dfU5qTW0euvR$@)a zy~p6r`vIg6LOoLKFwwRAcBsDNe#5Y3Lgp5=!76_yqGH7Wk2@*(L=fuFXmJTkY$>9y z-*!a=mjhiMTPw&eAo+9H7uN#2G;vKAK(iEe{q;FPa9d;egM1ZnmhAd}l7LC)V8Tku>)B;QjDu;T z%HtvBf=_OczYj=c%y-;3g4hg$WnCSYj)j(6VuC{GE>OwATK3F}QJNyCx4^3jYClUd z9rKe;3bde3fKp0e42j0whHh&%*+{Yh-Ro4Fx!`Ox5DrMk5Q*F9+YwBqLQ4@kfT0Cq z$;&&@7fUrODCcZmdX`Up?f=x;ttnjT>h>9^t*L&Y>H!FJ^48EA`H?;#0rjGB=<;Oi zu-^360hRTNj8dd$BPO+m^bkk;$)>#>?8j})KwUL04Jdj*=J=F%dGEBMueu_}#UmZa z#DJqAP$@{RLBI7B+Cwn{MWeA1-EZ-24w7^iM!m6^QDvSwQ5K`V`dW&;!;^^p!&xvm zgh9rKh9oNtk!-G-LJ*p}UP@~b>s1I5kCZNuP>h4>_J;_3%HQR;CB{0t{)KUPT zB7Ij?uznB!v<4;$lW1q+fl#9}FOywwEXAcHXcQGTrgv)LTiKkCORZq080PB(2Z04c z(X$_Fo~8>v{=Yf(LjeiNwl*vz>};8ebkD}W3pC3(gm+LgtNZgBAo-E3gMPTTST;jY^hLQ9ImaanUp! zgF;pn>`yf(sNwd8LN_G|`LC)_DL-NdbBz1a|ku;|6NvWn^%9V zi%PZ5c{3-)dH$!h&FWHfObuRV^`RXB#Nb>-s+Wd4XosEHsIf;JlIMrw;$^?gVJ|;8A949mmXHx=n!D zT>N@ulcB43px#^8>kA)eUAG#qE{_78w&}bAITy5{u8^8^X=&-{{+gAJ1{qTW6T?G? zlnrpI1{*&n(w?@l-uOqdJaG41?}abG$YIaioP$zYZ!3tfX9lJh#<=OYAdelYfSSiH z{&KXSA+ph{=4Ahi>wuq;QJT$`5>cgWCJsAd;p|&4!`8yGWpO{OX+&23v$}8`ON;bW z${aRoZL7?@funWbYzq9#y6CT%MI1nPjy&LY%RmH$6A>(pE5u?20#jjKrOG+!CrgXA z3}r2}nzOC`(dzqFk5~}ilt9}9k~^DINvPd_MFub4rtP~^733DZ+T~@v1OO(%l7Q#*@mOKz%`BhdA;cteyW-|0;4gxuMN# zU1KUmvFSPXj~s!9*htLoBkS#+icv*oFCw%BrlPaTYbt*K9$Q^ReH%JxH5nc8dcPCS zcNt3-VLvK=OpI_ZrgGSCcIiP`70&;3HsN?y=IQFn_ig^8SN0dZ3kB;*>q~&} zwA$Sq9&|-G5OQ(@f=M+_z3A^iNg{KCWb+TJ>BYWSk8|dE31uw$w4hT7~GucA+3k_D^R{-F-Zq z>ADz=l2qkrbQo*x!tN<^ATS}(zb_y>l~R;u!B7yMz4dcTj7`j2cNKbJ(;XCC-iHS3kv+59< zETlz2M1rHh*S58KEb4`T2!PfpV{@j$vsZ70TPoxO34-Vc1Lpu!fUzjJcu%RiG-|Xi zV_&=A$RFK9|AlxCmdWy zPN9u79ojp`6*RhL!i>70hhNa$J2i^(nLudxRlRScec0-<`L%LGA>Oz~Ah_|LFF1v~t?`cx6E63zAdpNiR59|5zl z%?aI17%StZKLBbb%C$5jZ5v##MXMxo{x}=H>fhE__XH0WCd~hWrxVQJP41|_W8Fz+ z7U05KjnqQJRr1@bzO!Sqb7e3xS0`AW=pL-@@ebT|vjAA-yV!=dWeBbr)3BuvOP*yX zC0}2`9@L>6X5HBIWuD&(BX*|23xcl338*ipynU+rrxoW^74Qdd?or)!Gqm$r`eVtp zSe^LdR#p){^R_`kQ!{Y-se|qUMnr_TW#4c_gqh;dx)G&2{0-685r_f-$9@Is@V|2? zID{H)o@<}aLLB1my5XcPnTP|kLQyK+L-t7e5k)E+w`;eD_6@DRA}V-lDh20qXp zJQKPGHnt!4~|y&hhrF zL&o1ePZWb+2V@4KiicMX5Rs@T3Ck#nI-}q(9W(U6nYZ`5F344qZnRJ(#Y6SpwhP;b zE%VN{zjZx*sXQ$$tv7f8kh#BFNo4hTjsXlWkZ&G*+|@DEF*H=~Ra1*bsm$^CnSuVo zcd2-tQnqm=QDg9DzSxbW4Nsi42?J2)F#5ExBT-n$$V;-;&dC@|197*;?mblEjchZ? zZHaJ1GAOEj_3L=$UDs1$7%InzNg17ylSNMiLZp5VTzG*tkI?xqL<@Z(i<{m0i}zTa z)xh*vF-*399eI^}_AHbnVWIrruT-GpET6x8`QlbndF|>~aBE8$wSk)J5YJfK7U&`U zA`kS196O)ZL7lZSS6o zCgexMm$?7V0(fCDwqcfcVqF_DTFbM-`P|u_&~)50F?)oF`64Oni?ejwkKJ#=kB<^likZgzfr6W5PzvUsNi8 zREK6=gbo^mGWPv5<~f3ao*8sOz~PCCv+D9=Ui1xn(LB@&w5zqV6J2v{2nc+Ckso(m z_`9&XpljK!%miF=I7*GaR3#CWam%BHsz{|>_&U!o0Q|ZS4c(CWWxvqM{^_y)Y5-}N zUfYEsi?L&*$9V6NYJlyxG}_r(Wa--i95`6ke;83FYho~ zL>SIVEcW`^;U3NK_`9x!w%RFb|5Of1Lh1_wWv!^R5Ek>0O?Yp-!9qTlt1`BN^&%-HG|JFCPHNfptL_ zIXKAAFKlu8!buf;zC5BuC`EMb22NP*m4elqt!mpnl}AhukESCqaUsXv-`uV&3VqK? zdr4BPo1+^RgO?PMec+sQ@kR`9XS)gi=N()ntrTmsRcn#iPFNy1O)d=;7gNwu6$bwl zL1czB{cpa8)1e&57 z-5G5cP8M}XO|(URTU`Cp0S()kVh7#4AF?aAAO@Wd<9eph^IaD76vXPURUYNN=pdv1 z%mwcRay9v8L_8{^T}f|pynG-EykP?3>6&n$KT1=xIdHnrLD$2EV}<=SX9tF_Ek?p} z(BgndNTICqoqtmTH4VpRXFV!SyeQ^Z^efBHJlfohfU~0LK%>ONAQ1xpRThbUw*~DA zqO=P)BC&Ry3)EC18S{t+CzdfVWKK-v8lmGnQ~?h7I+=!Q#xPi(Q}Mu_ts)7vj;zXQ z?{TH^ad;ZVx5?RIy}ISkJimz%RBVF9!%goybnuV>>V+9K`qi0ffO(d5dZdbHRWJP- zeKifpw@El1=zw3<_vM%OwzbiAhjr6bkAMzfaEttIqOHCjQ2zRFPHKa7-vC5ci~>R8 zVQ(&_Q79a%k+WtX55lC`eq(I0 znlYW1*4YV8&Nx`q6!N_1(z^nfh=t`rEPF#^h=b-$2Tdzj8OS~PB7o9lm#OV3jzrEVm-)L zpc4X)tu=9X&HjDGKy`qUN50wV4t;BBbbPLDb|qBpo~qhlW%;ItKPjFzX4{^^XqZm6 zfVNiXiDMY>WD1A4UqpO&mT=Q(6d33fnXS>FNIBu|7$zfx&Sqvy0(sJSG;!NLaLzLR}rFw~|a%qWpazyLYLP6Sima!okn2n7YD9dyQ1r2~2 zq*VOP*b-XQ!J8kh7&P6nhlwh%90)7^K^3*(GXqcPFzUAV02d{!cC!%cK*2f*%x4_ualpI!CybN7T3L~?gMr2sK@>yv|W7i*o*YA zD`7lgnBW_@;`>VnL~l$5eCy)#&*q^OZdK5HNL5XBA55wC2Mp}gd79sl6HiVYqFr^) zXnN-2qZpMj4<*m+Y(E@jo9=&6mcCu|tV#=2yE(DtDD*`~im`s%c4M}|u|VKGFhe8Q zsMXEMUovMZH-Iml{Ib^z^(aMAaVTDQ*j?2^@*`|a`OJ$YnuFQ|P{G)`?#>tMg&)+E z90`6N;$Byq=3-+lQ9e)SjmD-8RqQcrze?OJq9BYErsE$uNjNZL+}6EqgULeE5WnS*Uq8WF3%Wl)z&jIqTLtqV zJw2Yk@Z$Q!M;RGY!BRuQOn;5>@ z_J5e#$34DASroXL>-G6AIGuL9J@fiJcQnJN z_UTwJc*}m}dl&0SH!S6sJzY zG3jdt&QgtJ)%DH$z~KR%D_8{TSq94C(?U8FwZ6;TG@ z5Iap08VL$5uInG!t2-7uEsEZ z780<)9R&zD;D`oU2=Mvys)gN9H>0wO@IyVoHk?qLsDs|2i7~1$j0TisEC$RSDpLYI zf$g#B0UBjj4WKjRQml_=vgLO}MZ?LVYJH~A8M^p<(;Lo?gPu@0`if``;^7QbjCrF> z#Y-4thnx&^2q?4R!vRAUr6n$Z$w=&v4|!}h8v;us5&xRZw!+EFMRS6sxvoe3FAxZJ zxLFe-JribM-eGdy>niR-a)L7%ZtG(H2%Bx|ZLbsXIyyDawrAD_1()<+hPfgbtwFI6 z?ZTP#QlU5p@M^*dt?jRmKMui2aXSg z;)|G+IC%^@?fcRSN8?|HNfBjEDmuhLU@niCu8QgmJW6EaeIMm7G4R@oibs+*GZY4G z`Fi;PG)NN6IolIBp{VSVh?r7Q;fO==HlqseQJ=QBO#VP5U`e5eJ-6GIoU@Q1h3;ve zYwXoO+g=y+b?Ix*IFR<0kjLc5Xsm*1bGLdQfB`k(&~QS6_ZIqUV;VeNOE8+|3XKUA zW(V4)Ix@R&HspXvbVOxsqW1Yfp$tk+#@-EeyqaAZ><5cZoZBS&AOgn01;?b#;eji6 z*%z&6AKHP41)zx@fWhUUD`e8F;>_4_K?&%0p@fv>-;#s^EBu(QvdKz<@D3wl38oxk z)A>1U+=vm2#7T{y4Jfc9Ff4ea#~!Md1I&3nON8SXFpxc~_tde4@>H_){&<>_Po{C` z!qTf`)YROJ*2>`h$I$To0|UqKr6CynRm*a9K1An@%<0tyeOvH@5sY^KZh~7J z>Zow4Hb*+j1IE~30AtIg1Ca8M$0w*k-ZngV@>uDeBhFube8M3ma+UlTN$|ub=Z&^7 zM~zR#z~iJSicn%R*(`ixIp)z{_!NrNh`v5LdKm1mBQ6XcF(xZ(+DN3yn#x0$5MkqF zC1(RsLy8hN!jnvZX=|a>8j)luw>-$^x~9=S9DQ)ZS=N24v&*zg?LpJfg~^WOv#wz) zUHWNhuYp?YPYkWIPX;-*Dtvrk)cYI%|55ez@l5aU|KHE&bWT(!_Aa73%$DV*qTEC! zYm=?8$%d(%5R#kY2;JdxYO)#0jLJ>gqGq!?n!8B8=NNKCsZAk#n{G}M&MG%y{H{6Q z?;pR%@mM{kv%Oz;*Y&)f*YmnwYZi+)N8i1RxpC|LgWbH~6>CCPYzV1?P1{^~tyL?j z>KOs=(9y`o$wAbcs#^M71dKl_^ z%BO2VvoiTME^->8hV`v83}g}nJ$OS<}d zinYn9qb4yuu-!Hmt#_?UT9|y(9r-wKubJ^R9RnTp?zFwSE8gy>?cJsyu;f1FTfY7f zIAlZoVU*J4{u*3M60}p$Vg3i|A+7G~>+qVeu?}h38gMR0IB|31O6;kE2=2bC%$E? z>y)2A08)Uxla~66b;3)S#+N?7v>Ze&NA3GY{r>*nU$avo%J(%TX!YB3-+|^80=T`b zy{ze@p&(90%YH=t`@0iA{2TgmnBT->y1+`}e}cMN!fWkfLuVKrSlJ~|PUrIPf74-I zAcOD)G6;JB9T&z9W}fJ)a17C5|E{GaI6%s{)ZY1b8r}&~vmq4j%T z4evQGV>gx*Lz=LAO;Ad8VG@A<%o8jAds292aC8t>uYdHgB}M0VNOf&WZ??U3`AwcJ zFzvuzLCqFj3=QvgJM9F~;lkwmpY6BlU;<>Zc*5SjW;gdPngaZ<)=X{tM-Z`tDreR= z<-kf%tFe~-Ggg8QCBrJR1==U6!P%&-5}{c}=Ac=~Xm;qj?-!RX&l z^a5-qh&DN<|HZAN9ltReZ|BVy{YOb0jUm1|xtY&I$}b_C-JXym3S~A}Uo^|X?}fU- z&vmZj<?e!L)P}+9B## ztUL(=L!=Q^Ldupn>G{)-)STSf{bRUu6jTy$O?R!8B{8HrlE znIA?hg;$w~a6m{nfP-;}5wehmL>quhYJ9fV@Jb+_{YVE{^H-CbXX3A2ggPLFU{xiw4T z>;zr-PF1OlbYVJxRK=!ie&Krsch9{@T^OfFclV_Cxib-t04vWzq;*s*KM&|xphtULmxlq4pi+Nys7MjVODVelxVHS0&74Nti~ z{077N+UBbHm~L{EhJ75hVpP8I?%geVelK=Z3#yIAu_2kEnK_SNMNKtmQkwds9cKsL zcZZi~R2_FppgBz4x!*i}>~~(Db0G^vmBMv#`!$)6d6IhMn#r>>bh32tc@b1JEWnE$ z3JQjmT8+4FTKIctzA(G$u7ad1f#+r-mfAUoxfd9S2Cew??q$jv4-$^*!|~69&*~?O z;MX`l!K!<4byO8ONHK$hAy-Y@Ga&23dl^+B)r=}@kYKkx+hBF`TlM0u2tCspAGaw5`ZcIISGe~(tMuI0g02j@g! z$r0=tJ|Tij$90nh(WTMca!;N||FxEFV42_QEle~mBj&m7V@sV5u<3Xb4#o=G%sjdn zYeeN8V z{M>}&(-LiSnV)yj*@SaNx0}`5imMXr1`o6bxN|Fe$C9L=V>$EN(2-qd=S*&-oKZtr z{-{H)+bygiRVvD5)p+3#yw*v$B}0?g6y-g$P=YQEuZx!;Q=|ylxk@3oCX63OB9w}l zw~LDiOuCRkl%dgg^R5gn3=Dlc&aJ}1l)#X5vA)Gj=jGKp6RH0}YO4_mMOtqTs~brd zQu|I5%&OY|)kb|Rz;*JknkIr|z1i7JFIZ;@fK1>ToZYLJp9zIpXkH(l6R(jaA zhG9Z7F(J}{@vhcp&Ad`Q9JE`bXbCSFYfE%0wDwV%SBgqdCL$LEmTByeK&Z!LU>QHHXs8PS;SwAuMOamVFDsi$Y(_pjO7bY}8XOVUtd3e30DK}l}; zi&MKcx+Fx}b2B?PnTO(TV*k1sghuKn4g6jk5bA$MJ@ z7tFAIFp3;ex|Uz6N`Vm==2&iy%wNLCFyfGKEDgbS;dQHZ+>{jw`4KCoU^H+NHeE8w zZCdDz)e(82jnzllv`$eDx5^-+h8r>BE+9PN3Ij+&S_K)$fIqAwQRJ#BYv*SaL&Yg4G zc)Kg&PI=qMS0_Z=IBw{6cWxXs)(vauhrEg=X7=YM&787bt7)D8D%HFy{ZS%gMkvrc z3M{^ul9Wc~Q7o`wW`mpn*zT|unmkDM_?B}ymi(9lRfwc6-)VWm{wF_32y7y|PR!s? z*!h%jUE&k80weK%o7D6zYl{3NI7dbrDeb~5vLg#k$5xRJFQ+9j3-V;5CHz14I(sG@ zU^hLYOl?+gznxK|Q;k#euUt2E3p;cYW&8d0zS{9ELGyoWq-!9RG&iOY`;Bkes@Jx! zznt@t;nixrXb3VhTeRxSnpw@{o0O@yN~K@WuEy!N($TQ(3}yil)?b7o&)wq?=X@p( zDfg#Qq|bD!Go-#K@{jjV2X>*Zq)flKQRXMO(?9zSx?5He_pOGb+fNvxqqa zDmrF$HCW#hx(WIIax_CX!8`&bWv%1Ota%jW#%h+-qe>={SsPY+o} zjQ=(%b+(qp+OcRS^W2K7%c^w}e55rfDa8lfgy za1b^ycZhAN4WCahC+T9*(gNB(4G-xf8K6|5K2yf*)@>`K`l9CqA$xd7QC28T$8;~qvC7@ z&|Ni@sc&MbUWK>8YrYSr22)LzC7AE=47pl( zDc{*Du46CW-UqMS;u~&zAUMaLR?KAhn$x7J3#)UGK19`cji_&IF00lJak_rn>-Nj5 zSDzkzbb20o%z%)w@!Ztp=K+P{jnjrZxnLhRJ$~$wnfavk6spt6P;rekB644#KEcsAI0yWy9B zgIED1(rZ?Uj|D%514lRfn@Nyn4T1Tmz8jk!QG8IgfhH*cL_ycO2_NL8(t3VbFqPvk z4^V)md4e{qw5%+6oqvpujyOpVrYYS$i^1?4B*_IUq()}@m8PI3^=AR90~kaoF_X1=0++DXsIyJqP_}yD>zmj z)~cGfwF}>@q&kwEH+Rq8W9B~-3?8T&qbc|C4dV`2lV2l_s{K5>P=VQ#BIUa-OBz4`f`WW!!#U?4H$;fLhgSGUzZV zKi~!5M(-_1iOfGKmjXm63x$AJhXUWSxX^I=P^yi5Cb?=6Tm0{fCF&w0ja$H2Jf3*A zo=g<8xzdtPKZb_w0}dxL|GE^BZ$L5n{@iwH>XM_MAFzGXlPwUo1~r|V9sB%qg40I5 zwibE$A&w!8&e&XP3;3!~Yi_hBZ&uR+C5x^L%I;?G#1I!e*2frKX+ z_$XyaTock!@Y?SGv;bfb3@evm2DMc&;j0nEdr&>rfw_je3 ztJJ}URkH$K-&#T{;9eP;pRuYXUZ}&ajvnyAv%%esX%W44KXZ_&)rQ8z0@nKE?L>!}Ut(%Po*PV$^nE2Gf0h%uv+LbrdUcH(w zZdRXm+6d?$#@y|&UDw#ynAG|h%+2DUcfX1?8B1D|z4gYPZ*$W0l25Is-5#u5R=|ZN z03q`zeY$)MYaKhAIpe5Lez+#*@!JZf-8ElSoZo9h)m<*d!c27+sl_m!XAt*k7ml54 zist?^urRFwr6u?%8wn$8bn_yx3_fNI=Y~VW-u5C#BttY|4(CnHcizmW!y=BM;ACje zIt=R?_HG)Z#YTOU@`rqLxBCU#OV>Qx`Ar+DO1!$arwm(z#aGz7;3SM_9PBm1lQQC$=$yuXxEv~&>>>fh(*|T9=8t$RTolF5W(iTVHvuYXlx?5 zKOs?gP8Q8)+Tr*lh8E9Ks*nV~POh)H&csn1nXrSfFU?)F#H}K-hao%Xri5;j$A(>}Dy?zd<(|)btmQ#l!0KLSigGr2 z;C~nKen1q2Zc$&_o0kGvdp=J6?ar(wam5NL!H!$Qhwv-UiEZ*i5Jr0Dpc}V;0HxZW z#z;fROn=`C{><`C?2{M&IeI7yU37{U zjeU~c6Hvq~pdFBrltYJTojx=ZQk7v2I6|&|oX>?{P50>YB48Z>6B*QE!rX*ia!WZH zk)hSOc=w7wSKs@`ihk;qLaQ}s$6V*U@$Sm>VfD{W8}H;IoF}1L|A#g%gp!%wQE-5h zx#_b{OU}=?wB&jx>ecLHT_M;BFudu#R0tc{GVda*}0`lxo97>gE zmCPE9gl{q7HC1)lc7>~iPEEoXb|X?&fWRShH3iMP z){?~vI4mw;t&1~6(=1S8M%D_H(z@L1;e&^B??LcTlKe#@ewnTf$A8g|_(YEn0KS}- z$42H5@(n~e77he|X}5qjV`o0Mw0zJ@nvG`q%UI_yjBqzxj{lA@K9&}TFfF??h`M-^ zZZt4Ek*MqJ5{RAI0ah#`?-|e=_%w7_91tXB@@g|!LLP-vN6+$aLErrzL>{)w3t+54 zo3Jdp*IXuILV7|LMO2r8@#j6mkTFJPV;MB|x`e=T0pp+zEvW_FbwR(~M=&!Dz|ZlcGHaA5L0;?UH}vrI{Dd@C zdsoSCPDeMuG1U(Fh1?5!!2`V=fX=mO2Y}(yz$n+*w{KopXw9nfO-xfx#1jM=uSSUQ zeRSnzXECvu$)@lSaBPq`plIeixtu)JyKQG;rCzB(0~`VisT0 z6VTVbMmX2XP~8qsQx?u0WLm~WuaKduM7|139|7Q1(b{tNKC<@z;LuqfufaD7=(biF zJ~XM|P_aB6Ej;B@OLI{KLrf_$B<4}Z^Xm3Xl$U!1dC!1kgo!_KghOj(Nh!vCe^$PL zWxm&xJbC9{)BOCbUwW_lbb|l-JHv4WeGnMAg@uA`dKDF}#RC1~H6_V2?N@`KBeK?) z=(TRM=3X%+Yghm|$*)+F#EjoTQH{1S`VbN^g58NGr6FFT^Df zboo+)pXp@DWzUvLD9E2TOE=*|D8Yu#C@G;uMCZ!{VOe!$0Db>7VsJJ!`%?J~u32)@ zacV{|+V_VnQ(uy`D95tgU6di9G2KP*nP~+rZK_g3YA_A!#pPNad+w&|DS+rMnrbf& zkf6hWbO^RbITP0IJ0Qv_(8;2DKKFt>SKb`y&wc%?JItl^zkfC)3j=^UvnogOaFvl3m&gZ;xm9@)A-?9%J?z3BRDyF#vA%+dql*2(^0ydLAcrY(Vh2F*fFf4_fq-G~D{-6f zS}uPvr~z7^jL-XNTi&x}ST~{Krs45zs6T%~^L{7b_K8YYT<0-yc8v|os+W~X?2shb zYIXZL1!WoS)(Q%C2hYRa-M;G&!t1T_Xg4ap6T$uy6Tx@sbLwrL|GEqiV1%xLtm!Db zWDU7!Nls+x+BhHV?GtC!14*DPYFbE7oo*XH`bdj0cdge;Z;mTi**3gm!?|EX3lhe> z{Z(d8WpQ-P?7-OXF-dW1*WQaS?H`D~J5UvJcLR!Br*p+N(d3ZP)T}FaIi`Lg(YVa# z>XXdI_q{pBc#j`eDAM}p{|q@l9b1w*|7{K2+;A(sL8HVtOt@1EeU~N}mS5t+k^mNQ z*5!AhtB~V>R$s}0Wo9CUzB$iyjuS|Egznxx-f5uA__84$fem|uE?{bTTdEJ1Co2Zl z(e7T1zN!LO$qu^9NTn>O0chZk^vjo>)K)$T&+v~CW$RCy1g$T=ahP^+w1 z!gh(X>e|%ZH3OTx5ULEn1r6=eO!kkAj1&&X?Y=v5*fGAVMq1{9M|K*A7=qX1DMl;b!pqWXYQ{z_pRM|M(EW< zDqv)-lOk%YE7_hAxEy4h3;xF)h;IaVHsDS9Ko3(;@LVXj53`!eGVd976G_yoXR>2 z>LWm8^V0jquKAAIp`(pw6CUo|_~Goyt}f5K8tzYkD~*%#2f}(cOwOKzM+YHh^9OkJ zmNh23&e!#Cwfs&ODHK$n7y;TbKvDLEFGm5kBv!lW&^{(q1NqGQLwH?@erCs-tDtS0 zfc`Qi(6UYJ=hrqIcO*N$Yq-9vt9LsYV{RPY5mx(YvO~Lt(g)f=>^t`*b#|@;uB>>I z-oZtjqwR|FUY#P#fjN&+1j6aF=qAfg@@2p)7N3W`$I#OO#(tLQUuKX8=I1kEVS4OnDsO^vZ);Hi~DZ)bIUuw)Oi)$i3GHL@`OsE zK7H8xz4OanXuh7rW#z=xz_oId9Wk|^RyGVAYg(8d&YmCTxX$-qO+CMGajK0K7;SIN zu)r$m_B;wYhGx#=LHJL};zdN`mts?XUG!B*YU=+!T#O>1Na31+j4zw$01|l(PQ9Er zy^cKhBNk&OC*k58XvMOV?40nYTNX6p)YQ3Yb<2x8ev1bCJ)* z>mUi?CY)*Vu}}wXm8Y(O!>^lQ7L2u)($3_fxK4xt>1hbKc zT+!@BSPBM=$H&f-uw4D7g0HIEOlOXi^sr2}j^BlEq(0efk#Y%14TN1Z^K@}O9{jUU zfsPFjmT+Zy%La6?M-O1uTwMdLftp;;%)JHiuwp{d+1Xitz1I1IbGWEDdPzv;yB?>2 zm5`ea0KK%P`AzQ7u9?pru9|s0ZF%avQ5@JzcF!htJcb-G07*I+HhH*&>ZRXwfAOJ~ z;}s3$xoCcsY}0&)Y~4gQ&S;<2;^8@nCk49NxIuCX>u{`wiCAF28~!Neq%M3ZDcSW+ zdyfU2)YA_Gw07}Rk@V!tN0%U3_pV3Vv0rn4Xluu@Cf84T*?W?|zPp+_Gq5n_IyThQ zlnGo9mr~V*N7)Lwuut|a9wc}nl&2y&e;UnSel9MA_fdrrBhlDFoe8nd#CKq7NI12W zH=Xc(NJPszIM0GSg7;i0W}cxRpMUxm=ETfb!ww3Yvt#+?JX@*@d#LivkxC!FH9_!7LOD) z<`M1(YOk!c9uD34X$nBXZ~X(V&{zCpEF=#*Ah2neof_X7BHP6C@cjXNgt6~193?$m z_u7!@bl%{4#ccvx7Uoqbn=-IbGw@M;A`?Gl76tssX+L&SUZy_z^+z_mm7aFik8MIz zVpf|&Hb9=O6Q(J}cmej;P2fft5J^w!>P}B8{DO9$nVVKBdtuMT#m7tUO6$9NyAry3 z%efwj+BW|ASh$@Gp0oMB%csey@qyW|(xDV}i;3&3W_;z;_-Lysrke~|>_R$DS+XDD zPJhTf6HgFsBQYb>-0{(5VJ;oV_WY~W8{7&2xD+6c?3*@Rw9xUxl@Oa5_<~XiA-U+payUFJJmOwiOO{#vOTS_rbYe`)qLYg>h~EacD*l zR$}Gsz_HpUkkQ|PAgs+bB=J*Qpu+kGNitTXS0A8dz#?7vh3dfT=KT0rF zk(XPGDY5c^hlcg)ePm%EG5P{XDJXW_NFjy+V5w}PCi6`ea>^_90J{WSN{QG=eIOH^ z5#VadZZcz7NOA!?KzKN#YIpSBa$}DcJ8aB;>8OQ1@h=|06w)2cp?&;g|HKfmWdrvc zYKO+J4nRFm%4~0J)3kOs4#bWRrN%}fRTIS&i|%E_Iq73}8Dwq9Rh~Xd6Q79s+CE>MJ7+l;=8~5Hjmlsay^zQqxS$80CHPY)kjKsBTEl zl}TzbfLeJ#4Z!#eayvd;4N9E=g=ZVIBKxYEfiwN!!5*-HiHR{iJNpCB!!8|0aEIfk zt1z4WcTq*K5ISuxBiCe?%m*Vmc6tAlRsaqpW>R zT@#QE9Fl`>w@>#?z|G7VuaH%-UuFH|hgQAI*r;r<=J8>ByC}$yT-@00L1ZDo{ zUY`#*j0z*QlzB$AY+V1Jp+y{ty-Z z;qZsPoh~Q{FhY$xUTN=~STx<5m?Wq5P*Rixru$v^!tfLnfCc5F3VmIo5oK22j`@Ev z>kUvthXjD7F!Q>{8v|AqivIMtc1;i`?)sx*d>teJ6+KV&W0zwKs0mhGs##F)usi`? z5U9ihLK8#Rn@e;dubm_E3`{M9XHJ7>FXcG_va{Nu#K;@^+=oAZ{^mFY7vhMulYMm& z6m)i~oxFQ?y12aT?Je&N%RF8bsq6!r_r!Q>SK!cAx(HCu?)Bu(Gv~ zHep824TNH_-40LIvOVVx0yW+U@vrt>}}o8yCGo?RDb77*dCBC?wEk< zpWtCn7bSuBxdTukN6i2;pgn5+qe%@K?X#97PxhKL&6PBP5!krv^l#R;km5u7;d~}c z@5)~;OvhS(2}=+c5QKsRtJ1DNqKah_*{0P*F^EQhnxwkzD0jii*YO{PvtZ-!b1d0f zkp}#3hYv)byl2kK4c3BDwurB%5mt?ha;L4j!A*hFAB0@952z%t4uJlg x!w`VX zt45m6Yp3~MOiaw>5mJBegMbf3#VkySHP{L5T{&7jsh#PLH$P8*(%P?gSDO|F#>bnc z24fs|&3b+I(Q>+Y8&5A5M?rPi=s0K%I4pAx|xjA$uiT2<-VBxguu=vu24|&0Uer5Xmg3Vs! zZMfi7Dr=r$f??q+OxNwut~1}1-vCZNN|~G9x}d(gc>mqV&JXo>zj(R0gj=Aw$1_7= zzB|ChX=0iJ6Vo+g^Nq76sh`#1O`1>LxAu>xPXB&!sou(|$&OdLWrJ|8vVv&t==rn? z$Nfp%XTcy*ioVb%Tc>6`chCW#UggCI&&x$D_=7&KxJy?6q1%pB%wN(`dWp0fX3&tK zOqVBR2%W?ww>(>1g16)NI@OyQN|#FGZt{D{u=;ddoVs&ierRE&F!sjLU5$?(J$mpU zAiFK$Q4g5uo=v*LJ-^uHAFmBcENq?!rETNV@!fUW0*$G`a|@a&sKkX()g<}v-)GaC zk6GW!eN`GJTbC8&ZHFvefM+IS$_s3cA>jrVadc%xOsDgSk39aWRf;Un<1Dx0(zWW# zwii5fFTKL};G-?&-pla1p#IQ8iD|E5o>1y!6=%qB<$K}T<5jm(_Uadcf@dFDrW0Rm z0lMh3-+&vJhF^VJvhdFJk;ZjFow{Xf$&sL;_1TWvFazd&^l*51cwk{sW=;-V5^WSx z8vyLn{NS6P$A7kSfY`eMYWq{ab~g|0I{WqSTiJuD3mQ$zRa-`RE=9l!dTL~*oq4M) z9>Fp45E$@s&mtbBQ030)>WQ*{5^U)lNM>Uowt^hwpZe{rz>67CnQrORUYB%^Z)4|D zf>&Aak;Q!jP!7vrdujg;P9H*&BE?hNbucNxtCH{>*j>8S*GCV(1h@~Phpjh`>}mk? zG57gh$5qLLyH~eDVToF)jR&&}6N*RbYmR(yJ{P=-g6BRngVgL>4Sxb36`Z7Sj@->#oFbZPAx>(A>wD+}QeVSj!mV`EV`3 zKX@sh;)7*ea4HYXsXqk4A7U|4ODh{=B15_~Wfn6(@ow9hO18p%<8qwV+C+P!a;3Z@Vs!8_9icnUL8P+I7C+Am`jD&DsVfI;JLbYV0 z(37N%I|qfHouNHiS;uFU zVj^yLaN?!R%D_qX)q4(GvqyGqjO_^o@N-vsR)B%)uC%7NHnAh7sei!bb>Da!RMmsZ z?-wu)Q?LzR49&fNvG?S*(*M%}c&C2(eW~8~kZb0OZe(IPhFZm?@K`Uoyy-Qup^X8X zb1eJFRfN};2^PGr-ZMV&NA4sRgxxVNU%9Ue&>F-)!MTx4!!lsPP=$mQ&7`bXE0PfV zd@Rb$7`-HffOc>b*nkfSW&Kps`>G0?L63%z`!zdPre~$Bfcp%>ZNsajqfycBk00x7 z4!g547RCvU*FP&iw$!c}+S7DCC0V-DC$|cT&doVqyZqkWqtTb!Ch9i6)Q-8UH&FT` zsZ|>kb=tw01M-_wQeT;@oNa#dTXyJ@96R^So6?ZbT9;ktr+>dwJ2?A47tf_P8-14` zmJRK2*TK%WJk6U3BLW|DClF)<^p=3@g%eKw5AKeB+TnQ{k&z%wkp~D8EV{^o>dhCO zmnRq|1#bt_{Ma3TDL0d6hPQ(o-3d_P#geX&kp#H_vhpbwsI|37Vb93~A5x%i=ftbP z_^yKZ{7oSttK#7-!eu~y_wL?sO@&sY-@blq`Lae!Yu_8mau<=JGU&KUYnNXVfHhDk z;-uhK>$?I+4u{90K-m4ir&8;jG$~21Ho!2Cfk*PVV5R9q?L6~! z@Z9*QUV6?B^_NWBIZlxENGF;7EWjDdGmG&o{Po%)7nculcaH!l+G)WkCq;2jOGIAy zM(r|!Es5m`n0-ss#g^S9tB_*R$jbHU>4Go|tRlbU>R;06Ke8H=O4tDRmZ6kCiGDOMpl+8C`JfR8mS=j5#BSgI4yn3`jmIXG0DPnPFjn{+#3 zrv7KXG23BMd= zwu%tTj5u|VpbfQ35x4#2a9nZx?&~8((sg~Cax&riC%DMbw$&~%q<4HOec;Wa_b>{X zeq_438f1E(CTEw9H%_%%z*QMvp%V1`*Z=t)7{x6l4fME5b+ZD;XK}o7rw`VdZOW@N zSi8f`#U;CK7{+_D8aD1=!33i`QYZyRj&e^yk_9B+0X*8e?ICsa6k=h6B8y^(MfP^~ zW2K1F#f%+lnsdeWLMF=C9D~YnI#48&Q1IR5Vd3HZJ0bK-*bSLrZs zfs?B}UuQJ+Ko)`wWG#3_GKh2V;Q8oqHoM7lc{R_yl%)>MB!~u>^!tPBi>sVxO1R^t!pLu9 zF`_g6uQ7bB-(Hutg}Kq z0$Vw}`tc9l+}q{ur4kpc7vt2$;{lkq!a}&x6JCewHv$OS{<8PX^Ho~-aNOG5xZ!x{ zKYsSp6!=-abj@S^J_MA@-=8Qwu@;q+>6w_`ymY(?ZhNx;mph|w(;FXXaLq^EPRb(S z*|h*{K>Y|mAQ{0Y+=}y&>wxr-IC76y_W~i8%8CU9VG}$!^1D%&b9mz|s0e@(4HuFK z3Y{MIz~q@OkrdPdEPGJX#|Cemz&%lah#b{7V^v8iFDJ+v5Lb*A^vN5@!m|h1x%IiP zzyuW1t9_ElE0WObP@xz*cP?xpjCoAw&e+RsSGOKL3{AozEBm##QglilEfO{qJxD#hobwOb%{d9z6>LX z!NB>Gfl}tZlEFT5z|jESL;CO0stYK5bYDP{h)KV;LN$xD8@2npbMq(PpMH2YAsK*u zORaYk+(>mEbnhMF2apP}vj-eaQb;JP6&^H$Xhl?XUnKPPc6GlzayB8M@opPzBF7Qn zWqKccY$YM>9<3)k?tjW(gK0%ZgZy8d=7*)uZ@ zx6GNO{N}p8wRKbV2|$zKar9XE(?b>sVx5FLIVyd$oKz%FWfqK~SRJG!LA^O1dDoiR~gs;UQ~f z$S8ydDaz}U-;N9W-=;%>EsQ^-fwXTyfdDDg`+|cOY)}(V!rcm%#WBhwZ*w*acjHN4~nx z&b>9Jcl7GJ--d=#SVPFLx?H)yf@pEHcW?|1JwmSt7${4EZmA9 zC5RUYMYXaku-*>pyAYqmvZY)~tgN`EDyGw`*h>UCC{AL{U$P>pP-K`OE{!N$Q7D3c zy_TPVSZL!h9N!^A7?`9r7ry`n=5IAIS|a{xGM z`JyfFI1&^jY1`>Cyl`jbV8N$@tG?d6`(eOkJFVO|UsQknV(f2s+MG;(Haf32_iywM zU+&rd>(`5io*w+KHAZmp=>{H|bmrA;^7Nx(mogq%5x)K_lr(@~4h{niA1J>?5AOcH zuh;VI&0hExN!){>NKF2%QIaGnZYvVPlS=i znvBrX@-)%@yr0J2_ZQas4Fv&}c=pTtna=*C+FHkkS&$CjBcg8zvt?9j70@C!jIY05 z++()0H^#fBf1n5^V`}jqX`8|hq_|EM@BYoBYNt^)Xvz|!975s~e>n^72<{JfkSl&+ zK67Bxloyfc4OJUMiQ8M@R@QIeMZurumdw5F*xCddpYzh)#w23}%foZAGB05@a2w@F zhL;{9RSIDnD&2OmoOBXat8j9v@D>HX8%4T}kk4E4ko{OWb$uqXwwxQ9nmTUYj1JB_ z{fr@}ij2B_BnOFX+NCN(C>HVRHn8an;IGWEbcR0_be9q8fo<2LZA7p@v|XS`RvTki zD$S$q4Vy1U90Eki=})Ok*X&$oB-hY zi4a&-B~@gMPQHB`+EDUkax$^@)p%0FtmgIj>(3f5x;M2@{jVmKQJGHp87Dtdr1e zBI!aOL}C$kr$9=#cQGh4tbP$d3|w9IVrgZ@#mEgU=#{~=EDCxL?ShV|SaR0)V^Q+# zcR8W0G7ye5xQK0lvE;*|&ObJ?{xu1SckTIY-#^ zJ>S`p*sy!&&f;Q6$67c$5WxVkIXSy?YIbLL58CIK9d`GPL6EuKq5m!1R?Lxx_vhw5 z>v3zwaj`eHyo&1b@-6|`oqO-me7kqkqw{g!TGj+bwQXY4EAYFQVN*`g#C}E_0_n(`GWat*Qi7-$j4lG;Rc&!90szg5^J+pT9iwld16yfJ1D(wPG*M zY0D}f-5)J=?5%iCtZDy!yG6t`h6xns>rcD{#P08;@m^2pfqjuwF~VbAM92r({sa}< z=inhWO>6@oWnyAt>-^Yw>bFkETE|*g`>qEihfZMW<#0-b^yqyORk<^c?BYVB*@(!2 z4j{+8(KR+6Gd`9$)vy%iW9&mw{cjnj#>P913M-8ZZCf`DfBO2hS342WrIC5I7eX;y zeT>|4k8nzP(K#Y5G9y%2IZ#x510+%NZ|?*xd>tuyRCiwe>1$NfezJguT?rGwbJAwd zp$ZP*?8dYiQy=V7aN^-I>V`BV8>tl1HhiB4IzDQ-NhC9pumlbe)x@~{Z($xqAald% z5^X?ty=@m*+hfidJr`w`KozTQ8rtv^t6M52LDRyNr zy(-dQMm-*Jz{W1mS2y=m*Vt6U$VgJt+-&M}Iw(FG8?SRTpNe4NyBwx%^nqzw(v`yO`LMlmcLO!?juIYcV zD*N;A9!qpdps%y(W|RVS7A{q+rYW2r8}&xjJbelc|IUA#cFFSlN zZ9iXjk);RBG3YZhW}s(;Rh%!r@v46nqNml|xaxBRYwh%Zlu)vKu?Ceo5`xv`3;tkY z_e=VEGrP%`Zt0J=ph46})r&M(YAzSaL<|Uu1(y-5uvH+T32W%l<+KV>b;N;g1Lp%D zimt^Aq%>*O>b#68hPXzCuFSY4u*_N}A?6hn4~$PbCMEd|L9S(X9P+C}Lqkt@2F8zl zuyzitR{cup>t&qcb>VZ(}Cg_cPL%D z^%l-hZ?81>g^jtHILE^sy&T1n!Jge^WPlbDH&hvjx(%`2eVGbqZS)QLa5SC&1{q#} zUf`#PVroKGWP_+p3>x+Qke(b4o;~ouu-(mOf~#FCs^KTylbz+q8mHHT;NhG0cJ{W@_yHSm#9HxK;-{Gc)=A zc6?3o=qF!J4LiTb?c>D&YH@V;_xuPdExOy#rrt+dMoefb1h`V&Gz8kf5^4S3G;Zel z$FDLw7k}ck&N;DW;Ktj%7-QqltR?Cke;K;;zHn)lui_Tt6oY0C5r}O~|7h<==rsQ2 z>eZ{-MtLnQ9FWA+^u~nPwqCK-hq^CNBe0+1`1Y5AEaX%WEqkZUcvNVQT`-2uHxTi6 zz~_oCE8W(Y&7NDT=yG0r8NXh^B3BJA&$<{XyLi*Ldo%ww@q|^CsEVfep0?@;-%xtU zMifbHiTe1J5XjjQQEqV`zs-;yvO;Wx4mAt*FS}t`CO!1Q`u*#tMwf<%l0}D{Xm*LsB z&c-^4ZUsN{*hj^aSZ}nrW|xQkbIoOnSYSvcRN<{|uzK7ikyO#;KZ;)5itG#EbkR3F z31B#f!~T)$k^7E79(|Kt#<(SNSH? zNSDTsO~n_)17e+Qcu<%$1$6Jg!}|L9v4(L79L5t}5}pa9^ztc26;|TA=C40*36{&< zy1%0lakD#`XArE$!_9bG~XJZ4qGJ&`bjI3ll_lCb|gT z#>b5POs5Ql!O;SnXdARU+L(>9wfn4WX$hJeF444?%q zFB`aqxYcep4E9W(C6!BWLGv;79#98*2T%c??akzlC0$mz<>eEaeZ`S??5cRq*9>W+ z){F~Vys{{kH-gENcnh$T-+Pa|Im#OoAbo#nMsU+@k}s z+NI?+E5kUwXP$LE7#e+F1j-hG^+ONOB_$+lI$|Gzq%9#eH8!V!5J-f%#@p1B0pILT zpk$Mu^@yrHPVwVi0o{Rd9?%So{nSMY#HWP<=FNm^(B7Du{@l?-(BL~5=b*TzOXAtAD_n!sHVNCp zPgrzwBR3N_lPR3`h&z`Am$@Fsa%cgCHm2I4jcttw4`9sLOU~@Q5|MQcV`%$N{;d>A zct*FuOH^zxTHk}Ar!6UC?_+99zJQRW@j>#$%!DI!Lk2<4*jP$e`KlvAMyUwH5Hl3v zt0hY-S^jzcdC#8r?)(rl_8}O~~Y!vTk?0I~h|6cuPhlh)~|l zwOrFZC=69D-xj`#gYmF-ZZI;2?p+Gk*^bni+>-fI+Z}qN+_@-Bx6vL04kl7T?%PBb zc#tXkVK%+7&wWSyL^NEn%1wq=HB#7ib1Q#Io!vtJF*@AEifThEE3P7r+$^h(+-(^x z5%Cz+hD00*FpW9O2xfxa@>oBZ<%Y)V9A^>H?jAY``jOP7c{nql$Ziktq6}XWE!pcp zUWBK#VXUEKc4tEnaL)aO{bK0-oie3?%(|naV}r)aCA^$2!EfOCg3JKC3U&`ymVFJ_ zkyB$Iq9!~dy8q0nXfxbc}$FqUq7Ty zW-m0S{sLSoRQhH8slzoh1X|mCCDhKM@w9z8;8KLLi&t!`HexPkc;T^o#2hxMg3JOj zGR6>_68IqboFroNd+TDlBJx0BOT{ziW!x$|q3lp}8IMjVSRrOvKfD^T-QC*G-N?+G zP-?fW0&g9h1BH&9e8RfhAS0R(8V2ZO1lMeYYLC^?2o$(-L4z#uuj(6{n(cc^Y-kvI z`k>*#gEPsR_X?MY zM--XZ)yp0zim$C}dhj55b_`^dFCpJ@qzNLy_=chJDnoXUTXL!JqM&aX76yrt-x5WR z+_YMnC%lMcupa-5J54}Lv|?G)Hh)9tkrCYKO#um!187eru+T*tE)$M z=Dc3#bQuoW4SWZnTbgZ(fsZ`bu(lH}++(AKv_?IWW-r?pj#V_?-@pYcaiQ#sVw zRID|QkN1>KItZ7ajg2kO%>f;KV?0)kQlkFDMVN_CN^{nr5R%T{q@YSHH)g?%FK&m4nD|iY9E6=;=LSGK( zessLW&_{`>NbNU2BM*L^-I#c|uzE&S_OTo7fZnYWY&4C6PKZ5%Y$M#iMc2F4oDcAk zSQln7+3xBB(7~!+eeOY9m<_;+eDit^I~q3%ZI9e#>M8mkr|Cs2eQUja1AYB{HC4VV zKwd0Z$NZaK=H;_;)-@dZU;3Flq4Vz%bn5P`=j9 z$$M_h_6{SvkG39V=`bF9Yjd+EB&6*1YK{a41@V8(4*%i*eCK&x_CK&Y+9Y#bF+c@`dm$={xbKRgZ>WIER zkgAZJ5z=J<++1B_k{Y2+#N*AUsD%B-#Aj48X2{5_Zc=HZGCy!E<2rUsRdI7-@=e7B zuK`4vN`4qknCgF!faBt!Yi9#}}ja6CY(y)dW?NO&AlB2+CW>dI-op`*6 zH&C5Qz>WP*FHw8UXyfc^GqO{b_2hMTHvy(hiA zC#|JLg>m9cO&I0h-)Va79d~F$Z^NO7Q0lR9R2XB^U&hwgnp$2Ud^6YJO1duGyZkZ3Nk$0 zMj)aehv80UszyI#0Fl`@4v?y%0`BG4{Dj8zvO=Lufot<{SC>!jOtGZf)d2FlO3{wac< zHVE`|4=G6MlC)LO-=M6BTkrKp>n93M(9HZ z5NH2aQ3KWdw}0Cr$-(4JC(I)||Zx&xSJCQ}}6q3lI|=aEZlEysWKPQL6B zVj4g+;}z-umU%(GcT6`puVVEdLAZcxX8&WJ)>&jVp^!Y zp|Cd(v+E|$#bSdc4orH%OO?$b;~;>r+}oCOgmXgOybkE zkJDND{!sS?fe_?5g|3(nn7j&TM7wnMb!;oAS;$LpmrqH=;24h;w$>Pfi87#T$KdKXyOK-0 z*X+j^bW;)3-YPJ8%TQ1Wnew@R!DbxYeZp>* zq+M(v%-rEltbuZ?F|x#PilG)5x+Ps-O^V~ru~l%9fLI6S4Zs%TeL^#5pJ8ZdsH>~r zb^o#s@CsnF(@!aGfU1p~?rgjqLva_;-Ca+0$VbaK%5cN?HP5|gPHH1Y*cfPd>E|lY zw8$AD~*SR zKnfxZpLFV*G{>sdlfnEu4)zY4P1*rGD&IZkstX=>VsfF(rwp)GXD4{#ClI#u_4fW) z(50WvSJTLxGlD@xS1q%|PARr2%CsY%NyUv~M!6<#iKT8_d4)g1OY?CaZ$^h_hv8L$ z3540$F!>!ecLw=+dwcsme5|fEw!SWxy zr|MCQtu#0%M4!2R6lG*vy}MWhMW2B%seUA8D(qpKk2Tph$j4h@N7SYS{@?p}HYsk}VT*BXYWEuo!ocHW_~;y3qtW4VQi6u+QPi{z`zC%ffQdp2HA-UYlTk z+>&|LMmsv}p)8~_EZPgO5Ql|DZ3OUWscTn!YRD+xz#=8K?qUUu&g!a?=+yljR(%C_#=Ou#w*IHo*fa%5|MERF}88ZsB zAXv?9f1I;8NB&4ne$aL&X26oyt#;KSW*N^tc6a8Y&qAZ7VDM}3N~hpUu)uJ2(onEe zHV{ye?w@z?{OU+Ef5>sIJ$OZvUlcr5zjPFgSE*Y}YSYp7jG~IZ3I1iDYd-$hW+obY z4ue`h0bA?iUo~bd z)DMOkH!$q?MP&fY{sj{2-HoEI3>c=@cm$yt4^@TOgySTQEfaCv&E_eKJCkFjAb3zn znVl%jHNk4%TRfxuhZR+SOAS!PVsqQF<65GuLcgE<#W1r0YSHa@5>8N62V)Mlz03lX zzgDjqG&NJR^snIiz-oF8|KFM={t6^F`aDk2ri!xGMrxM1f>A?3nqYcq$@i{=3Cx~g zSlU5h%vx_F1j-XZKEc_Vx4YZod(4bngAU z>F%32+c!Vn{EKawvDH;^KISa1{+MW6dV%1ZaKSG1!eh0)d(*D$cw920{*C!-=OtMN)h#*GGdRY`iK+kb5SD4<-%h!)_I4R&_!sYF}$kG zzO}t+nv;``{WZ_a$|{PA^c6~cnVtsB+(w3YPO&d__(J8_QCfQTzpS$`d+*d6?+^zOb-`lv#>hD86K447r(EUw`iB#?4`Go z=~}}d?u030UO|GD=AAos%=U}#Il-YC{0xx{7!-G0{4^Q#<2}DRmVITT$cT={1NDOE zevZ6l@9TbkFZGTcdwKQd&9%?nU>Tm_Fpp>E?JgwnA| z9*t#?by_I2FZW)WPWh1LbHm3+wr7U7RM6-;qvKUX)X*R?L)K{>M^w*{oan+Hx$ z^GDDlr$vlNDrC|twZGN;+qERPPB!=V-MM{dv!tZt^`@BE2Ia;W%Tqlvw%f>*{QB*%i)YYj_8Xow6t znPrcSKNzaKZBLHG>*@M52fyj<<_623a?JJTAQJFjnsu?2RT6!#mS?h-vQPsZN-Gru zD{JQZi+_iaC=;d=7U44I<~Rmz?sBKoV8c#QdSqm$*A9EbBS#F5G3G*kvm`kfjhK;| zh@_jFtja^8&h1g;?N4v?xp4#T3357CevCPQ1Qylh{eRzS1$kH}+?*2sdHq0~!MJ7>e zJw*scM#Hi8=eii|cJHOy7|nn$AXmWL*uiGKwP{n$!U)I=*2$bG)aDjA^79<|y#uS` z0)C`mu1~U>TXK<3qLPfgLs(b~V=+tHZvRcMpW5$()cKv;mmPGEytpy`OD|tPH;MN@ zb5uG$4htWPxF(}5+ZgLJv`nZ>JG+^_%&N7uwY}C)eoRhIj*n}Uj?VDv1Ns^1B|6L^ z9nu&BwidZ6J|%Nuz;k%I$2ake!a1 z^q(By&kY1E{}a48GwEy2j8ns533%7x46KUcEg3wc{VdPw!^oJ6ulFA_-CHU>)g={WWx?ND*GuYMS(+X(t?;*t zsqYB5pW4!`&TH&0XzW&x>yXDL$=#D9lUn~~$tkj_Fmu?5ik6?=V2Ti&jw4=nQ?nGL z?1cOR?$_}(7?buPhzJ4il}utqiJ*PygAduTM&1ma(t`?ZMLGGtTpgS-^n)r#g`nXv zpX!WSzB?W9ZkcxmsH(DnjV}i>QWYSJ>WS zi5{7ik7%F%!}q!UE`^;tclA{Utu4QbgAH9t{TuT{Y4L}2c05ygfM;v(SHvqaNVZlf zIqa^0>Z(uuJv}`GHN`c-@Bo9oyu4gp{hyVz8yDFs@k(+_Qvb(LDn=)29oR%e#Aq)h zrINJ7LI*c>GR}jX*QRD=T#iQ_LJTWdSQYXoEXo_SRyHmH``{)WgGHef(Gr99e@nS0 zvk%rz4-ZEg@}Un`kN@esbDi-wtchY3TqN1dINlQe0#(3_LRbSsy zhg(1Vo?)8a>>5|xVZ-F@H({l={2kAVe=2KiQhQdyi^I+h!S)*sO((O=WEA5%ar>~f z5gzE_O|n?BG!`XOsD@7hSK)GsCXdi8DKYGI_jhFJK_8cb@3*ne>iNZmEdPnc#awG^ z%#nv35eD*ph0l|_3OpOXdg}PYSK-e#8>|Gce$QHcuPJCPXw25aC*djJS>qFW8`V14 zq!74M32x9!J^iDwANSZXy~?ZO%OzpkBOmSFeM0%qk`2ViICLEqBU^{9ty2pj7A*xl zE4hA2(3)3ey27le@bBq~SqkXyKs0zDC^N3*X`(Ty6^SF+h$n%I`*WQLAtT}NS*DXv zvK}k@`v=4tC=a%2v*DPKi!dV`*t=w@1*@G~d-= z=56rQ)~rB5VP&vE`avlIeyeH@i;&N1A*qN(u?pyC+O1z-fwUb&E&&1FH*ek?|GXR* zw*Bqyx5pJM2o|^g(oRO7K!~wu;-8Hdb#*J!_MPn>s0kQ(&u?<`cV+G$=w=jmjLz6n z8PW_>hCBwdorVxHE;-?h2hCon;?dzhFMdEHllZIGZ<(P&sb8)mTYuT1TdWKWn!q|o znvm6DKb6JewiFXd##UGVIx{l(YR)Toq2i9-Y(v@og>R3qhs)A!?m%1YSHil(K@UJobBZ#z*SLl&apDB z+=%w4_8R8qsk=WU<>chRqS#%%NA%#T{|L}4u%as`p$RT^3?wcdvfC9!RW-pQ@7G(; z3)59MWMk?*47_-a&HrOC56WYS1f|3l65^D6Ma2wfOh8*8QYi9~t;46%QnkcD))ERv z+9>y`Cis(IR|}f$BqU~g>TJTn)t`j|3~y>z>2PB)#|i&x%vuq57XsTgaf@wvg#?PKu!72=8G}) z&i?-X9o_EV1S2)8O@{o}1Hr>(L7)T|wrrqKkL)JDNb@;aZqsY`qagb3 z-H+ZrdfQoeR`A_a^K8x4r+R1IjV>0c5JNmPM*aOO(iRH!7C(ortgQI!>dq`xHg>z{ zFg-gidUkYsbjWLHT*APmNipCC5hUb_DcKE;7CT{Tz)ZN{^OG6fHPPu*Hu#&>kq+n69X04~D z%ErU?p&SaM1X~o0MMR{e5X4CaOD2i@neUe3JU3P)dE9Z+hj#DlGvB@~znaThovr`3 z^GR05zK=T0oOZc2|DNBt$_QKg)rN*D%%D99X*f#oKn8aKst4d}EoDZE zX;5_*K7dQTi7f18X$$}IxK%T1;uRf|K75ZYXT;0t`%(e zSXE3}$_b#a2oEb;{W;I%I&4VORGI~#4;z>Y_|;3iZoaPa!p zAEy|LsftJyQz|aN|G-ZHU>6hwe(2mBwA>ogxp{NMhtg$o_bD2VQb=lq{7r)xJ^N|l@1nBseJT4;$)CXA08tMKDt@r6=R*+2yDFeNcBTeRp~qeg;hKcs zo+806Gw!(gh{`){n?%DOit)$Ew~&qQq9vpVGOnltn1%?m+U-hmhh&|I>M`Q;X7uf!{B17oPtJ{8lBD%x{kIFZ1EZ21UihRNhj@!~`Ir z2@f6Jqeqv17Os7*5so5Sd>aJRU7DI3MyVWck7vq4!~s@ZJQQ_6$^yG57tLXX%5+%J zj~g*HS{O?4>|7}Z^O6RZN=8Ib>}!J=cqRnSwW`DdQFau8{9+^F|IE|r1V-j_mF;m6 z=O7NnfGz4xeXyn94DhSfAr~2V$}Y)x>6Ey27_xZV(;W_eOL>p(`z>cjD)>#m^K$V2 z-mN9UxG;CQFni&besUHZtO}nOsI{?}tXQ0^T?~|P0@UdW2=exx9+V9FwpZ}g+F-sy zp{J*&4N;;UQTkddihA!!{uJL+rzItWx+O-7FJ9;LTPi~tk8YJ`@wrxzL^CiQm13{$a`L45_y5NIy&PY>P83Zvf|vduvPZISo+96EFkZA4YfX*@ z!CHhTR!`NziCI1RkX1d~nl!LDs${JY1cL0`(VPFy;zE{XndIvh|2RK-;gi%0wz7&3 zSp_s^S2yFw_=H1u50keu>wdtuFoosTwCcs^JAuxE>7WKQq2OKll@L}q_%9$VJd%4K z${nwZUx%TQC?R9@#6#*hDF!~IV2XnxvWzBVok5#aF<(*MTSI=zy78vKL42rXd9ds7 zxl&lgh-k1_1jX$lkJ;DAM|Kx#fTIAb&4w?6qDsd@=qXK5RI0~PV@BMsFV~72`VKM8 z^=5x6H3zJGf4%)McY)IiuYB$!N3xyy?W_jZ&ixcxDnWY-LiBw0sk^7Ir>8Ho(e>@y z-3rT}H`XlXX$Gx+QGjwR6RIgnq${2#e5kg2F$*}A1154Qhp?7NS*~&e(M(1Ixl7qZ zAS?CCs)&^1@kA#c3vp5+)l!cpX-U;`wJNnC_7BTV-TRwKZNw%dX9F%?90h)`!5BMy z9amdJi&sI%U-MSMyCxrela#*OFlg%J9ml|-1g>Q8SKR^qm%jFva}JF5m6pi24e<376p1h#Ix7E3dP1hxD*}Lu+3JST1&)49S}pD644@} z*?A$ADj`4rGC_o4MF~SRK9Pd1W#F2Ks#}Fm;uXG3lnp{sCiYh99NkJxe5x{tB3NBj zY2gC?H(dQFso=W^w#$IO_VvF0(PK|&lO-Sh75o&ITl`b<=G96#6s`-{VG#n&)K0w= zh@<%%K0O{c0=%bR0dHBevij9<>7BX3&(C2Ryzqkw3ICxYL`39Ib`RJx07j8&1QL^k zBdC5&e6tgNwNI6_E&Pn!2|&ZYs}KkWaO~8descXp+b=uS_UX!?Em6-C))6eQIzqMr zWbi=OsSeIz0Cz-*NR_99eM=ycNJwZXoB`5iaov07+eX9HSM8C72Fu^hH?Mszhiv(C zzd6oW6~~Xh>9_)&UZ}1=>vws)-xIGsKyLvEJq)T@bh2I$(mo6d#o#g>qqIv1B3Ot0m&|Ds?p6@H9MH zhFvT>qK#o6w!-tKZp)f)gyTA6Gy%^Zf;+gwPZqf3gHFVRgYYciP4p6s;QJ#^La=2c zE@y!ykVwY#4?~ z^Imj0GUdG@Pd%U>dg)Z1Cx|1`pe1)74PYA zhFNXkJ7R1ic#50)=vpp@TFYRe+3euAzdhQE2Nn8tb)?$8qAnWv99A!ZJrwKFu#KQ9 zf}1b`K4ow-O->Ykh}rKs5EC!LepSP%l0w7fZ%WO9f{nZ1zSW++JlC#eSqSpb=Qs`* z=#y2(Rz|k{A$CGYnY4}u#1;4r;IhWY$2rFgj`4pM1}=VZlVvXe5k}G^6$`L{7$HDAB=w9KQ05#KJ$VQCR+ZAJfbNmqZXT z6x*ac7>gyqyZH6e2(h6Q7e#i$f1U%Y0-pTdfm?DYb_pw<#8~&tgcw4SOXa@KNpB9E zEI+9bIGhk!SpD@uuDC-OUI}k0 zz{_iLn$Mq~Z_+BbZEha)r89WugX2=`+n}$ontqW^4BQW? zsey_BcSBOq80;sLJ_0G+^v?5=RPA;Wf*MRgU5fL^*4Fqx{wDWrk|I%F9CHsUS$Ph!(O0RlfAHy|pcAy# zU&z%4t7r_6Rtn{?97jeQp^K#41` zjs8pZX|Gt_kz;}F{tBEztK;;!F}iz-6^1^7hSbG=4}{>v_FR*IU;sa;Bh+|ot&l&7O+JWuZ%iWXqt@#0@e#WG?J9%U zrEC9H#7^vCu5O&r*tZdpQA(h_I1Yq#**XZ#EWkQYkHpC=D^Vbc}=U`S9ee+`~;ofM2LG;bAex{yo6 zquE!y;9^9;)DLea%PFg3a@34;P`S#Ag%76H*ese7ej({W21r7{Iv7!bRqB@^-|V@? zy&Vfd5JJE(g|T26(G2zcFYm~88ugjvyGTPsuOfxbF1b<$AdE)uVzjnK@`qv!1Ba#x z*9r@R<{o?L``)}_)BX$sFhD2Z*C&rp&dXo2rQc?GLqcd!XN$!)W%yE|iK(go8=a}| za0aaI>Ytvw&>lAFw~!_1auqC^3x>Z;9zMZD^@1K8c9TB?ouG+ z#-i%CL6C`Az{dBWK9OI!m#5;1Ql)LKD4T?nQEwu85C>0#){`9KK)t9@*vsMow#uSpgg*K>Lk@!;F2O)Jk@}#w z{--gG1jCK7wG5Jw`D#C!gEsjSo&v=U!^Wy$OhOYal(!;I&c;*}nhjnSk`9-paVmrT zY^`S{B)j_=dJDah3Vz?Rs+WJZD;O;Fet&nVFgwDa4ZLGuKF0xkJjXi4YA*4)jCxcm zi;SbjqqjVjk;ap7$|8v^jP%A);7Wunx=pb~RsTx$Lg0K+F#pfyRVZR9U-)0?s1(DZ zHU5yg5qq7JlE`rqQkeE$6gZu`=;6zXq1b&0ShGYeOvCmMKRZlK5ii%NKSX6{6l0X^ zhT+nJ5$p&z&WlaLy70HjqxX=^CY5dwO(Z z>a9#U3k%hgRl#%C!86|l^O2Ljo&Md<1G-8S|Y@w z4n`6pDV@sfbt2^ope|C!>#&zG4Wdp&ukx1Go4Oyayu7E4J@r(Mh5`FTAZlcXz`tUl z^1!pg4a$NTy)*=m-ZC8eCIWH@2QbV{$MhC{ToZf|@D;3UmRn{t`7YN3#3WvO%gtq5 z7xsf2dC92iDCVHyjezC6OLWThHj)cJ(*Vr3am;vq)BJo#z-8w-M}CUqQcn%PS7Ei& z@a95Af8Xe)a5jQLi>rofiK${*;IkoeVuKCA#-qV93ke1w3lSv|nuRd&44E86E-~V{ yRQxq~Q`?_QL9|;SI-)}`>UX#OGX7y%u+A&-WoS@@qVqcVp`AEQc|txH`u_m=PZOO0 literal 0 HcmV?d00001 diff --git a/integrations/lingbot_va/DESIGN.md b/integrations/lingbot_va/DESIGN.md new file mode 100644 index 000000000..4adba40f2 --- /dev/null +++ b/integrations/lingbot_va/DESIGN.md @@ -0,0 +1,96 @@ + + +# LingBot-VA V2 design record + +## Inputs and references + +- Target baseline: FlashDreams `8fd97fa38f04bc32c288760fa0fbf5da52464cea`. +- Draft integration: PR #312, + `f98cae4a18ddf6c189a6cfa2099265d6d570e337`. +- V2 reference application: `integrations_v2/red_screen` plus the finite + `color_fade` loop. +- Upstream inference reference: + `robbyant/lingbot-va@7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb`. +- Checkpoint snapshot: + `robbyant/lingbot-va-posttrain-robotwin@8c9dea8abbc5c91cc9e18bc3264b8915083bbe70`. + +## ADR-1: session-owned destructive engine + +Accepted: each session owns one `LingbotVAEngine`; the application owns only an +immutable config and an engine factory. + +The VAE decode cannot fit alongside the full DiT/text/cache footprint on the +supported capacity path. Generation therefore releases KV, DiT, tokenizer, +text encoder, and streaming-encoder caches before moving the VAE to the decode +device. That transition is destructive. Application-owned reusable model state +would promise reuse that the implementation cannot honor. + +Reset closes the current engine and clears the loop's finished flag. The next +step constructs a fresh engine lazily. Close is idempotent. A failed run closes +partial state while preserving the original inference exception even when +cleanup also fails. + +## ADR-2: generic typed action artifacts + +Accepted: extend the generic V2 result/session/sink contracts with named tensor +artifacts rather than hiding actions in LingBot-specific files or metadata. + +`SessionDesc.tensor_artifact_schemas` declares `actions[step, channel]`. +`StepResult.tensor_artifacts` carries the tensor. The generic +`TensorArtifactOutputSink` concatenates declared chunks and atomically writes +`actions.npy`. LingBot code never imports that sink and never chooses an output +path. + +## ADR-3: one honest model step with deferred decode + +Accepted: the first V2 version generates N dual-stream chunks, releases +denoising state, decodes accumulated video frame-by-frame, and returns one +`StepResult`. + +This keeps the UI thread independent of model execution without claiming +per-chunk presentation. A streaming cadence can be added only after a measured +decode path fits without invalidating cache/model ownership. + +## State machine + +`NEW -> RUNNING -> FINISHED -> CLOSED` is the successful engine path. +Any exception from `RUNNING` triggers cleanup and transitions to `CLOSED`. +Calling `run` outside `NEW` is an error. Session reset replaces the closed or +finished engine with a new `NEW` instance on the next model step. + +## Memory ownership + +| Phase | GPU/active | CPU/host | Released at boundary | +| --- | --- | --- | --- | +| load | DiT; optionally VAE/T5 | tokenizer; offloaded components | partial state on failure | +| encode | T5 then VAE as needed | three input PNGs | prompt/observation temporaries | +| denoise | DiT, CFG caches, latent/action state | accumulated completed chunks | per-step temporaries | +| teardown | VAE only after transfer | DiT/T5/tokenizer references | all KV and denoising state | +| decode | VAE plus one decoded frame | accumulated output frames/actions | VAE cache and each GPU frame | +| finished | none | returned video/actions/metrics | all model components | + +## Fixed contracts + +- Robotwin layout: high camera full resolution above two half-resolution wrists. +- Video: TCHW, 256x320 high-camera crop, 10 FPS, float `[-1, 1]`. The VAE + decodes `2N` latent frames to `8N - 3` pixel frames. +- Actions: 32 steps per chunk, 16 channels in order + `0..6, 28, 7..13, 29`. +- Default CFG: video scale 5, action scale 1. Conditional and unconditional + branches own distinct video KV and both branches advance whenever a CFG cache + exists. Every action denoise pass attends to committed prior chunks plus the + matching branch's current video KV before its own fresh action KV. +- Cache attention window: 72, matching pinned upstream Robotwin config. +- Checkpoints: local root or revision-aware Hugging Face snapshot with explicit + component subfolders. + +## Deliberate exclusions + +- no legacy V1 runner or `flashdreams.runner_configs` entry point; +- no application-owned files, MP4 encoder, threads, or model components; +- no multi-GPU/FSDP claim; +- no speedup claim without matched-output evidence; +- no root CUDA/Torch policy change. diff --git a/integrations/lingbot_va/GPU_EVIDENCE.md b/integrations/lingbot_va/GPU_EVIDENCE.md new file mode 100644 index 000000000..05931f096 --- /dev/null +++ b/integrations/lingbot_va/GPU_EVIDENCE.md @@ -0,0 +1,99 @@ + + +# LingBot-VA GPU and parity evidence + +This record was produced on 2026-08-25 from FlashDreams baseline +`8fd97fa38f04bc32c288760fa0fbf5da52464cea` and this integration worktree. It +is validation evidence, not a general performance claim. + +## Fixed inputs + +- GPU: NVIDIA RTX PRO 6000 Blackwell Workstation Edition, 97,887 MiB. +- Driver: 595.84. +- PyTorch/CUDA: 2.12.1+cu130 / CUDA 13.0. +- Upstream source: `robbyant/lingbot-va` at + `7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb`. +- Checkpoint: `robbyant/lingbot-va-posttrain-robotwin` at + `8c9dea8abbc5c91cc9e18bc3264b8915083bbe70`. +- Input PNGs: the upstream Robotwin example files and hashes recorded in + `assets/example_data/lingbot-va/robotwin/README.md`. +- Precision/device: BF16, one CUDA device, seed 42. + +The real checkpoint contained 841 transformer entries. Two obsolete +`patch_embedding.*` entries are intentionally dropped; all 839 remaining keys +mapped bijectively to the 839 native network entries. Strict loading produced a +5,088,872,670-parameter transformer. + +## Upstream flow parity + +`tools/compare_upstream.py` loads the pinned upstream and native transformers +sequentially, then runs the same first-chunk video followed by action tensors +through the same cache lifecycle. The explicit acceptance gates are maximum +absolute error <= 0.07 and mean absolute error <= 0.012 for both streams. + +| Native mode | Stream | Maximum absolute error | Mean absolute error | RMS error | +| --- | --- | ---: | ---: | ---: | +| eager | video | 0.04296875 | 0.00751040 | 0.00951632 | +| eager | action | 0.06250000 | 0.00875314 | 0.01267146 | +| compiled | video | 0.05468750 | 0.00970979 | 0.01229867 | +| compiled | action | 0.06250000 | 0.01116651 | 0.01560142 | + +The comparison caught and then regression-tested a defect where the native +action block loop received current-video KV but omitted it from attention. The +pre-fix action maximum/mean errors were 1.015625/0.203186. The table contains +the post-fix measurements. + +Reproduce the compiled bound check from the repository root: + +```bash +PYTHONPATH=flashdreams:integrations/lingbot_va \ +python integrations/lingbot_va/tools/compare_upstream.py \ + --checkpoint-root /path/to/resolved/snapshot \ + --upstream-root /path/to/robbyant-lingbot-va-7c6ffa9 \ + --compile-native \ + --maximum-video-error 0.07 --mean-video-error 0.012 \ + --maximum-action-error 0.07 --mean-action-error 0.012 +``` + +## Real multi-chunk V2 run + +Matched GPU-resident and offloaded runs used two chunks, default CFG (video 5, +action 1), 25 video steps, 50 action steps, and compilation disabled to isolate +model correctness. Both produced: + +- video `[13, 3, 256, 320]`, finite BF16; +- actions `[64, 16]`, finite float32; +- different stable chunk means: 0.179792 and 0.333868; +- a valid 13-frame, 320x256, 10 FPS H.264 MP4; +- byte-identical resident/offload video and action artifacts. + +| Artifact | SHA-256 | +| --- | --- | +| `demo.mp4` | `df0c193137a673f4f8d6b2372b4bf7afc01a9937c4280b7fa5a51912b5e93c1a` | +| `actions.npy` | `463b307b667c1ca13a47bbbc5a17f68604621dfe3c3a10fc5860077216928d95` | + +Fresh-process engine measurements were: + +| Mode | Prompt | Observation | Denoise | Decode | Total | Peak allocation | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| resident | 0.240 s | 0.221 s | 4.735 s | 0.260 s | 33.220 s | 43,329,760,768 B (40.35 GiB) | +| offload | 5.055 s | 0.776 s | 4.744 s | 0.429 s | 34.009 s | 39,804,415,488 B (37.07 GiB) | + +After `close`, only the process CUDA allocator context remained (about 0.031 +GiB); model components and caches were released. Compilation was separately +exercised through a complete one-chunk engine run and returned finite +`[5, 3, 256, 320]` video and `[32, 16]` actions. Cold compilation is excluded +from the table and no speedup claim is made. + +## Remaining experimental limits + +- One GPU only; no FSDP or context-parallel claim. +- The engine returns one complete rollout step after destructive teardown and + decode; it does not promise per-chunk interactive presentation. +- The VAE integration intentionally depends on Diffusers 0.38 private + streaming state and must be retested before the dependency window is widened. +- The official checkpoint currently carries two obsolete patch-embedding keys; + both upstream and native loaders ignore/drop the same entries. diff --git a/integrations/lingbot_va/README.md b/integrations/lingbot_va/README.md index d99e57805..f6bbdda03 100644 --- a/integrations/lingbot_va/README.md +++ b/integrations/lingbot_va/README.md @@ -2,89 +2,130 @@ SPDX-FileCopyrightText: Copyright (c) 2026 Hongyu Zhou SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. SPDX-License-Identifier: Apache-2.0 - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. --> -# flashdreams-lingbot-va - -LingBot-VA Image-to-Action-Video (I2AV) integration, packaged as -a [`flashdreams`](../..) plugin. +# LingBot-VA Robotwin I2AV -This plugin adapts [LingBot-VA](https://github.com/robbyant/lingbot-va) into the -standard flashdreams runner/pipeline interface, achieving **2.3× speedup** over -the original repository implementation. Note that the upstream repo wraps models -with FSDP; compared to the original implementation with FSDP removed, the -speedup is **1.48×**. +This workspace package implements the LingBot-VA dual video/action model. The +V2 application adapter lives in `integrations_v2/lingbot_va`; model code here +has no CLI, MP4, metrics-file, or action-file ownership. +The port is based on FlashDreams PR #312, with its CFG cache ownership, +checkpoint loading, configuration propagation, lifecycle, and output contracts +reworked for the V2 API. The original unverified 2.3x/1.48x performance claims +have been removed; only measurements produced by the checked-in implementation +and matched parity harness are reported. ## Install -This plugin is a workspace member in the repo-root `pyproject.toml`, which -is included automatically when you set up the flashdreams environment: +From the repository root: ```bash -uv sync +uv sync --project integrations_v2/lingbot_va ``` -No separate install step is needed. +The tested model dependency window is Diffusers 0.38.x and Transformers 5.x. +The engine uses private Wan VAE streaming fields, so widening the Diffusers +range requires a real-model retest. -## Run +## Run through V2 + +The reproducible default uses the official checkpoint revision +`8c9dea8abbc5c91cc9e18bc3264b8915083bbe70`: ```bash -uv run flashdreams-run lingbot-va-robotwin-i2av \ +uv run --project integrations_v2/lingbot_va flashdreams-run-v2 \ + lingbot-va-robotwin-i2av \ + --mode mp4 \ + --output-path outputs/lingbot_va/demo.mp4 \ + --stats-path outputs/lingbot_va/metrics.json \ + --tensor-artifact-dir outputs/lingbot_va \ + -- \ + --checkpoint-root robbyant/lingbot-va-posttrain-robotwin \ + --checkpoint-revision 8c9dea8abbc5c91cc9e18bc3264b8915083bbe70 \ --input-image-dir assets/example_data/lingbot-va/robotwin \ - --output-dir outputs/lingbot_va/robotwin_i2av \ - --checkpoint-root /path/to/lingbot-va-posttrain-robotwin \ - --num-chunks 10 \ - --benchmark True + --num-chunks 10 ``` -### CLI arguments - -| flag | type | default | description | -| --- | --- | --- | --- | -| `--checkpoint-root` | str | `robbyant/lingbot-va-posttrain-robotwin` | Local path or HuggingFace repo ID for model weights. Must contain `transformer/`, `vae/`, `text_encoder/`, `tokenizer/` subdirs. | -| `--input-image-dir` | path | `assets/example_data/lingbot-va/robotwin` | Directory containing three observation camera PNGs (see below). | -| `--output-dir` | path | `outputs/lingbot_va/robotwin_i2av` | Where to write `demo.mp4`, `actions.npy`, `latents.pt`, and timing JSON. | -| `--prompt` | str | `"Grab the medium-sized white mug, rotate it, place it on the table, and hook it onto the smooth dark gray rack."` | Text prompt describing the manipulation task. Can also be a path to a `.txt` file. | -| `--num-chunks` | int | `10` | Number of autoregressive chunks to generate. Each chunk produces `frame_chunk_size` (2) video frames and `action_per_frame × frame_chunk_size` (32) action steps. | -| `--seed` | int | `42` | Random seed for diffusion sampling. | -| `--benchmark` | bool | `False` | Print per-chunk and total pipeline timing, and save `timing_flashdreams.json`. | -| `--compile-network` | bool | `True` | Apply `torch.compile` to the DiT for faster inference. Set `False` for debugging. | -| `--enable-offload` | bool | `False` | Offload VAE/text-encoder to CPU after use to reduce VRAM (slower). | -| `--save-video` | bool | `True` | Decode latents and save `demo.mp4`. | -| `--save-actions` | bool | `True` | Save predicted actions to `actions.npy`. | -| `--num-inference-steps` | int | `25` | Diffusion steps for video denoising. | -| `--action-num-inference-steps` | int | `50` | Diffusion steps for action denoising. | -| `--guidance-scale` | float | `5.0` | Classifier-free guidance scale for video. | -| `--action-guidance-scale` | float | `1.0` | Classifier-free guidance scale for actions. | -| `--snr-shift` | float | `5.0` | Flow-match sigma shift for video scheduler. | -| `--action-snr-shift` | float | `1.0` | Flow-match sigma shift for action scheduler. | - -### Input images - -The runner expects these files under `--input-image-dir`: +Use `--no-compile` for correctness debugging and `--enable-offload` when GPU +memory is constrained. `flashdreams-run-v2 lingbot-va-robotwin-i2av -- --help` +lists every effective model override. + +### Checkpoint modes + +`--checkpoint-root` accepts either: + +- a local snapshot root containing `transformer/`, `vae/`, `text_encoder/`, + and `tokenizer/`; or +- a Hugging Face repository ID, optionally pinned with + `--checkpoint-revision`. + +Existing paths are always treated as local. Prefix a not-yet-created relative +local path with `./` so it fails as a local path rather than being interpreted +as a repository ID. All `from_pretrained` calls use a resolved root plus an +explicit subfolder and `local_files_only=True`. + +### Three-camera inputs + +The input directory must contain: - `observation.images.cam_high.png` - `observation.images.cam_left_wrist.png` - `observation.images.cam_right_wrist.png` +The high camera is encoded at 256x320. Each wrist camera is encoded at 128x160, +and their latents form the upper bar of the upstream Robotwin T layout. The +repository defaults are the official upstream example images; their source and +hashes are recorded beside the assets. + ### Outputs -| file | description | -| --- | --- | -| `demo.mp4` | Decoded video (all chunks concatenated). | -| `actions.npy` | Predicted actions array, shape `(num_chunks × action_per_frame × frame_chunk_size, action_dim)`. | -| `latents.pt` | Raw latent tensors before VAE decode. | -| `timing_flashdreams.json` | Per-chunk and total timing (only when `--benchmark True`). | +One V2 model step returns the complete rollout: + +- video: float tensor `[time, 3, 256, 320]`, range `[-1, 1]`, 10 FPS; +- `actions` artifact: float tensor `[step, channel]`; +- timing and peak-allocation metrics. + +Each chunk produces 2 latent frames and 32 action steps. Wan's temporal decoder +turns `2N` accumulated latent frames into `8N - 3` pixel frames. The decoded +T-layout is cropped to its 256x320 high-camera view for the V2 video channel; +the action artifact has 16 selected Robotwin channels, ordered by channel IDs +`0..6, 28, 7..13, 29`. MP4, JSON, and NumPy serialization belong to generic V2 +runtime sinks; the engine itself performs no output I/O. + +## Lifecycle and limitations + +Video decoding needs the DiT, text encoder, and KV state released first. A +session therefore owns a destructive, one-run engine. Reset closes that engine +and lazily creates a new one. The initial implementation honestly returns one +long model step after all chunks are generated and decoded; it does not claim +per-chunk interactive streaming. + +Only one GPU is currently supported. Multi-GPU/FSDP execution and live +RoboTwin control are outside this I2AV adapter. See `DESIGN.md` for ownership, +state transitions, and failure semantics. See `GPU_EVIDENCE.md` for exact +checkpoint parity bounds, real multi-chunk artifacts, memory measurements, and +the opt-in reproduction commands. + +## Real-model verification + +The checked-in GPU test is opt-in because the checkpoint is about 23 GiB: + +```bash +LINGBOT_VA_REAL_MODEL_RUN=1 uv run --no-sync pytest \ + integrations_v2/lingbot_va -m ci_gpu -s +``` + +Set `LINGBOT_VA_CHECKPOINT_ROOT` to reuse a resolved local snapshot. The +separate `LINGBOT_VA_REAL_MODEL_COMPILE_RUN=1` gate exercises the cold +`torch.compile` path. The upstream comparison harness and accepted numerical +bounds are documented in `GPU_EVIDENCE.md`. + +## Provenance + +- Source architecture/inference reference: + `robbyant/lingbot-va@7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb`. +- Official Robotwin checkpoint: + `robbyant/lingbot-va-posttrain-robotwin@8c9dea8abbc5c91cc9e18bc3264b8915083bbe70`. +- Initial FlashDreams draft source: PR #312 at + `f98cae4a18ddf6c189a6cfa2099265d6d570e337`. diff --git a/integrations/lingbot_va/tools/compare_upstream.py b/integrations/lingbot_va/tools/compare_upstream.py new file mode 100644 index 000000000..05bc2ec3e --- /dev/null +++ b/integrations/lingbot_va/tools/compare_upstream.py @@ -0,0 +1,421 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Compare native LingBot-VA video/action flows with pinned upstream code. + +The harness runs the same deterministic first-chunk tensors through the +upstream ``WanTransformer3DModel`` and the FlashDreams-native transformer. It +loads the models sequentially so the comparison also works on GPUs that cannot +hold two copies of the five-billion-parameter network. +""" + +from __future__ import annotations + +import argparse +import gc +import importlib +import importlib.machinery +import sys +import types +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import torch +from einops import rearrange +from torch import Tensor + +from lingbot_va.constants import ( + ROBOTWIN_ACTION_DIM, + ROBOTWIN_ACTION_PER_FRAME, + ROBOTWIN_ATTENTION_WINDOW, + ROBOTWIN_FRAME_CHUNK_SIZE, + ROBOTWIN_LATENT_CHANNELS, + ROBOTWIN_LATENT_HEIGHT, + ROBOTWIN_LATENT_TOKEN_PER_CHUNK, + ROBOTWIN_LATENT_WIDTH, + ROBOTWIN_ACTION_TOKEN_PER_CHUNK, +) +from lingbot_va.transformer import ( + LingbotVATransformer, + LingbotVATransformerConfig, +) +from lingbot_va.utils import get_mesh_id + + +@dataclass(frozen=True, slots=True) +class _Fixture: + """Deterministic CPU tensors shared by both implementations.""" + + video: Tensor + action: Tensor + text: Tensor + video_timesteps: Tensor + action_timesteps: Tensor + video_grid: Tensor + action_grid: Tensor + + +@dataclass(frozen=True, slots=True) +class _Difference: + """Absolute-error summary for one matched output.""" + + maximum: float + mean: float + root_mean_square: float + + +def _parser() -> argparse.ArgumentParser: + """Build the manual parity command parser.""" + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--checkpoint-root", + type=Path, + required=True, + help="Resolved checkpoint snapshot containing transformer/.", + ) + parser.add_argument( + "--upstream-root", + type=Path, + required=True, + help="Pinned robbyant/lingbot-va checkout root.", + ) + parser.add_argument("--device", default="cuda:0") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument( + "--compile-native", + action="store_true", + help="Compile the native video/action block loops before comparison.", + ) + parser.add_argument( + "--maximum-video-error", + type=float, + help="Fail when video flow maximum absolute error exceeds this value.", + ) + parser.add_argument( + "--mean-video-error", + type=float, + help="Fail when video flow mean absolute error exceeds this value.", + ) + parser.add_argument( + "--maximum-action-error", + type=float, + help="Fail when action flow maximum absolute error exceeds this value.", + ) + parser.add_argument( + "--mean-action-error", + type=float, + help="Fail when action flow mean absolute error exceeds this value.", + ) + return parser + + +def _fixture(seed: int) -> _Fixture: + """Create one realistic first-chunk input without loading model state.""" + generator = torch.Generator(device="cpu").manual_seed(seed) + video = torch.randn( + 1, + ROBOTWIN_LATENT_CHANNELS, + ROBOTWIN_FRAME_CHUNK_SIZE, + ROBOTWIN_LATENT_HEIGHT, + ROBOTWIN_LATENT_WIDTH, + generator=generator, + ).to(torch.bfloat16) + action = torch.randn( + 1, + ROBOTWIN_ACTION_DIM, + ROBOTWIN_FRAME_CHUNK_SIZE, + ROBOTWIN_ACTION_PER_FRAME, + 1, + generator=generator, + ).to(torch.bfloat16) + text = torch.randn(1, 512, 4096, generator=generator).to(torch.bfloat16) + video_timesteps = torch.tensor([[0.0, 500.0]], dtype=torch.float32) + action_timesteps = torch.tensor([[0.0, 500.0]], dtype=torch.float32) + video_grid = get_mesh_id( + ROBOTWIN_FRAME_CHUNK_SIZE, + ROBOTWIN_LATENT_HEIGHT // 2, + ROBOTWIN_LATENT_WIDTH // 2, + 0, + 1, + 0, + ) + action_grid = get_mesh_id( + ROBOTWIN_FRAME_CHUNK_SIZE, + ROBOTWIN_ACTION_PER_FRAME, + 1, + 1, + 1, + 0, + action=True, + ) + return _Fixture( + video=video, + action=action, + text=text, + video_timesteps=video_timesteps, + action_timesteps=action_timesteps, + video_grid=video_grid, + action_grid=action_grid, + ) + + +def _install_flash_attention_import_stub() -> None: + """Let the torch-attention upstream model import without FlashAttention.""" + if "flash_attn" in sys.modules: + return + module = types.ModuleType("flash_attn") + module.__spec__ = importlib.machinery.ModuleSpec("flash_attn", loader=None) + + def unavailable(*args: Any, **kwargs: Any) -> None: + del args, kwargs + raise RuntimeError("The parity harness selects attn_mode='torch'.") + + setattr(module, "flash_attn_func", unavailable) + sys.modules["flash_attn"] = module + + +def _upstream_outputs( + fixture: _Fixture, + checkpoint_root: Path, + upstream_root: Path, + device: torch.device, +) -> tuple[Tensor, Tensor]: + """Run pinned upstream video then action flows with one shared cache.""" + _install_flash_attention_import_stub() + sys.path.insert(0, str(upstream_root / "wan_va")) + try: + upstream_module = importlib.import_module("modules.model") + wan_transformer_model = upstream_module.WanTransformer3DModel + finally: + sys.path.pop(0) + + model = wan_transformer_model.from_pretrained( + checkpoint_root / "transformer", + torch_dtype=torch.bfloat16, + local_files_only=True, + attn_mode="torch", + ).to(device) + model.eval() + model.create_empty_cache( + "parity", + ROBOTWIN_ATTENTION_WINDOW, + ROBOTWIN_LATENT_TOKEN_PER_CHUNK, + ROBOTWIN_ACTION_TOKEN_PER_CHUNK, + device=device, + dtype=torch.bfloat16, + batch_size=1, + ) + with torch.no_grad(): + video_tokens = model( + { + "noisy_latents": fixture.video.to(device), + "timesteps": fixture.video_timesteps.to(device), + "grid_id": fixture.video_grid[:3].unsqueeze(0).to(device), + "text_emb": fixture.text.to(device), + }, + update_cache=1, + cache_name="parity", + action_mode=False, + ) + action_tokens = model( + { + "noisy_latents": fixture.action.to(device), + "timesteps": fixture.action_timesteps.to(device), + "grid_id": fixture.action_grid[:3].unsqueeze(0).to(device), + "text_emb": fixture.text.to(device), + }, + update_cache=1, + cache_name="parity", + action_mode=True, + ) + video = _upstream_video_tokens_to_tensor(video_tokens).float().cpu() + action = action_tokens.float().cpu() + model.to("cpu") + del model, video_tokens, action_tokens + gc.collect() + torch.cuda.empty_cache() + return video, action + + +def _upstream_video_tokens_to_tensor(tokens: Tensor) -> Tensor: + """Invert the upstream patch-token ordering to ``[B, C, F, H, W]``.""" + tokens = tokens.reshape( + tokens.shape[0], + ROBOTWIN_FRAME_CHUNK_SIZE, + ROBOTWIN_LATENT_HEIGHT // 2, + ROBOTWIN_LATENT_WIDTH // 2, + 1, + 2, + 2, + ROBOTWIN_LATENT_CHANNELS, + ) + return ( + tokens.permute(0, 7, 1, 4, 2, 5, 3, 6) + .flatten(6, 7) + .flatten(4, 5) + .flatten(2, 3) + ) + + +def _native_outputs( + fixture: _Fixture, + checkpoint_root: Path, + device: torch.device, + *, + compile_network: bool, +) -> tuple[Tensor, Tensor]: + """Run native video then action flows with the matching cache lifecycle.""" + transformer = LingbotVATransformer( + LingbotVATransformerConfig( + checkpoint_root=str(checkpoint_root), + dtype=torch.bfloat16, + compile_network=compile_network, + guidance_scale=1.0, + action_guidance_scale=1.0, + latent_height=ROBOTWIN_LATENT_HEIGHT, + latent_width=ROBOTWIN_LATENT_WIDTH, + frame_chunk_size=ROBOTWIN_FRAME_CHUNK_SIZE, + action_per_frame=ROBOTWIN_ACTION_PER_FRAME, + attn_window=ROBOTWIN_ATTENTION_WINDOW, + ) + ) + transformer.load_model(device) + cache = transformer.initialize_autoregressive_cache( + text_embeddings=fixture.text.to(device), + batch_size=1, + ) + cache.start(0) + video_input = rearrange( + fixture.video.to(device), + "b c (f kt) (h kh) (w kw) -> b (f h w) (c kt kh kw)", + kt=1, + kh=2, + kw=2, + ) + video_timesteps = torch.repeat_interleave( + fixture.video_timesteps.to(device), + (ROBOTWIN_LATENT_HEIGHT // 2) * (ROBOTWIN_LATENT_WIDTH // 2), + dim=1, + ) + action_input = rearrange( + fixture.action.to(device), + "b c f h w -> b (f h w) c", + ) + action_timesteps = torch.repeat_interleave( + fixture.action_timesteps.to(device), + ROBOTWIN_ACTION_PER_FRAME, + dim=1, + ) + with torch.no_grad(): + video_tokens = transformer.predict_flow( + video_input, + video_timesteps, + cache, + input={"grid_id": fixture.video_grid.to(device)}, + persist=True, + ) + action_tokens = transformer.predict_action_flow( + action_input, + action_timesteps, + cache, + input={"grid_id": fixture.action_grid.to(device)}, + persist=True, + ) + cache.finalize(0) + video = rearrange( + video_tokens, + "b (f h w) (c kt kh kw) -> b c (f kt) (h kh) (w kw)", + f=ROBOTWIN_FRAME_CHUNK_SIZE, + h=ROBOTWIN_LATENT_HEIGHT // 2, + w=ROBOTWIN_LATENT_WIDTH // 2, + kt=1, + kh=2, + kw=2, + ).float().cpu() + action = action_tokens.float().cpu() + transformer.network.to("cpu") + del transformer, cache, video_tokens, action_tokens + gc.collect() + torch.cuda.empty_cache() + return video, action + + +def _difference(reference: Tensor, actual: Tensor) -> _Difference: + """Summarize absolute error after checking shape and finiteness.""" + if reference.shape != actual.shape: + raise ValueError(f"Shape mismatch: upstream {reference.shape}, native {actual.shape}") + if not torch.isfinite(reference).all() or not torch.isfinite(actual).all(): + raise ValueError("Parity outputs must be finite.") + error = (reference - actual).abs() + return _Difference( + maximum=float(error.max()), + mean=float(error.mean()), + root_mean_square=float(torch.sqrt(torch.mean(error.square()))), + ) + + +def _check_threshold(name: str, value: float, threshold: float | None) -> None: + """Fail one explicitly requested parity bound.""" + if threshold is not None and value > threshold: + raise SystemExit(f"{name} {value:.8g} exceeds threshold {threshold:.8g}") + + +def main() -> None: + """Load both implementations, print differences, and enforce given bounds.""" + args = _parser().parse_args() + checkpoint_root = args.checkpoint_root.expanduser().resolve() + upstream_root = args.upstream_root.expanduser().resolve() + device = torch.device(args.device) + fixture = _fixture(args.seed) + upstream_video, upstream_action = _upstream_outputs( + fixture, + checkpoint_root, + upstream_root, + device, + ) + native_video, native_action = _native_outputs( + fixture, + checkpoint_root, + device, + compile_network=args.compile_native, + ) + video_difference = _difference(upstream_video, native_video) + action_difference = _difference(upstream_action, native_action) + print(f"video_shape={tuple(native_video.shape)} video_difference={video_difference}") + print( + f"action_shape={tuple(native_action.shape)} " + f"action_difference={action_difference}" + ) + _check_threshold( + "maximum video error", + video_difference.maximum, + args.maximum_video_error, + ) + _check_threshold("mean video error", video_difference.mean, args.mean_video_error) + _check_threshold( + "maximum action error", + action_difference.maximum, + args.maximum_action_error, + ) + _check_threshold( + "mean action error", + action_difference.mean, + args.mean_action_error, + ) + + +if __name__ == "__main__": + main() diff --git a/integrations_v2/lingbot_va/README.md b/integrations_v2/lingbot_va/README.md index 79c899f30..3b28aa20f 100644 --- a/integrations_v2/lingbot_va/README.md +++ b/integrations_v2/lingbot_va/README.md @@ -1,5 +1,34 @@ + + # LingBot-VA V2 application -This package adapts the LingBot-VA Robotwin I2AV model integration to the -FlashDreams V2 application, session, and model-loop APIs. See the model -package's README for checkpoint and input details. +This package adapts the LingBot-VA Robotwin I2AV model to the FlashDreams V2 +application/session/model-loop API. It registers the slug +`lingbot-va-robotwin-i2av` in `flashdreams.applications_v2`. + +```bash +uv run --project integrations_v2/lingbot_va flashdreams-run-v2 \ + lingbot-va-robotwin-i2av \ + --mode mp4 \ + --output-path outputs/lingbot_va/demo.mp4 \ + --stats-path outputs/lingbot_va/metrics.json \ + --tensor-artifact-dir outputs/lingbot_va \ + -- \ + --checkpoint-root robbyant/lingbot-va-posttrain-robotwin \ + --checkpoint-revision 8c9dea8abbc5c91cc9e18bc3264b8915083bbe70 \ + --input-image-dir assets/example_data/lingbot-va/robotwin \ + --num-chunks 10 +``` + +The application describes its natural session before initialization: TCHW, +256x320, 10 FPS, blocking backpressure, present-only-new behavior, and an +`actions[step, channel]` tensor artifact. Model loading is lazy on the model +thread. The finite loop emits one complete rollout and then reports finished. + +Use `-- --help` after the application slug for checkpoint, input, compilation, +offload, seed, guidance, inference-step, and scheduler-shift overrides. The +model package README documents checkpoint modes, camera/action contracts, +provenance, opt-in GPU tests, measured parity evidence, and limitations. diff --git a/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_real_model.py b/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_real_model.py new file mode 100644 index 000000000..c3a7ead01 --- /dev/null +++ b/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_real_model.py @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Opt-in GPU coverage for the real LingBot-VA checkpoint. + +The production-path test downloads about 23 GiB unless a resolved snapshot is +provided and writes a short MP4, metrics JSON, and actions array:: + + LINGBOT_VA_REAL_MODEL_RUN=1 uv run --no-sync pytest \ + integrations_v2/lingbot_va -m ci_gpu -s + +The separate compile gate is intentionally explicit because cold Inductor +autotuning can take minutes:: + + LINGBOT_VA_REAL_MODEL_COMPILE_RUN=1 uv run --no-sync pytest \ + integrations_v2/lingbot_va -m ci_gpu -s -k compile +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +import numpy as np +import pytest +import torch + +from flashdreams.runtime_v2.application_runner import ApplicationRunner +from flashdreams.runtime_v2.metrics_output_sink import MetricsOutputSink +from flashdreams.runtime_v2.mp4_client_window import Mp4ClientWindow +from flashdreams.runtime_v2.tensor_artifact_output_sink import ( + TensorArtifactOutputSink, +) +from flashdreams.t2v_v2.testing import real_model_run_skip_reason +from lingbot_va.constants import DEFAULT_CHECKPOINT_ROOT +from lingbot_va.engine import ( + LingbotVAEngine, + LingbotVAEngineConfig, + LingbotVAEngineState, +) +from lingbot_va_v2.app import LingbotVAApplication + +pytestmark = pytest.mark.ci_gpu + +_CHECKPOINT_REVISION = "8c9dea8abbc5c91cc9e18bc3264b8915083bbe70" +_REPOSITORY_ROOT = Path(__file__).resolve().parents[4] +_DEFAULT_INPUT_DIR = _REPOSITORY_ROOT / "assets/example_data/lingbot-va/robotwin" +_RUN_SKIP = real_model_run_skip_reason("LINGBOT_VA_REAL_MODEL_RUN") + + +def _compile_skip_reason() -> str | None: + """Return why the expensive real compile test cannot run here.""" + if not os.environ.get("LINGBOT_VA_REAL_MODEL_COMPILE_RUN"): + return "set LINGBOT_VA_REAL_MODEL_COMPILE_RUN=1 to test torch.compile" + if not torch.cuda.is_available(): + return "the model needs a GPU" + return None + + +_COMPILE_SKIP = _compile_skip_reason() + + +def _checkpoint_root() -> str: + """Return an optional local snapshot override or the official repository.""" + return os.environ.get("LINGBOT_VA_CHECKPOINT_ROOT", DEFAULT_CHECKPOINT_ROOT) + + +def _input_dir() -> Path: + """Return an optional three-camera input override or checked-in example.""" + return Path(os.environ.get("LINGBOT_VA_INPUT_DIR", _DEFAULT_INPUT_DIR)) + + +@pytest.mark.skipif(_RUN_SKIP is not None, reason=_RUN_SKIP or "") +def test_real_model_v2_offload_writes_video_actions_and_metrics( + tmp_path: Path, +) -> None: + application = LingbotVAApplication() + video_path = tmp_path / "clip.mp4" + metrics_path = tmp_path / "metrics.json" + ApplicationRunner( + application, + Mp4ClientWindow(video_path), + metrics_output_sink=MetricsOutputSink(metrics_path), + tensor_artifact_output_sink=TensorArtifactOutputSink(tmp_path), + ).run( + application.session_desc(), + [ + "--checkpoint-root", + _checkpoint_root(), + "--checkpoint-revision", + os.environ.get( + "LINGBOT_VA_CHECKPOINT_REVISION", + _CHECKPOINT_REVISION, + ), + "--input-image-dir", + str(_input_dir()), + "--num-chunks", + "2", + "--enable-offload", + "--no-compile", + ], + ) + + actions = np.load(tmp_path / "actions.npy", allow_pickle=False) + metrics = json.loads(metrics_path.read_text(encoding="utf-8")) + samples = {sample["name"]: sample for sample in metrics["samples"]} + + assert video_path.stat().st_size > 0 + assert actions.shape == (64, 16) + assert actions.dtype == np.float32 + assert np.isfinite(actions).all() + assert not np.array_equal(actions[:32], actions[32:]) + assert metrics["steps"] == [ + { + "step_index": 0, + "frame_count": 13, + "sample_count": 6, + } + ] + assert {name: sample["unit"] for name, sample in samples.items()} == { + "prompt_encode_s": "s", + "observation_encode_s": "s", + "denoise_s": "s", + "decode_s": "s", + "total_s": "s", + "peak_allocated_bytes": "bytes", + } + assert samples["peak_allocated_bytes"]["value"] > 0 + print(f"\nwrote {video_path}, {tmp_path / 'actions.npy'}, {metrics_path}") + + +@pytest.mark.skipif(_COMPILE_SKIP is not None, reason=_COMPILE_SKIP or "") +def test_real_model_compiled_engine_returns_finite_outputs() -> None: + engine = LingbotVAEngine( + LingbotVAEngineConfig( + checkpoint_root=_checkpoint_root(), + checkpoint_revision=os.environ.get( + "LINGBOT_VA_CHECKPOINT_REVISION", + _CHECKPOINT_REVISION, + ), + input_image_dir=_input_dir(), + num_chunks=1, + device="cuda:0", + compile_network=True, + video_inference_steps=1, + action_inference_steps=1, + ) + ) + try: + output = engine.run() + assert output.video.shape == (5, 3, 256, 320) + assert output.actions.shape == (32, 16) + assert torch.isfinite(output.video).all() + assert torch.isfinite(output.actions).all() + assert output.metrics["peak_allocated_bytes"] > 0 + finally: + engine.close() + + assert engine.state is LingbotVAEngineState.CLOSED From 292f50e8643112eb2bc057b8ac9737bb62dfba9e Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Tue, 25 Aug 2026 15:39:59 -0700 Subject: [PATCH 08/18] Complete LingBot package compliance Signed-off-by: Jonathan McCaffrey --- integrations/lingbot_va/pyproject.toml | 1 - integrations_v2/lingbot_va/pyproject.toml | 12 ++++++++++++ pyproject.toml | 4 ++++ uv.lock | 2 -- 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/integrations/lingbot_va/pyproject.toml b/integrations/lingbot_va/pyproject.toml index 40a463fbb..c2e64413c 100644 --- a/integrations/lingbot_va/pyproject.toml +++ b/integrations/lingbot_va/pyproject.toml @@ -38,7 +38,6 @@ flashdreams = { workspace = true } [project.optional-dependencies] dev = [ "pytest>=8.0", - "tomli>=2.0", ] [tool.setuptools.packages.find] diff --git a/integrations_v2/lingbot_va/pyproject.toml b/integrations_v2/lingbot_va/pyproject.toml index f4485f86e..74e61ddaa 100644 --- a/integrations_v2/lingbot_va/pyproject.toml +++ b/integrations_v2/lingbot_va/pyproject.toml @@ -1,5 +1,17 @@ # SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. [build-system] requires = ["setuptools>=69", "wheel"] diff --git a/pyproject.toml b/pyproject.toml index 462379488..14e3fb449 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,12 +57,14 @@ extraPaths = [ "integrations/fastvideo_causal_wan22", "integrations/hy_worldplay", "integrations/lingbot", + "integrations/lingbot_va", "integrations/sana", "integrations/self_forcing", "integrations/wan21", "integrations/wan22", "integrations_v2/color_fade", "integrations_v2/cam2v_lingbot", + "integrations_v2/lingbot_va", "integrations_v2/null_model", "integrations_v2/red_screen", "integrations_v2/t2v_causal_forcing", @@ -92,12 +94,14 @@ extra-paths = [ "integrations/fastvideo_causal_wan22", "integrations/hy_worldplay", "integrations/lingbot", + "integrations/lingbot_va", "integrations/sana", "integrations/self_forcing", "integrations/wan21", "integrations/wan22", "integrations_v2/color_fade", "integrations_v2/cam2v_lingbot", + "integrations_v2/lingbot_va", "integrations_v2/null_model", "integrations_v2/red_screen", "integrations_v2/t2v_causal_forcing", diff --git a/uv.lock b/uv.lock index bec7c2ebf..da13ab0f1 100644 --- a/uv.lock +++ b/uv.lock @@ -1362,7 +1362,6 @@ dependencies = [ [package.optional-dependencies] dev = [ { name = "pytest" }, - { name = "tomli" }, ] [package.metadata] @@ -1372,7 +1371,6 @@ requires-dist = [ { name = "flashdreams", editable = "flashdreams" }, { name = "pillow", specifier = ">=10" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, - { name = "tomli", marker = "extra == 'dev'", specifier = ">=2.0" }, { name = "transformers", specifier = ">=5.0,<6" }, ] provides-extras = ["dev"] From 6c6e054e1c5d7dafa81071f01292f92398682f94 Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Tue, 25 Aug 2026 17:24:28 -0700 Subject: [PATCH 09/18] Format LingBot integration Signed-off-by: Jonathan McCaffrey --- .../lingbot_va/lingbot_va/pipeline.py | 81 ++++++-- .../lingbot_va/lingbot_va/scheduler.py | 30 ++- .../lingbot_va/transformer/__init__.py | 31 +++- .../lingbot_va/transformer/checkpoint.py | 20 +- .../lingbot_va/transformer/impl/kvcache.py | 12 +- .../lingbot_va/transformer/impl/modules.py | 3 + .../lingbot_va/transformer/impl/network.py | 174 ++++++++++++------ integrations/lingbot_va/lingbot_va/utils.py | 4 +- .../lingbot_va/tools/compare_upstream.py | 37 ++-- 9 files changed, 283 insertions(+), 109 deletions(-) diff --git a/integrations/lingbot_va/lingbot_va/pipeline.py b/integrations/lingbot_va/lingbot_va/pipeline.py index 300fec79a..2a261d251 100644 --- a/integrations/lingbot_va/lingbot_va/pipeline.py +++ b/integrations/lingbot_va/lingbot_va/pipeline.py @@ -25,16 +25,23 @@ from torch import Tensor from tqdm import tqdm -from flashdreams.infra.pipeline import StreamInferencePipeline, StreamInferencePipelineConfig +from flashdreams.infra.pipeline import ( + StreamInferencePipeline, + StreamInferencePipelineConfig, +) from flashdreams.infra.pipeline.base import StreamInferencePipelineCache -from lingbot_va.scheduler import LingbotVAFlowMatchScheduler, LingbotVAFlowMatchSchedulerConfig +from lingbot_va.scheduler import ( + LingbotVAFlowMatchScheduler, + LingbotVAFlowMatchSchedulerConfig, +) from lingbot_va.transformer import LingbotVATransformer, LingbotVATransformerCache from lingbot_va.utils import get_mesh_id class LingbotVAOutput(NamedTuple): """Output from one AR step of the LingBot-VA pipeline.""" + latent: Tensor action: Tensor @@ -155,8 +162,12 @@ def generate( # type: ignore[override] frame_st_id = autoregressive_index * fcs # Initial noise - latents = torch.randn(1, cfg.latent_channels, fcs, lh, lw, device=device, dtype=dtype) - actions = torch.randn(1, cfg.action_dim, fcs, cfg.action_per_frame, 1, device=device, dtype=dtype) + latents = torch.randn( + 1, cfg.latent_channels, fcs, lh, lw, device=device, dtype=dtype + ) + actions = torch.randn( + 1, cfg.action_dim, fcs, cfg.action_per_frame, 1, device=device, dtype=dtype + ) # Timesteps video_timesteps = self.video_scheduler.padded_timesteps @@ -164,10 +175,21 @@ def generate( # type: ignore[override] # RoPE grid IDs video_grid_id = get_mesh_id( - fcs // ps[0], lh // ps[1], lw // ps[2], 0, 1, frame_st_id, + fcs // ps[0], + lh // ps[1], + lw // ps[2], + 0, + 1, + frame_st_id, ).to(device) action_grid_id = get_mesh_id( - fcs, cfg.action_per_frame, 1, 1, 1, frame_st_id, action=True, + fcs, + cfg.action_per_frame, + 1, + 1, + 1, + frame_st_id, + action=True, ).to(device) # Open cache window @@ -183,8 +205,10 @@ def generate( # type: ignore[override] noisy[:, :, 0:1] = latent_cond x = rearrange( noisy, - 'b c (f p1) (h p2) (w p3) -> b (f h w) (c p1 p2 p3)', - p1=ps[0], p2=ps[1], p3=ps[2], + "b c (f p1) (h p2) (w p3) -> b (f h w) (c p1 p2 p3)", + p1=ps[0], + p2=ps[1], + p3=ps[2], ) t_val = float(t) @@ -195,16 +219,23 @@ def generate( # type: ignore[override] ts[:, :cond_tokens] = 0.0 pred = self.transformer.predict_flow( - x, ts, transformer_cache, input={"grid_id": video_grid_id}, + x, + ts, + transformer_cache, + input={"grid_id": video_grid_id}, persist=last_step, ) if not last_step: pred = rearrange( pred, - 'b (f h w) (c kt kh kw) -> b c (f kt) (h kh) (w kw)', - f=fcs // ps[0], h=lh // ps[1], w=lw // ps[2], - kt=ps[0], kh=ps[1], kw=ps[2], + "b (f h w) (c kt kh kw) -> b c (f kt) (h kh) (w kw)", + f=fcs // ps[0], + h=lh // ps[1], + w=lw // ps[2], + kt=ps[0], + kh=ps[1], + kw=ps[2], ) latents = self.video_scheduler.step(pred, t, latents) @@ -215,24 +246,38 @@ def generate( # type: ignore[override] for i, t in enumerate(tqdm(action_timesteps, desc="action denoise")): last_step = i == len(action_timesteps) - 1 action_cond = ( - torch.zeros(1, cfg.action_dim, 1, cfg.action_per_frame, 1, device=device, dtype=dtype) - if frame_st_id == 0 else None + torch.zeros( + 1, + cfg.action_dim, + 1, + cfg.action_per_frame, + 1, + device=device, + dtype=dtype, + ) + if frame_st_id == 0 + else None ) noisy_a = actions.clone() if action_cond is not None: noisy_a[:, :, 0:1] = action_cond noisy_a[:, ~action_mask] *= 0 - x_a = rearrange(noisy_a, 'b c f h w -> b (f h w) c') + x_a = rearrange(noisy_a, "b c f h w -> b (f h w) c") t_val = float(t) n_tokens_a = x_a.shape[1] - ts_a = torch.full((1, n_tokens_a), t_val, dtype=torch.float32, device=device) + ts_a = torch.full( + (1, n_tokens_a), t_val, dtype=torch.float32, device=device + ) if action_cond is not None and frame_st_id == 0: - ts_a[:, :cfg.action_per_frame] = 0.0 + ts_a[:, : cfg.action_per_frame] = 0.0 pred = self.transformer.predict_action_flow( - x_a, ts_a, transformer_cache, input={"grid_id": action_grid_id}, + x_a, + ts_a, + transformer_cache, + input={"grid_id": action_grid_id}, persist=last_step, ) diff --git a/integrations/lingbot_va/lingbot_va/scheduler.py b/integrations/lingbot_va/lingbot_va/scheduler.py index ec7a5c08e..6918745f2 100644 --- a/integrations/lingbot_va/lingbot_va/scheduler.py +++ b/integrations/lingbot_va/lingbot_va/scheduler.py @@ -22,7 +22,11 @@ import torch from torch import Tensor -from flashdreams.infra.diffusion.scheduler import FlowPredictor, Scheduler, SchedulerConfig +from flashdreams.infra.diffusion.scheduler import ( + FlowPredictor, + Scheduler, + SchedulerConfig, +) def warp_sigmas(sigmas: Tensor, shift: float) -> Tensor: @@ -62,7 +66,9 @@ def __init__(self, config: LingbotVAFlowMatchSchedulerConfig) -> None: self.register_buffer("timesteps", timesteps, persistent=False) @staticmethod - def build_schedule(config: LingbotVAFlowMatchSchedulerConfig) -> tuple[Tensor, Tensor]: + def build_schedule( + config: LingbotVAFlowMatchSchedulerConfig, + ) -> tuple[Tensor, Tensor]: sigma_start = config.sigma_min + (config.sigma_max - config.sigma_min) if config.extra_one_step: sigmas = torch.linspace( @@ -93,13 +99,17 @@ def padded_timesteps(self) -> Tensor: def step(self, model_output: Tensor, timestep: Tensor, sample: Tensor) -> Tensor: """Apply one upstream Euler step in sigma space.""" - timestep_cpu = timestep.detach().to(device=self.timesteps.device, dtype=self.timesteps.dtype) + timestep_cpu = timestep.detach().to( + device=self.timesteps.device, dtype=self.timesteps.dtype + ) timestep_id = torch.argmin((self.timesteps - timestep_cpu).abs()) sigma = self.sigmas[timestep_id].to(device=sample.device, dtype=sample.dtype) if int(timestep_id.item()) + 1 >= self.sigmas.shape[0]: sigma_next = torch.zeros((), device=sample.device, dtype=sample.dtype) else: - sigma_next = self.sigmas[timestep_id + 1].to(device=sample.device, dtype=sample.dtype) + sigma_next = self.sigmas[timestep_id + 1].to( + device=sample.device, dtype=sample.dtype + ) return sample + model_output * (sigma_next - sigma) def sample( @@ -112,7 +122,9 @@ def sample( del rng sample = initial_noise for timestep in self.timesteps: - flow = predict_flow(sample, timestep.to(device=sample.device, dtype=sample.dtype)) + flow = predict_flow( + sample, timestep.to(device=sample.device, dtype=sample.dtype) + ) sample = self.step(flow, timestep, sample) return sample @@ -124,7 +136,11 @@ def add_noise( ) -> Tensor: """Apply upstream forward corruption at the nearest scheduler timestep.""" noise = torch.empty_like(clean_input).normal_(generator=rng) - timestep_cpu = timestep.detach().to(device=self.timesteps.device, dtype=self.timesteps.dtype) + timestep_cpu = timestep.detach().to( + device=self.timesteps.device, dtype=self.timesteps.dtype + ) timestep_id = torch.argmin((self.timesteps - timestep_cpu).abs()) - sigma = self.sigmas[timestep_id].to(device=clean_input.device, dtype=clean_input.dtype) + sigma = self.sigmas[timestep_id].to( + device=clean_input.device, dtype=clean_input.dtype + ) return (1.0 - sigma) * clean_input + sigma * noise diff --git a/integrations/lingbot_va/lingbot_va/transformer/__init__.py b/integrations/lingbot_va/lingbot_va/transformer/__init__.py index bdb7b75f1..6087d07ba 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/__init__.py +++ b/integrations/lingbot_va/lingbot_va/transformer/__init__.py @@ -51,6 +51,7 @@ # Cache # --------------------------------------------------------------------------- + @dataclass(kw_only=True) class LingbotVATransformerCache(TransformerAutoregressiveCache): """Per-rollout AR cache.""" @@ -86,6 +87,7 @@ def finalize(self, autoregressive_index: int) -> None: # Config # --------------------------------------------------------------------------- + @dataclass(kw_only=True) class LingbotVATransformerConfig(TransformerConfig): """Config for the native LingBot-VA transformer.""" @@ -113,6 +115,7 @@ class LingbotVATransformerConfig(TransformerConfig): # Transformer # --------------------------------------------------------------------------- + class LingbotVATransformer(Transformer[LingbotVATransformerCache]): """Native LingBot-VA transformer with torch.compile support.""" @@ -123,7 +126,7 @@ def __init__(self, config: LingbotVATransformerConfig) -> None: self.config: LingbotVATransformerConfig = config self._anchor = nn.Parameter(torch.empty(0)) # Use object.__setattr__ to avoid nn.Module type checks on compiled modules - object.__setattr__(self, '_network', None) + object.__setattr__(self, "_network", None) def load_model(self, device: torch.device) -> None: """Build, load weights, and optionally compile the network.""" @@ -132,7 +135,9 @@ def load_model(self, device: torch.device) -> None: net.eval() ckpt_path = os.path.join(cfg.checkpoint_root, "transformer") - idx_path = os.path.join(ckpt_path, "diffusion_pytorch_model.safetensors.index.json") + idx_path = os.path.join( + ckpt_path, "diffusion_pytorch_model.safetensors.index.json" + ) if os.path.exists(idx_path): ckpt_path = idx_path state_dict = load_checkpoint(ckpt_path, map_location="cpu") @@ -164,7 +169,7 @@ def load_model(self, device: torch.device) -> None: ), ) - object.__setattr__(self, '_network', net) + object.__setattr__(self, "_network", net) @property def network(self) -> WanVADiTNetwork: @@ -239,7 +244,11 @@ def predict_flow( ).to(noisy_latent.device) flow_cond, video_kv_cond = self.network.forward_video( - noisy_latent, timestep, cache.network_cache, rope_freqs, persist=persist, + noisy_latent, + timestep, + cache.network_cache, + rope_freqs, + persist=persist, ) if persist: assert video_kv_cond is not None @@ -247,13 +256,19 @@ def predict_flow( if cache.network_cache_uncond is not None: flow_uncond, video_kv_uncond = self.network.forward_video( - noisy_latent, timestep, cache.network_cache_uncond, rope_freqs, persist=persist, + noisy_latent, + timestep, + cache.network_cache_uncond, + rope_freqs, + persist=persist, ) if persist: assert video_kv_uncond is not None cache.video_kv_uncond = video_kv_uncond if self.config.guidance_scale > 1.0: - return flow_uncond + self.config.guidance_scale * (flow_cond - flow_uncond) + return flow_uncond + self.config.guidance_scale * ( + flow_cond - flow_uncond + ) return flow_cond @@ -296,7 +311,9 @@ def predict_action_flow( if persist: cache.video_kv_uncond = None if self.config.action_guidance_scale > 1.0: - return flow_uncond + self.config.action_guidance_scale * (flow_cond - flow_uncond) + return flow_uncond + self.config.action_guidance_scale * ( + flow_cond - flow_uncond + ) return flow_cond diff --git a/integrations/lingbot_va/lingbot_va/transformer/checkpoint.py b/integrations/lingbot_va/lingbot_va/transformer/checkpoint.py index 63fd15978..2620403f6 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/checkpoint.py +++ b/integrations/lingbot_va/lingbot_va/transformer/checkpoint.py @@ -50,11 +50,23 @@ (r"^condition_embedder\.text_embedder\.linear_2\.(.+)$", r"text_embedding.2.\1"), (r"^action_embedder\.(.+)$", r"action_embedder.\1"), (r"^action_proj_out\.(.+)$", r"action_head.\1"), - (r"^condition_embedder_action\.time_embedder\.linear_1\.(.+)$", r"action_time_embedding.0.\1"), - (r"^condition_embedder_action\.time_embedder\.linear_2\.(.+)$", r"action_time_embedding.2.\1"), + ( + r"^condition_embedder_action\.time_embedder\.linear_1\.(.+)$", + r"action_time_embedding.0.\1", + ), + ( + r"^condition_embedder_action\.time_embedder\.linear_2\.(.+)$", + r"action_time_embedding.2.\1", + ), (r"^condition_embedder_action\.time_proj\.(.+)$", r"action_time_projection.1.\1"), - (r"^condition_embedder_action\.text_embedder\.linear_1\.(.+)$", r"action_text_embedding.0.\1"), - (r"^condition_embedder_action\.text_embedder\.linear_2\.(.+)$", r"action_text_embedding.2.\1"), + ( + r"^condition_embedder_action\.text_embedder\.linear_1\.(.+)$", + r"action_text_embedding.0.\1", + ), + ( + r"^condition_embedder_action\.text_embedder\.linear_2\.(.+)$", + r"action_text_embedding.2.\1", + ), (r"^proj_out\.(.+)$", r"head.head.\1"), (r"^scale_shift_table$", r"head.modulation"), ] diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/kvcache.py b/integrations/lingbot_va/lingbot_va/transformer/impl/kvcache.py index b3de6bed9..d41abc714 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/impl/kvcache.py +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/kvcache.py @@ -129,15 +129,21 @@ def write_video(self, k: Tensor, v: Tensor) -> None: """ batch, _, heads, head_dim = k.shape action_k = torch.zeros( - batch, self.action_chunk, heads, head_dim, - device=k.device, dtype=k.dtype, + batch, + self.action_chunk, + heads, + head_dim, + device=k.device, + dtype=k.dtype, ) action_v = torch.zeros_like(action_k) full_k = torch.cat([k, action_k], dim=1) full_v = torch.cat([v, action_v], dim=1) self.kv_cache.update(full_k, full_v) - def write_action(self, k: Tensor, v: Tensor, video_k: Tensor, video_v: Tensor) -> None: + def write_action( + self, k: Tensor, v: Tensor, video_k: Tensor, video_v: Tensor + ) -> None: """Write full [video|action] KV to the current chunk (overwrite). Called once on the final action denoising step. diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py b/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py index 97c7e3c6e..e392fd976 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py @@ -43,6 +43,7 @@ # Cache # --------------------------------------------------------------------------- + @dataclass class VABlockCache: """Per-block cache for a VA transformer block.""" @@ -55,6 +56,7 @@ class VABlockCache: # VASelfAttention # --------------------------------------------------------------------------- + class VASelfAttention(MultiHeadAttention): """Self-attention that takes committed KV as plain tensors. @@ -108,6 +110,7 @@ def forward( # VABlock # --------------------------------------------------------------------------- + class VABlock(nn.Module): """Transformer block for video-action models. diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/network.py b/integrations/lingbot_va/lingbot_va/transformer/impl/network.py index fa1adffa0..469de099f 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/impl/network.py +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/network.py @@ -41,6 +41,7 @@ # Config # --------------------------------------------------------------------------- + @dataclass class WanVADiTNetworkConfig: """Network config for the LingBot-VA video-action DiT.""" @@ -67,6 +68,7 @@ class WanVADiTNetworkConfig: # Network cache # --------------------------------------------------------------------------- + @dataclass class WanVADiTNetworkCache: """Per-block caches for the entire network.""" @@ -81,6 +83,7 @@ def __getitem__(self, index: int) -> VABlockCache: # RoPE helper # --------------------------------------------------------------------------- + def compute_rope_freqs_from_grid( grid_id: Tensor, head_dim: int, @@ -95,9 +98,27 @@ def compute_rope_freqs_from_grid( w_dim = head_dim // 3 device = grid_id.device - f_base = 1.0 / (theta ** (torch.arange(0, f_dim, 2, device=device, dtype=torch.float64)[: f_dim // 2] / f_dim)) - h_base = 1.0 / (theta ** (torch.arange(0, h_dim, 2, device=device, dtype=torch.float64)[: h_dim // 2] / h_dim)) - w_base = 1.0 / (theta ** (torch.arange(0, w_dim, 2, device=device, dtype=torch.float64)[: w_dim // 2] / w_dim)) + f_base = 1.0 / ( + theta + ** ( + torch.arange(0, f_dim, 2, device=device, dtype=torch.float64)[: f_dim // 2] + / f_dim + ) + ) + h_base = 1.0 / ( + theta + ** ( + torch.arange(0, h_dim, 2, device=device, dtype=torch.float64)[: h_dim // 2] + / h_dim + ) + ) + w_base = 1.0 / ( + theta + ** ( + torch.arange(0, w_dim, 2, device=device, dtype=torch.float64)[: w_dim // 2] + / w_dim + ) + ) f_angles = grid_id[0].to(torch.float64).unsqueeze(-1) * f_base.unsqueeze(0) h_angles = grid_id[1].to(torch.float64).unsqueeze(-1) * h_base.unsqueeze(0) @@ -105,11 +126,14 @@ def compute_rope_freqs_from_grid( # Interleaved format: each angle duplicated as [theta_0, theta_0, theta_1, theta_1, ...] # to match RotaryPositionEmbedding3D._cat_freqs(interleaved=True) - freqs = torch.cat([ - f_angles.repeat_interleave(2, dim=-1), - h_angles.repeat_interleave(2, dim=-1), - w_angles.repeat_interleave(2, dim=-1), - ], dim=-1).float() + freqs = torch.cat( + [ + f_angles.repeat_interleave(2, dim=-1), + h_angles.repeat_interleave(2, dim=-1), + w_angles.repeat_interleave(2, dim=-1), + ], + dim=-1, + ).float() return freqs.unsqueeze(1).unsqueeze(1) # [L, 1, 1, head_dim] @@ -118,6 +142,7 @@ def compute_rope_freqs_from_grid( # Network # --------------------------------------------------------------------------- + class WanVADiTNetwork(nn.Module): """Video-Action DiT network with native flashdreams building blocks.""" @@ -171,18 +196,20 @@ def __init__(self, config: WanVADiTNetworkConfig) -> None: ) # Shared transformer blocks - self.blocks = nn.ModuleList([ - VABlock( - dim=self.dim, - ffn_dim=config.ffn_dim, - num_heads=config.num_heads, - cross_attn_norm=config.cross_attn_norm, - eps=config.eps, - apply_rope_before_kvcache=config.apply_rope_before_kvcache, - cp_method=config.cp_method, - ) - for _ in range(config.num_layers) - ]) + self.blocks = nn.ModuleList( + [ + VABlock( + dim=self.dim, + ffn_dim=config.ffn_dim, + num_heads=config.num_heads, + cross_attn_norm=config.cross_attn_norm, + eps=config.eps, + apply_rope_before_kvcache=config.apply_rope_before_kvcache, + cp_method=config.cp_method, + ) + for _ in range(config.num_layers) + ] + ) # Video output head self.head = Head(self.dim, config.out_dim, config.patch_size, config.eps) @@ -205,17 +232,24 @@ def update_parameters_after_loading_checkpoint(self) -> None: def _fuse_shuffle_op_into_last_layer(self) -> None: """Fuse channel shuffle into head.head weights (same as WanDiTNetwork).""" from einops import rearrange + kt, kh, kw = self.patch_size self.head.head.weight.data = rearrange( self.head.head.weight, "(kt kh kw c) in_dim -> (c kt kh kw) in_dim", - kt=kt, kh=kh, kw=kw, c=self.out_dim, + kt=kt, + kh=kh, + kw=kw, + c=self.out_dim, ).contiguous() if self.head.head.bias is not None: self.head.head.bias.data = rearrange( self.head.head.bias, "(kt kh kw c) -> (c kt kh kw)", - kt=kt, kh=kh, kw=kw, c=self.out_dim, + kt=kt, + kh=kh, + kw=kw, + c=self.out_dim, ).contiguous() def initialize_cache( @@ -243,10 +277,12 @@ def initialize_cache( dtype=text_embeddings.dtype, ) cross_attn_cache = block.cross_attn.initialize_cache(context_text) - block_caches.append(VABlockCache( - self_attn=self_attn_cache, - cross_attn=cross_attn_cache, - )) + block_caches.append( + VABlockCache( + self_attn=self_attn_cache, + cross_attn=cross_attn_cache, + ) + ) return WanVADiTNetworkCache(block_caches=block_caches) def _extract_cache_tensors( @@ -260,22 +296,30 @@ def _extract_cache_tensors( (committed_k_stack, committed_v_stack, cross_k_stack, cross_v_stack) All shapes: [num_layers, batch, seq_len, heads, head_dim] """ - committed_k = torch.stack([ - bc.self_attn.kv_cache._k[:, :bc.self_attn.n_committed_tokens] - for bc in cache.block_caches - ]) - committed_v = torch.stack([ - bc.self_attn.kv_cache._v[:, :bc.self_attn.n_committed_tokens] - for bc in cache.block_caches - ]) - cross_k = torch.stack([ - bc.cross_attn.text._k[:, :bc.cross_attn.text._n_cached] - for bc in cache.block_caches - ]) - cross_v = torch.stack([ - bc.cross_attn.text._v[:, :bc.cross_attn.text._n_cached] - for bc in cache.block_caches - ]) + committed_k = torch.stack( + [ + bc.self_attn.kv_cache._k[:, : bc.self_attn.n_committed_tokens] + for bc in cache.block_caches + ] + ) + committed_v = torch.stack( + [ + bc.self_attn.kv_cache._v[:, : bc.self_attn.n_committed_tokens] + for bc in cache.block_caches + ] + ) + cross_k = torch.stack( + [ + bc.cross_attn.text._k[:, : bc.cross_attn.text._n_cached] + for bc in cache.block_caches + ] + ) + cross_v = torch.stack( + [ + bc.cross_attn.text._v[:, : bc.cross_attn.text._n_cached] + for bc in cache.block_caches + ] + ) return committed_k, committed_v, cross_k, cross_v def _forward_blocks_video( @@ -300,7 +344,9 @@ def _forward_blocks_video( (head_output, k_fresh_list, v_fresh_list) """ x = self.patch_embedding(x) - e = self.time_embedding(sinusoidal_embedding_1d(self.freq_dim, timesteps).type_as(x)) + e = self.time_embedding( + sinusoidal_embedding_1d(self.freq_dim, timesteps).type_as(x) + ) e0 = self.time_projection(e).unflatten(-1, (6, self.dim)) block_e = e0 head_e = e.unsqueeze(-2) @@ -309,9 +355,12 @@ def _forward_blocks_video( v_list: list[Tensor] = [] for block_idx, block in enumerate(self.blocks): x, k_fresh, v_fresh = block( - x, block_e, - committed_k[block_idx], committed_v[block_idx], - cross_k[block_idx], cross_v[block_idx], + x, + block_e, + committed_k[block_idx], + committed_v[block_idx], + cross_k[block_idx], + cross_v[block_idx], rope_freqs, ) k_list.append(k_fresh) @@ -335,7 +384,9 @@ def _forward_blocks_action( (action_output, k_fresh_list, v_fresh_list) """ x = self.action_embedder(x) - e = self.action_time_embedding(sinusoidal_embedding_1d(self.freq_dim, timesteps).type_as(x)) + e = self.action_time_embedding( + sinusoidal_embedding_1d(self.freq_dim, timesteps).type_as(x) + ) e0 = self.action_time_projection(e).unflatten(-1, (6, self.dim)) block_e = e0 head_e = e.unsqueeze(-2) @@ -344,16 +395,21 @@ def _forward_blocks_action( v_list: list[Tensor] = [] for block_idx, block in enumerate(self.blocks): x, k_fresh, v_fresh = block( - x, block_e, - committed_k[block_idx], committed_v[block_idx], - cross_k[block_idx], cross_v[block_idx], + x, + block_e, + committed_k[block_idx], + committed_v[block_idx], + cross_k[block_idx], + cross_v[block_idx], rope_freqs, ) k_list.append(k_fresh) v_list.append(v_fresh) # Action output: shared modulation + separate projection - e_chunks = [c.squeeze(-2) for c in (self.head.modulation + head_e).chunk(2, dim=-2)] + e_chunks = [ + c.squeeze(-2) for c in (self.head.modulation + head_e).chunk(2, dim=-2) + ] x = self.head.norm(x) * (1 + e_chunks[1]) + e_chunks[0] return self.action_head(x), k_list, v_list @@ -380,7 +436,13 @@ def forward_video( # Compiled block loop (pure tensors, no cache access) output, k_list, v_list = self._forward_blocks_video( - x, timesteps, committed_k, committed_v, cross_k, cross_v, rope_freqs, + x, + timesteps, + committed_k, + committed_v, + cross_k, + cross_v, + rope_freqs, ) # Cache write (outside compile boundary) @@ -428,7 +490,13 @@ def forward_action( # Compiled block loop (pure tensors, no cache access) output, k_list, v_list = self._forward_blocks_action( - x, timesteps, committed_k, committed_v, cross_k, cross_v, rope_freqs, + x, + timesteps, + committed_k, + committed_v, + cross_k, + cross_v, + rope_freqs, ) # Cache write (outside compile boundary) diff --git a/integrations/lingbot_va/lingbot_va/utils.py b/integrations/lingbot_va/lingbot_va/utils.py index 6f2825fc4..3d0385311 100644 --- a/integrations/lingbot_va/lingbot_va/utils.py +++ b/integrations/lingbot_va/lingbot_va/utils.py @@ -98,7 +98,9 @@ def data_seq_to_patch( def resolve_prompt(value: str | Path) -> str: """Resolve an inline prompt or a text file whose first non-empty line is used.""" if isinstance(value, Path): - lines = [line.strip() for line in value.read_text().splitlines() if line.strip()] + lines = [ + line.strip() for line in value.read_text().splitlines() if line.strip() + ] assert lines, f"prompt file {value} has no non-empty lines" return lines[0] assert value, "prompt must be a non-empty string or a path" diff --git a/integrations/lingbot_va/tools/compare_upstream.py b/integrations/lingbot_va/tools/compare_upstream.py index 05bc2ec3e..ce19a1149 100644 --- a/integrations/lingbot_va/tools/compare_upstream.py +++ b/integrations/lingbot_va/tools/compare_upstream.py @@ -263,10 +263,7 @@ def _upstream_video_tokens_to_tensor(tokens: Tensor) -> Tensor: ROBOTWIN_LATENT_CHANNELS, ) return ( - tokens.permute(0, 7, 1, 4, 2, 5, 3, 6) - .flatten(6, 7) - .flatten(4, 5) - .flatten(2, 3) + tokens.permute(0, 7, 1, 4, 2, 5, 3, 6).flatten(6, 7).flatten(4, 5).flatten(2, 3) ) @@ -335,16 +332,20 @@ def _native_outputs( persist=True, ) cache.finalize(0) - video = rearrange( - video_tokens, - "b (f h w) (c kt kh kw) -> b c (f kt) (h kh) (w kw)", - f=ROBOTWIN_FRAME_CHUNK_SIZE, - h=ROBOTWIN_LATENT_HEIGHT // 2, - w=ROBOTWIN_LATENT_WIDTH // 2, - kt=1, - kh=2, - kw=2, - ).float().cpu() + video = ( + rearrange( + video_tokens, + "b (f h w) (c kt kh kw) -> b c (f kt) (h kh) (w kw)", + f=ROBOTWIN_FRAME_CHUNK_SIZE, + h=ROBOTWIN_LATENT_HEIGHT // 2, + w=ROBOTWIN_LATENT_WIDTH // 2, + kt=1, + kh=2, + kw=2, + ) + .float() + .cpu() + ) action = action_tokens.float().cpu() transformer.network.to("cpu") del transformer, cache, video_tokens, action_tokens @@ -356,7 +357,9 @@ def _native_outputs( def _difference(reference: Tensor, actual: Tensor) -> _Difference: """Summarize absolute error after checking shape and finiteness.""" if reference.shape != actual.shape: - raise ValueError(f"Shape mismatch: upstream {reference.shape}, native {actual.shape}") + raise ValueError( + f"Shape mismatch: upstream {reference.shape}, native {actual.shape}" + ) if not torch.isfinite(reference).all() or not torch.isfinite(actual).all(): raise ValueError("Parity outputs must be finite.") error = (reference - actual).abs() @@ -394,7 +397,9 @@ def main() -> None: ) video_difference = _difference(upstream_video, native_video) action_difference = _difference(upstream_action, native_action) - print(f"video_shape={tuple(native_video.shape)} video_difference={video_difference}") + print( + f"video_shape={tuple(native_video.shape)} video_difference={video_difference}" + ) print( f"action_shape={tuple(native_action.shape)} " f"action_difference={action_difference}" From a295b00742792245535cb1d13f517d6bad9d835e Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Tue, 25 Aug 2026 21:14:59 -0700 Subject: [PATCH 10/18] Fix LingBot lint checks Signed-off-by: Jonathan McCaffrey --- integrations/lingbot_va/lingbot_va/engine.py | 2 +- .../lingbot_va/lingbot_va/pipeline.py | 1 - .../lingbot_va/transformer/__init__.py | 2 - .../lingbot_va/transformer/impl/modules.py | 4 +- .../lingbot_va/transformer/impl/network.py | 1 - integrations/lingbot_va/tests/test_action.py | 1 - .../lingbot_va/tests/test_cfg_cache.py | 31 +++++++------- integrations/lingbot_va/tests/test_engine.py | 40 ++++++++++++------- integrations/lingbot_va/tests/test_loaders.py | 3 +- .../lingbot_va/tools/compare_upstream.py | 5 +-- .../lingbot_va/lingbot_va_v2/app.py | 31 +++++++------- .../lingbot_va_v2/tests/test_app.py | 14 +++---- .../lingbot_va_v2/tests/test_real_model.py | 14 +++---- 13 files changed, 76 insertions(+), 73 deletions(-) diff --git a/integrations/lingbot_va/lingbot_va/engine.py b/integrations/lingbot_va/lingbot_va/engine.py index 8bd5f6271..25fe4e7f2 100644 --- a/integrations/lingbot_va/lingbot_va/engine.py +++ b/integrations/lingbot_va/lingbot_va/engine.py @@ -57,8 +57,8 @@ ROBOTWIN_HEIGHT, ROBOTWIN_OBS_CAM_KEYS, ROBOTWIN_SNR_SHIFT, - ROBOTWIN_VIDEO_INFERENCE_STEPS, ROBOTWIN_VAE_TEMPORAL_SCALE, + ROBOTWIN_VIDEO_INFERENCE_STEPS, ROBOTWIN_WIDTH, ) from lingbot_va.pipeline import LingbotVAInferencePipelineConfig diff --git a/integrations/lingbot_va/lingbot_va/pipeline.py b/integrations/lingbot_va/lingbot_va/pipeline.py index 2a261d251..30624bd35 100644 --- a/integrations/lingbot_va/lingbot_va/pipeline.py +++ b/integrations/lingbot_va/lingbot_va/pipeline.py @@ -30,7 +30,6 @@ StreamInferencePipelineConfig, ) from flashdreams.infra.pipeline.base import StreamInferencePipelineCache - from lingbot_va.scheduler import ( LingbotVAFlowMatchScheduler, LingbotVAFlowMatchSchedulerConfig, diff --git a/integrations/lingbot_va/lingbot_va/transformer/__init__.py b/integrations/lingbot_va/lingbot_va/transformer/__init__.py index 6087d07ba..53ad2348c 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/__init__.py +++ b/integrations/lingbot_va/lingbot_va/transformer/__init__.py @@ -36,7 +36,6 @@ TransformerAutoregressiveCache, TransformerConfig, ) - from lingbot_va.transformer.checkpoint import state_dict_transform from lingbot_va.transformer.impl.network import ( VideoKV, @@ -46,7 +45,6 @@ compute_rope_freqs_from_grid, ) - # --------------------------------------------------------------------------- # Cache # --------------------------------------------------------------------------- diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py b/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py index e392fd976..03d85c54b 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/modules.py @@ -31,14 +31,12 @@ from flashdreams.core.attention.rope import apply_rope_freqs from flashdreams.recipes.wan.transformer.impl.modules import ( - CrossAttnCache, CrossAttention, + CrossAttnCache, MultiHeadAttention, ) - from lingbot_va.transformer.impl.kvcache import VAKVCache - # --------------------------------------------------------------------------- # Cache # --------------------------------------------------------------------------- diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/network.py b/integrations/lingbot_va/lingbot_va/transformer/impl/network.py index 469de099f..2825bdf20 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/impl/network.py +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/network.py @@ -29,7 +29,6 @@ Head, sinusoidal_embedding_1d, ) - from lingbot_va.transformer.impl.kvcache import VAKVCache from lingbot_va.transformer.impl.modules import VABlock, VABlockCache diff --git a/integrations/lingbot_va/tests/test_action.py b/integrations/lingbot_va/tests/test_action.py index 64802fd53..69425b4f7 100644 --- a/integrations/lingbot_va/tests/test_action.py +++ b/integrations/lingbot_va/tests/test_action.py @@ -17,7 +17,6 @@ import pytest import torch - from lingbot_va.action import ( LingbotVAActionProcessor, LingbotVAActionProcessorConfig, diff --git a/integrations/lingbot_va/tests/test_cfg_cache.py b/integrations/lingbot_va/tests/test_cfg_cache.py index 392ce0562..c6f19dfd6 100644 --- a/integrations/lingbot_va/tests/test_cfg_cache.py +++ b/integrations/lingbot_va/tests/test_cfg_cache.py @@ -15,24 +15,27 @@ """CPU regressions for LingBot-VA conditional and unconditional KV ownership.""" -from types import SimpleNamespace from typing import Any import pytest import torch -from torch import Tensor, nn - from lingbot_va.transformer import ( LingbotVATransformer, LingbotVATransformerCache, LingbotVATransformerConfig, ) +from lingbot_va.transformer.impl.kvcache import VAKVCache +from lingbot_va.transformer.impl.modules import VABlockCache from lingbot_va.transformer.impl.network import ( VideoKV, WanVADiTNetwork, WanVADiTNetworkCache, WanVADiTNetworkConfig, ) +from torch import Tensor, nn + +from flashdreams.core.attention.kvcache import BlockKVCache +from flashdreams.recipes.wan.transformer.impl.modules import CrossAttnCache pytestmark = pytest.mark.ci_cpu @@ -147,19 +150,17 @@ def test_action_block_loop_attends_to_committed_then_current_video_kv() -> None: current_video_v = torch.tensor([[[[4.0] * 12]]]) text_k = torch.zeros(1, 1, 1, 12) text_v = torch.zeros_like(text_k) - cache = WanVADiTNetworkCache( - block_caches=[ - SimpleNamespace( - self_attn=SimpleNamespace( - n_committed_tokens=1, - kv_cache=SimpleNamespace(_k=prior_k, _v=prior_v), - ), - cross_attn=SimpleNamespace( - text=SimpleNamespace(_k=text_k, _v=text_v, _n_cached=1), - ), - ) - ] + block_cache = VABlockCache( + self_attn=VAKVCache( + kv_cache=BlockKVCache.from_tensor(prior_k, prior_v, seq_dim=1), + video_chunk=1, + action_chunk=1, + ), + cross_attn=CrossAttnCache( + text=BlockKVCache.from_tensor(text_k, text_v, seq_dim=1) + ), ) + cache = WanVADiTNetworkCache(block_caches=[block_cache]) captured: dict[str, Tensor] = {} def record_action_inputs( diff --git a/integrations/lingbot_va/tests/test_engine.py b/integrations/lingbot_va/tests/test_engine.py index 7144dd27b..f17925873 100644 --- a/integrations/lingbot_va/tests/test_engine.py +++ b/integrations/lingbot_va/tests/test_engine.py @@ -15,11 +15,11 @@ """CPU tests for LingBot engine configuration and one-run lifecycle.""" +from collections.abc import Callable from pathlib import Path import pytest import torch - from lingbot_va.constants import ROBOTWIN_OBS_CAM_KEYS from lingbot_va.engine import ( LingbotVAEngine, @@ -30,6 +30,8 @@ expected_output_shape, validate_input_images, ) +from lingbot_va.scheduler import LingbotVAFlowMatchSchedulerConfig +from lingbot_va.transformer import LingbotVATransformerConfig pytestmark = pytest.mark.ci_cpu @@ -69,19 +71,24 @@ def test_pipeline_config_applies_every_model_override(tmp_path: Path) -> None: resolved = build_pipeline_config(config, tmp_path) + transformer = resolved.diffusion_model.transformer + scheduler = resolved.diffusion_model.scheduler + assert resolved.checkpoint_root == str(tmp_path) assert resolved.enable_sync_and_profile is False assert resolved.diffusion_model.seed == 17 - assert resolved.diffusion_model.transformer.checkpoint_root == str(tmp_path) - assert resolved.diffusion_model.transformer.compile_network is False - assert resolved.diffusion_model.transformer.guidance_scale == 2.5 - assert resolved.diffusion_model.transformer.action_guidance_scale == 1.5 - assert resolved.diffusion_model.scheduler.num_inference_steps == 7 - assert resolved.diffusion_model.scheduler.shift == 4.0 + assert isinstance(transformer, LingbotVATransformerConfig) + assert transformer.checkpoint_root == str(tmp_path) + assert transformer.compile_network is False + assert transformer.guidance_scale == 2.5 + assert transformer.action_guidance_scale == 1.5 + assert isinstance(scheduler, LingbotVAFlowMatchSchedulerConfig) + assert scheduler.num_inference_steps == 7 + assert scheduler.shift == 4.0 assert resolved.action_scheduler.num_inference_steps == 9 assert resolved.action_scheduler.shift == 2.0 assert resolved.attn_window == 72 - assert resolved.diffusion_model.transformer.attn_window == 72 + assert transformer.attn_window == 72 def test_engine_is_one_run_and_close_is_idempotent() -> None: @@ -133,17 +140,20 @@ def test_expected_shape_scales_only_with_chunk_count() -> None: @pytest.mark.parametrize( - ("changes", "message"), + ("config_factory", "message"), [ - ({"num_chunks": 0}, "num_chunks"), - ({"video_inference_steps": 0}, "step counts"), - ({"video_snr_shift": 0.0}, "SNR shifts"), - ({"guidance_scale": -1.0}, "guidance scales"), + (lambda: LingbotVAEngineConfig(num_chunks=0), "num_chunks"), + ( + lambda: LingbotVAEngineConfig(video_inference_steps=0), + "step counts", + ), + (lambda: LingbotVAEngineConfig(video_snr_shift=0.0), "SNR shifts"), + (lambda: LingbotVAEngineConfig(guidance_scale=-1.0), "guidance scales"), ], ) def test_engine_config_rejects_invalid_values( - changes: dict[str, int | float], + config_factory: Callable[[], LingbotVAEngineConfig], message: str, ) -> None: with pytest.raises(ValueError, match=message): - LingbotVAEngineConfig(**changes) # type: ignore[arg-type] + config_factory() diff --git a/integrations/lingbot_va/tests/test_loaders.py b/integrations/lingbot_va/tests/test_loaders.py index e0884e017..8c779faf9 100644 --- a/integrations/lingbot_va/tests/test_loaders.py +++ b/integrations/lingbot_va/tests/test_loaders.py @@ -19,9 +19,8 @@ from typing import Any import huggingface_hub -import pytest - import lingbot_va._loaders as loaders +import pytest pytestmark = pytest.mark.ci_cpu diff --git a/integrations/lingbot_va/tools/compare_upstream.py b/integrations/lingbot_va/tools/compare_upstream.py index ce19a1149..a41662250 100644 --- a/integrations/lingbot_va/tools/compare_upstream.py +++ b/integrations/lingbot_va/tools/compare_upstream.py @@ -35,24 +35,23 @@ import torch from einops import rearrange -from torch import Tensor - from lingbot_va.constants import ( ROBOTWIN_ACTION_DIM, ROBOTWIN_ACTION_PER_FRAME, + ROBOTWIN_ACTION_TOKEN_PER_CHUNK, ROBOTWIN_ATTENTION_WINDOW, ROBOTWIN_FRAME_CHUNK_SIZE, ROBOTWIN_LATENT_CHANNELS, ROBOTWIN_LATENT_HEIGHT, ROBOTWIN_LATENT_TOKEN_PER_CHUNK, ROBOTWIN_LATENT_WIDTH, - ROBOTWIN_ACTION_TOKEN_PER_CHUNK, ) from lingbot_va.transformer import ( LingbotVATransformer, LingbotVATransformerConfig, ) from lingbot_va.utils import get_mesh_id +from torch import Tensor @dataclass(frozen=True, slots=True) diff --git a/integrations_v2/lingbot_va/lingbot_va_v2/app.py b/integrations_v2/lingbot_va/lingbot_va_v2/app.py index c27b66a9d..258fca343 100644 --- a/integrations_v2/lingbot_va/lingbot_va_v2/app.py +++ b/integrations_v2/lingbot_va/lingbot_va_v2/app.py @@ -23,21 +23,6 @@ from pathlib import Path from typing import Protocol -from flashdreams.api_v2.application import IApplication -from flashdreams.api_v2.loop import IModelLoop -from flashdreams.api_v2.session import ISession -from flashdreams.runtime_v2.session_desc import ( - BackpressureMode, - PresentationMode, - SessionDesc, -) -from flashdreams.runtime_v2.step_result import StepResult -from flashdreams.runtime_v2.tensor_artifact import ( - TensorArtifactOutput, - TensorArtifactSchema, -) -from flashdreams.runtime_v2.user_input_events import UserInputEvents -from flashdreams.runtime_v2.video_tensor import VideoTensorLayout from lingbot_va._loaders import validate_checkpoint_root from lingbot_va.constants import ( DEFAULT_CHECKPOINT_ROOT, @@ -66,6 +51,22 @@ ) from lingbot_va.utils import resolve_prompt +from flashdreams.api_v2.application import IApplication +from flashdreams.api_v2.loop import IModelLoop +from flashdreams.api_v2.session import ISession +from flashdreams.runtime_v2.session_desc import ( + BackpressureMode, + PresentationMode, + SessionDesc, +) +from flashdreams.runtime_v2.step_result import StepResult +from flashdreams.runtime_v2.tensor_artifact import ( + TensorArtifactOutput, + TensorArtifactSchema, +) +from flashdreams.runtime_v2.user_input_events import UserInputEvents +from flashdreams.runtime_v2.video_tensor import VideoTensorLayout + _FRAMES_PER_SECOND = 10 """Native Robotwin video playback rate.""" diff --git a/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py b/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py index d0f7e677a..c4ed7ab87 100644 --- a/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py +++ b/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py @@ -24,6 +24,13 @@ import numpy as np import pytest import torch +from lingbot_va.constants import ROBOTWIN_OBS_CAM_KEYS +from lingbot_va.engine import LingbotVAEngineConfig, LingbotVAEngineOutput +from lingbot_va_v2.app import ( + ACTIONS_SCHEMA, + LingbotVAApplication, + create_app, +) from flashdreams.api_v2.application import IApplication from flashdreams.api_v2.client_window import IClientWindow @@ -39,13 +46,6 @@ ) from flashdreams.runtime_v2.user_input_events import UserInputEvents from flashdreams.runtime_v2.video_tensor import VideoTensorLayout -from lingbot_va.constants import ROBOTWIN_OBS_CAM_KEYS -from lingbot_va.engine import LingbotVAEngineConfig, LingbotVAEngineOutput -from lingbot_va_v2.app import ( - ACTIONS_SCHEMA, - LingbotVAApplication, - create_app, -) pytestmark = pytest.mark.ci_cpu diff --git a/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_real_model.py b/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_real_model.py index c3a7ead01..990581cb9 100644 --- a/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_real_model.py +++ b/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_real_model.py @@ -37,6 +37,13 @@ import numpy as np import pytest import torch +from lingbot_va.constants import DEFAULT_CHECKPOINT_ROOT +from lingbot_va.engine import ( + LingbotVAEngine, + LingbotVAEngineConfig, + LingbotVAEngineState, +) +from lingbot_va_v2.app import LingbotVAApplication from flashdreams.runtime_v2.application_runner import ApplicationRunner from flashdreams.runtime_v2.metrics_output_sink import MetricsOutputSink @@ -45,13 +52,6 @@ TensorArtifactOutputSink, ) from flashdreams.t2v_v2.testing import real_model_run_skip_reason -from lingbot_va.constants import DEFAULT_CHECKPOINT_ROOT -from lingbot_va.engine import ( - LingbotVAEngine, - LingbotVAEngineConfig, - LingbotVAEngineState, -) -from lingbot_va_v2.app import LingbotVAApplication pytestmark = pytest.mark.ci_gpu From 2bc449f8ad330e628187b0444698f9434eb5184f Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Tue, 25 Aug 2026 22:04:22 -0700 Subject: [PATCH 11/18] Refine stacked LingBot V2 integration Signed-off-by: Jonathan McCaffrey --- .../lingbot-va/robotwin/README.md | 17 -- .../robotwin/observation.images.cam_high.png | Bin 50260 -> 0 bytes .../observation.images.cam_left_wrist.png | Bin 24723 -> 0 bytes .../observation.images.cam_right_wrist.png | Bin 50408 -> 0 bytes integrations/lingbot_va/DESIGN.md | 96 ------- integrations/lingbot_va/GPU_EVIDENCE.md | 99 ------- integrations/lingbot_va/README.md | 243 +++++++++++++----- integrations/lingbot_va/lingbot_va/config.py | 3 - .../lingbot_va/lingbot_va/constants.py | 4 - integrations/lingbot_va/lingbot_va/engine.py | 3 +- integrations/lingbot_va/tests/test_engine.py | 22 +- integrations_v2/README.md | 1 + integrations_v2/lingbot_va/README.md | 16 +- .../lingbot_va/lingbot_va_v2/app.py | 18 +- .../lingbot_va_v2/tests/test_app.py | 28 +- .../lingbot_va_v2/tests/test_real_model.py | 24 +- 16 files changed, 251 insertions(+), 323 deletions(-) delete mode 100644 assets/example_data/lingbot-va/robotwin/README.md delete mode 100644 assets/example_data/lingbot-va/robotwin/observation.images.cam_high.png delete mode 100644 assets/example_data/lingbot-va/robotwin/observation.images.cam_left_wrist.png delete mode 100644 assets/example_data/lingbot-va/robotwin/observation.images.cam_right_wrist.png delete mode 100644 integrations/lingbot_va/DESIGN.md delete mode 100644 integrations/lingbot_va/GPU_EVIDENCE.md diff --git a/assets/example_data/lingbot-va/robotwin/README.md b/assets/example_data/lingbot-va/robotwin/README.md deleted file mode 100644 index 3d356077c..000000000 --- a/assets/example_data/lingbot-va/robotwin/README.md +++ /dev/null @@ -1,17 +0,0 @@ - - -# Robotwin I2AV example inputs - -These three PNGs are copied without modification from -`robbyant/lingbot-va@7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb`, where they are -published under Apache-2.0 in `example/robotwin/`. Their upstream introduction -commit is `5ed0eb32046b34fe5c14f929d81e87ab6ebe02ef`. - -| file | SHA-256 | -| --- | --- | -| `observation.images.cam_high.png` | `78cab76d394114ba912f882ac9b00ddc017f98c946482cb9267c87de82486b72` | -| `observation.images.cam_left_wrist.png` | `fbe55b713e1b3d4505fda6be3b00132213ee1725a78cb357ed2bbd5b3a3a7a93` | -| `observation.images.cam_right_wrist.png` | `b9e6821b38073567232f6dd8f6389b123d09d3d1a70758c27d388e547e228249` | diff --git a/assets/example_data/lingbot-va/robotwin/observation.images.cam_high.png b/assets/example_data/lingbot-va/robotwin/observation.images.cam_high.png deleted file mode 100644 index 7546ec3aa2064e4e32d03250439bff582b9d6042..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 50260 zcmXtf3p~^N|9^FyW2@ABiloG5)7%m#gmRg=Y{bZh2`9NuI7O6er{*%F+$E$FW+~xXzuk-sqn$gVmdB0!J*Yov!zTU4fNLFTIyQO#U*s(*5V2-!l zv16wp`1v2$E^xn@g)Q8%Lv9BFZ)A6UXUhx5edkdlOuN8Z?7qrtEzjLZWc%#4(m8YM zKcF}5A2SMUeKNQ+n<06Dvqd(xZpJHSW$CY7H{VJXNXz`LcgZ|Wk$MrqP1fj$tF|p5 zCj15QE7h5CXgU=?aLQPfDVmtn+_tDWx1D)?$FwcgJWzJ zd_=A|`F2nuNZm?tDyt{FBZ~^Br(K{XzXTo^7^ptzi(8M#Ena#+362s(? zG?6PF&*nLDCW$J#R6(Z%K0 zgkz8yCP@x>tc-x;D3(~s!eP!MWIqsfO(V>MOCTD>1TNU^hw*EjIE#r)B0u!`Sf-+ znGeaBXr?)M+)uJ(IO$TaIWrv5=Ql1cUA$s3g*jnDqu^;2W;LRpif6JLRnWpc znX~^eUJu4y{!QK2Y%uB3^r>^fx6l>zN{)#WL_|II(V)fGLRCQz(J&1C8AP0Vg|ani zu@%FBTiP>QW(Mi}$!JTM5nCp`%FAmgHz#~ytfuh2)zE{y+`NFwOLBYi&*_~S+A53c zV)yRlqin;(1eqizp#{e=F;kSGdl%&rKX}H|=#U=yk&Gwnz6j6JGfr9ah$`xo+Gw|n(@K^@1;?e!avA=g?(mpao z=4|F_qkGtr;`o!VQ0f+4W7n4_xJJ~y$-RgO{Mh2xfczE#J>!q92!-~datA&b;Ls*MDGLlA1faCKmqV0y##A+S}r%q)_8XIHgdTY zlIj*oaYu+8MP!%CDci)4MN|xxfo0jsbnkog__hyZ38yjNH<;p8hIo+Xuq?s8#c?AXS}(`nU7BnHJOT88*|hGPj%)y?J* z@j`s9lwXCWZ%6e(emh$dTwXu;9jZ&Is{}tu9Kd5F#Z&KLJg*c;(X_fa-qYO$+}t>a zv{lX;$AO$;%AwMgSdc1)ZmS?(+z#X6Jz_}3^k)V16ida6=e@i}13uT@SXmwnUYwbm zyB)T2_wJp?ik#*Z(*zP_})RCPD+Oz6hfSCjUm*$b^wm-I|UAwt)fclOZcOSM*7 zp>cOQ1Ofm!F^z=dHT=Pb`1hhSGALLgb|MB1-L}h;O8=-#Wm4QEi6z7mDg(;IIk7Em z+4K~eZ;v|2-bQvKJD4a+EA$kYw-C|h_;#%hVhgPs09T<%goWivnmDoO#OGzDir`xb zAfNluN*&Z!Dix6XWAH+P-M^TOdn@ovY-|fylImF;Wz0cdZ9k(61@NQR2$Hdy1rSaE zWc&&(DWmb@CL}n|sFhRVw@L*J<(`e$ni}uy%nWe3mJqxDSmci%hYUhjzqiem=UO89 z{MH|xqwkNMy>azw=-TpNU!NymWaOn;&r9FvjRE)RDC5)K^xVNt#5MxU|tY z$3&SThOvVu5>$vJH%d2&T|hTyfG0r{G1ayVC6@HKWw%nwY7&>VOeD4tRpRgc3bNrA zshEVhf~063k>-{vv+$3&k}L{KvYhM4riknLIUiy|=u$lP$~iKla(Ut|45V#W37v+$ z=uMZ4+m9`eJ2DaTk?Dx)j=Klx3F0UqBz86;Ei6c)q5J99D!aWE70xB)6r1=YxHd=0XgjmHwl?!3aE zgR@H0>^f{SiD@7AXcjw8+g*}8gyYc)AIsqgWC2F9fUaS~aQr_E!7*+&C`oc6nNbaa zOznxL5j491#vq~!paCC(7|w|ihi8!C7)kK5q!A*A4)Np2jVdhs)GIE%C%szX@Z8MQ z#@D{_-l52a#qovKVB=$v^Z&e4tKZXg?>iH=HD8nZ+S=OMwKk%v z&%aXUl3xEE^^@n23`fuUA7%9jB5pdp`d~a%Ol>T{Pp8qjEiE`WSJ#*~vY$!z71yqi z6yMe_hY3K5;r@O|BOk9qWpW)cMO+gstC20q=qaJ!PZx$>mSo~|rNEy_hF^0+2ru6c zFkm`XojHJWR#c?&AttE%7zzgDHTy(bJJ1ghk)w=EBf)XXHee&l;~iTdfPO5C4rQXG zXSK2cF1M$%10Fnh;IEZ!dE?LR7nxD>$6X`0ehlh`+}*b?`DM}4^58&fVd33l29aCy z{|@l^(^C1Dg1pkKIuB+&PQyp8uWTEN-^YW>F=ET5n{vqBaEvFbF`WdW)rImDrz_FK zjc`sAcYj{x72`%LiEB<+ngo$!z`^^f9ZZIpwCOG-PnN!1mz53sT0x^GP)s<61xqqH zuA0Sr23YRID;I3J66+C{R!HwxWHqXES@FP1OS`yu6mBY7I>x94QfK7SE@)>pqHM)@ zm7a|2t|};uoqR=U+P!Fz?BRd2Gl$NEF0M8=*YNq4onUo!s~Zby`s-6$wd=8>qMd0z zx~GS7>aWku&!rv*Ozm195mwoES@*ph!+5j9^^za49eS7!OcFlnpuL=O2= z`-h%r(k+ql^KNBWIN9&jwW3<6QHfghiVdnlH;P4dlbkRfA*#BOfqEf@neb3?${76< zUFT%z+!Z<$&oWiTF;+0B#&aG0OcT|SbtTPWwscWHV>OAyr$%R(<1Z*8&9UWJDn`<1 zzsu;;@=X|>}9J-W(AAe+gEP^J^0#xIi32CI>iV8uTVgAQJzd?!cd+7q&z%$3$Eb2J&p%5SNN*Mq!FCf+S*d~>-Sc95A-9}mj_?G zcrjQLVi2}EetiGTP)*p%KYyUgTyppu7N3F&fa_tIDIDwZqRxRFQh_xoS~RoNDk zaGHBjWIp8r48y`dFT|_jY#15s6cF2=msUY1R^Ti^txSOdrlC=QTz4ZEllUnlKG-J5 zViEN{m|#E!U4J^^vKk8dIHnwNBt#P<@qN*{%G{ zvO2vR8yl|{DnstfhF{%&QJ8mT<7?`AY;3IVgZk@BV>M>St*wW0@^S;JfQ+xWwAYXD zSaIL}{e|QIeE;X4vF~GJGgl(FnXWyu5EJbQ1P_*lSH5E0LFrzx!3`p|o2ggMx!H_F z@N#>UPP~!@yosUswSw$?ETPBJFQLo%bMD5QOsA7Z!a(MkrIWf)REFd8~LvQh_Zx5=fTU3C($D}!Q#|oe-08cE-(+`j) z25^#4()Sc(kVb4txNa8!rh6vXqWfZ??nXmgPo9Yi3h-?5 z2x8NMQ>8N5{_$zvCLFv|F~r!za12-bsaAj@W00Ng6CKip+?(c*_E4s6pm+AOzP`?l z@cGs0>=02i(UqCcn^VJH!iBGX9~lm+W1O96Kn}-KbzUaWq&0#Jn$Il3IO`$)+8*b0eGhiifcpxf)Nw z19%HAp(P&D!{|ShEI+)0G$$s63N~Rh_W4-~gm3+A#OIWsp)xaJOV>3M&|u*8YLm zLJeb7`?E{!+{%K998lC81OV|kGQu+Gq$w~bXK#eg1K9@vs_&tzufH|bX)R39;o%p+ za9TZha3*SNqO~T(YoOIruX4ao58}62it=CH{_^EZO~~x$s6V%vYIQ9OvvtL_ABaH+ zra3&F1my4~Nd|@E$fzaZTB$6E|1ld_(>N1MQHuE#9c9PqJ6UYg>p?S{LU)zGB#5|Z zlwUNl0?68YNlnbieRrA}HW~GSUFrijnXY&=fpZeSV$5+=W_bgD0vBM%IdSF}U`sm* z-W(HOwGPOq(W>H6Of5c}#He&jmh<BLXggHlOaVryGuP9bn?k#SI zl#JXY4M^T}3L8^_o5U5>GHj`%0}H90UiIsnvvm=R)1LgZ`e%SJ9KL%jQsbPcnrqz| zowE^}6H7mW!z#1<%*+NX;hkyjy1J*&oY6f~7dSeTTa%|BzIw=Dt2n1VVxR&N6Z*SE zp#33Q55!0N$x}OsQ{+^V?QJ{x?t9oGBkxDjED^$m-2!h6l^PEO9FRJ9nRfCAc_zvi z%}a=!72p{%iMZB@`@j=I7;=Lg`7tQkQpAM#Two?LNNkgIIV3C*{D~@%7@Eoe)p9t7 zB>5IT-GL4@x`5q*li3n!-XJD!z#=JtBx8BnuoR6zjaDyKf-qP(0MJx4*s6Hw0LT(I zvRk2ujC@53BSrl;m%8vD14G-(z4bqO52?9WJr-S=jhg?S${1C1ITN)!H4XH!UYl07 zdjFk_dnrEm_dULypZ_vH|4ZL((Utm*dF#meUW3+;{;^~>zrBe||4;O!orHO~A(+)g z^k+rV2QZX$pm8W*3viP}h6#em)_moC!0$2kB7*3BghtVJPf6)P`6=R5-{Rq#d)Ppd zdbruKHBI38kEH<8(!IgO13;}*FnU~4DC~4jE(Oav0cZsf&oY$>G38PZ>{UClqERIcuWaKc{u>KiM!GjfX&EUg2S=D6$j>T84M;JvK$po-<+fWpU?b4(C?$WAJm_6Y9{RBU5O0iK4?aMZkYU-Ony zHw`GZaEO>@w+Ee0gUq!Fho660ze$nwDx zIL$LC^rgtG|`TriL%R*w#TUg>8UdE%$(r}#{g$8TGS6e7niW_z3Z9v(WtHN z`i*z>&m%Y2zJLGz;`aLaeE=v|)~5Ofs%D#jy-%_13<|2Q1}QN-+_4En=f?Z`jUPiB z8%(ur6~E~&b{KFvKs<=k#Nj|xSTvpko5IO`mIf2-1{~qcLD_j*3?3{>iK{sGL5n5H zAQ6cbSiyNW30o$hE~Y~lN*~ZhT3Q8#Wzt;Y2g&vjIS}6I77(1YjFk2ip{dSKBMqGg zi_d-2QqqXpJ5mXt4gNAVy;?Pa9S@zyf!~3DSR1=|IG~mzG;uv~B0atbg_2}Rs`5k_ zAH%|yXSC}#TkW@3t&dyhnFwf|&tGtoooHaD&P=!1Q<6 zOb}s-aR9pDaR*QgES%wDMnx82!D=Uj-H}OZ#YBu8%9go$6XKB|o}A>hB8t%kMV6-~ z4ATwck#3QIj{uoZ7!eh2;qIlE6%zLw3xEOxLWn`}PBzDa9adtMq9wr|928G+Xryt( ziH(ha;Isx>wOykYy6v|&ma1F>w6X!G167)rmj}Efpnf<0V5MjB(XW33p3^<^0Ki&C z21vN=vdE3hsP!taT&5aWcxTUk8(pD3HaOuA%z8A$B(}Jz1?>fPjcj8-@{9z?3J%h- zx63c99e4;aRrS6-!u2Nf6hNk`6VLEKk7@&$0&l{&3yePm>_ezE{F;Rg3Y4+9?b!eG zLPPc-Ep0K6NmXgpUAFKm=MYQ{hNB9uNC~J=i4^tF(6uSo?WNxBwSl4MHx|DABUF#a zE(zJHdAxIhzW^!4b_QL)emy)qC_H?0b!=>Pwe7C{_SEdwa{b2FvaR{8&GpV*8TZGk!!nNLVz!SY2tvaZUDhJaad(D2{+#1?iDT`VvE-ZYnTPX?*(W%|DC{Gz^ho{AEBsw`D$8nCHoaOq=m z!chGus(yYG-U(wZC2V>C|GReczu%9zWwL3NA6;;fG0Aa6@ZojCaJQl2rm@pEuFiLtZT%Q98+uS5W)JE~Rlv~M8}nbgC+#oQr1u2{1ziVm8*Q7* z{qp5y{+IUve*uxp{Q9jv*Ql%8sXq5D&-bsbV4FCkS;aYVmgL;dYZ&nE+9CpM2T_GB zaY)Gj5Dz#Nb>*mFpw03OIt0QJLj@%Zd~Ih&uWg&C zOb|Ioa6B$LE{zcDn68x7g7bjNcMIbC9xYAv@m;sqdR?Q|r?a)ow6hn`1Fg$n$D5gl z)U>nDrQ9C>XAFq@+%LI$D>Fmu@9I~XdD|NS+l$uQi>s?~&1k<(v#zo4c^tc`Mxoqw zNC%|}SgL6|(0gG%l;n|^lvRVN7mSPxy%#YbBmD9Ec6i0H3sOp6NMBgm-nN~lh4ejf zPBVebQ+C85fB$@T6Qu!@Ga}`XKTto{i2J2_Gsv9Vx}1Dy0^4Q7$tR6i01;gKu+ahp z=0S0pq;}eBDK0`FER&#xR+mUJW7TDx=GF#R>UuG1`z_u$U{XFZ4@KYOA3Ag>YV~FP zU~S0C+?Ut(_L<#--7{bA8r)d@{-WDoTcdJddc3*0udi>suX()r$3Q^+QZd+@*{J0O z?a$$XmwyA$^CTWkZ`YaJPH88Cw^vlW0`lu8MqHpSBOb9rLYj*w9>ps!vpcyil4h7L zn{el)TaeoBYX{4O-4!BsF3OQnMjGKXuBiEAJ=*%71_ie%b_vu?r3U;^m(0EQzNcSb z9Ba5tV=CUa8IifR0w>xt; zrp^wIabeMQcQ5S*C=;UPXI_Ag6*1g-&XRK*JU03hfy7V$mp^b!-Cr7K4uJi5*sqg$ zqpPh)@X;cnH8>%;JoA_L2`z(HuUwf6*xs6zja+^4<+!z4*@*_q+;Q70il~r4t)%&(K*RW{;2-ETUwO49U8=u3xMn?}D3-JAG z$X1;KM4uLA!GpnA{y(?NFY$oLIR}bzS9`h2zlp!B5>j$+V*^rdQV2^`?c5mCWqGjClEA8`P zb7E+;LO!i5xt}?zoB6b#moa$ZF@kCWKiA}0bM~x`%)q(jnC`!8K1F;l+`s)}dwtZP zZhPzY`(tgjCM|8ngZ_>gZ3=ytwMV_Y3cW@F)ozW01jruRdKorb7ZMt*V5I!U+{xgP zPGSASE*FL`4@hNjjgYWVTX1gssJUdER=VAvoy$rNb>xs`?4GME<7A?R>D_JfACY80 z+7Z%UaQ~!H)^OhWbSKoYF`1*;oFk|RnS#-2hW_G+Sr&{Y_0md1<>=O z*mZNh6@dP-wnYD1L1m<2E`O^e;hyFP$06qKpFuYR-9X!J5<@ceW3fF9P@fvN7PPJ# zxsUJ@RM<6R#Dp)L{JvXkZZj@FG+M5lHW_o8o0-1aL<~P-grJcD;}}&4v_~%l7lZxF z^2EkdyJ?rT3Xcy=F9-1lJ(JWkZ>+q$yEUG-)eADLap98jswWC&2>-F@`zBiY_$O|!YS>Te==mNdJX4Ms9j}p+p36K!4LYD;S z@4f>x;|fBbmiXjPk4WuXu<|CU8AWxH;_bPA|F+jO#oN@#+8 z=yCmz`6S{iQ`*-2H9#_+RuUemH`*|KF%_I6!dU3WXLj3ExU@s8W$7X^Y3>QyHt`5e zLYjpY?m9%=h)!D#bekJ%Fa7_s0A()7x(r=zEnuZ7Ob1vOsaQAu_9Nxo8jq?0=jqaK z7utyj3nbiY4ucvLTq83e_Ty`+_11XQ*8A;M15km+=KMpRIt6#0@g6OC`|)xUs8T=r zek_0eI$briwK*HP0<=2-)F-KnH1~vIIloQ8#9y;)J8&fV(=0+l>|!UvdUuo6_Dqbn-`+mF!6?JHHCS>;cX4|s6FCdFfE$+3pwH{hJ^6kFUfHCV4yW z!PLoh2|`bR-Z9bp5l~EE5p19hfRL(iQ&T5s;*>!viTJz()UCm*f|?7#do-IK9t@oM z_WaqWnrHof+5_63uU~S^FxVL0zr8%Wz4d)KzxZET6*V}p`zaBnbtxz3!DroiL7)Fn z=p+xc8GxI3H8iAOpRPWEO%WIMnJjZC&aGp{s|lO7Le|g)^`GdCap~#->{3V6Y@tR! zq(g7brwZtyrHwZ5YsOfxAi_X9?C0~q%Dgoy@Lc-Z1qN9;t=(L61aQBZpUkg}QDV3r zFPoSIx-8x@%0q*%VYfY71=YOMcFJ$KA z=j|`+pzl@l?D_NiHQgGSI+-3B@qZ6K$ax^V$gFcEWLC6967zzD#fU6ihED)>~Vb> zTv>NW!qozf;jG$%y+2Ck0mH+q%c5b6OzD7`$}>aLNow&uE{HIUfE$1UB7p~Zm&@kh zaqtnUH^Z0Ft-`B~`&e9SClC5EY2a_@kZ3ur*ad+GmBP07F^zGAs%u}PGx{X^oqyHT zF@w-~aJh$MT`yb+pa1@1^Ic)4#em11*6M4M%fU53NMeJ05s9IO zu4+kzIZK&Gs&h|Cpzhi*%G#44>0VIiC~UeT3r(?^m{ewXreJ-PJCrZVK+>ckLpGftj;*HJ#IR>U-?@Opsqd;mC%i`;g_+(#eAx% z-u~wWIHb(*7|-D=XM^@x)urD-fvl?OR}DPzuFK=ZyBu#lpf{R7k52|X-vZpbR#m?_ zJy}+--t;Ce-@Wlg4&SeT&9_%)GR)41ap-5a`%ztDkA;wc1t}adx6Te*Zf{m@FN)kp zy>}qJ_atJ7bAYo>=2CSnh&ko*EpEAi#mY`1OfOW7y z)zTZ!S=!`2gq8}3<`K$da5D1|oF4GVaF6O9XvuQ=m2wx5OQ&H*p!#uYdji!zP`b1c ztrgtbHb>pjsZYMG8Ew-0M&y>kIua7x6Ot3qPv+THhpFH&j5IHzrt=533g=MFJJD1URBXaV|!EKI&{6_b;G)?K%Nltfsq)A(IuWLQ04W#n#u{x;^zit z9SDr498K^J$~gUICs>pCrM()aQ_=kvrf#4kiXM>zy??0q`iZ%2T5#z}Uo?0jhPmE= z&S>XxYgzs9Kf$TRe*KN_BO_m4-<6#X-s22p9YO2T`ENNuitz*6?(;lO1X?8|Pk#ig zmNWEh>2ugDs5SN5nWq>8^5A5!vJURRY2ptfr zZ%zRlBZulY1_Pqjdz=F9;>~rB8$nli-GB=OlQ~Pjz?8c705ZZUU*Umn^*&(Rv_T2n zhkO1|IA(_D*}m^8j&?ikeStaZSyT%n6s_%M-GR;(kAW!Vn!U^MD;&e8#w8Z6T0 zDV6&)Vu&RW{wVs61{;lRHrFuKFpcrjz{inGa3dHKiQJ$zkF1)(ZhuEzi>AL1he6Zb z{wu3&5DflV2b!gf z_SU@nxB1^x(Y(*~D|4>~_s>R#Zg;-6KK9-tCWRhqiS$>slW;nwz(|;bpMd92K^?)LV6YK>b#wI{o_NigQj;OW#zMUc!i+Ody=^ z;3AA7WLV;6OHV@;8u!HO#*<5EWfK-29I`y${%PAsQG_y5SJ3%ALfR6SQU#)mj=JfkIfsbsa_q+ zzY8*J<)0U~i;HD2a8+^Fc8aEk)@`5^Kt0YV5cT#p217PEIhWrDR;ErvX3R|G^j2^aB8K^pa1!M6)zIq!0El?%XNtI|Xuh9XYH7RB^%Rc|%^ z&Z79)2$U{DVQ|zNLjKebXa(VZg47t{MXKJIiUx3ry@A>=nS64gNGp+L~JB^^Q zZgYlLyrlTy!~BJ-eX2EsIfHzf4I{a{?{b@!_4~H>pjV+3*hc`}E#I=i$WhQ{iCWAL z`Fvwk(Cez#_rSddeyxk>6!p8V*XFNXN-jV>2U~J~e0GZNdl;mx3lpe$8 zxhHf`zk#9)x-~k20>7!mWOc3M#|QYpho*LxA6_t2Zu$sxYu#Mak&wus(LveG)m*)R zp~&fF&l(v*M_R>_c7Z5;?XQ9d!mhk4_@d%4KgFwdS|b-2PDWCrOOO&bnR}a5{AhV1 zeL@1~Y%$`YN>eN6R%pghjD1FPx>PjJX)+yKbOe0zd9t96v-}~wwp|KGaOyGwJ%C~p zQZG&I5}x;smR0jKY;A13aK)guYHf97_;#$l{T}iyxvq0$oO3Y+@~O#w=i=p6oU1DT zM(R?{AUK)>n!D*Pf#)llf{V13w1tzJOgk8eMI{4l>9qu{sM>Vw($ zEho|(%>97gkC>gj(3OH7P@^DJxWod@y<@E&uVdbGP20RVNyjtnr=fBi9!TxgqH zurJ=0A1K-VOZP7=4BMdHd~8PwUZ#Q$`f0+A@xK>1b3#q_Y$On)x0%-Zb=B3^!DvtM zyW$Kkd+CB(kJ5wU+ih&#$yS}3AeTv+XBUQHkDbGSgCpLE+#Ona zsH3k$v7!qP%slPG5rl3IKbZxE0qom((;Y?q1f1JC$2IlTy?ngvx&3`?>ice+Ne&%6MZq0>FSK9lP< zBWnI_kEC6Np7IlEL_&D?)La7Ox0_m;xhBxw7J7$^0#q0fHyOC*JH!&VvVAC! zMxD%f3azP~5&Af=2h^~$C(rRyOWP^5j0|=K2;M6fxGKwENhz!5#`f^${;kDAaQ0|0 zSRb`LLF*+x=UhWnTx-{2ycd`Z=oHclAE^X7e$(*Ee0mS>o^V+!o6qMjEI1uMemqdv z*z4EPK+7jP2vU;pXq~eOl&d^_xf{LQ zfbOEvIArJI+P81I{X;;{S9W_&RyO>5;qkl2-a97K9mr=7a47;MBMWZm$y^BXY~v~h1BlsS;=p7{gl(W zUvZy;urCC5ofKt-nfMjZQ#7cGSe&ym@e!VBJCbbm8zr3Tz ziz@^)U}zW=c5z>uPV3Ww);6*|*Pl}hGPsN`mvUjc^u&o1TdP~^QEQ!|c@fKhRyn00 z#C>W@h~xu!Cp$7wDqT9hnn3YAaiZ{Zb>s$GzfDao3VK}ti?*~4`u9?j)t}&&5j+tc zsgz_OsY65W|F_xSZNZtm&?r~48he4zw}%Oot{96GIiQzrQv1+Fb$Q}fLU<{_)R|0g z9<_`@kiX?1r~~FdqLeol)CC;P(;(`%(!k(gt`3G8rq0Gsnb5hW^qx^L*K=D~DFpnVrcuvkq!XN@3HR0?F4@E5V&*w=LTlk_~3!R0ES$$E#S|5ar?Lw1R=qz&T0A(@O_Oq=b9 zuf+%xP>gB^%>uOP? z$8*1MPb(G00LM3on26DUTpu^uOg>y5$%?D(67>5P7sR(qP=hLI!PNA7fx4W&BG2P2 ztkD}FAoeDx<0-fHf}RbBOz8eJIrRj5bYSkK=>Az3;BePFUV+)T8yoIzA7F$e5ueG2 zVU&x(lE4g?1_x#Zx;Q(#@Pe5$iLK_s=EM5(G&rJZl`04%nwpxTV5Ud$2vPMXRehJ6 zJ|=*pvqNTqRzpU>#R7BwMbvKYLPY7k#$MYn`8k@$NqD#2C zzy}VziYKV>M9?9vm2;OfPEdaXoP-D*v`~`S^kxpZm{O2SYkN9S6S~^xwIUn7HXJJ( z@qIXUWp-%x@*63~8w%bX6g_5T|NUIrcup?xA2owBfMZVZl3Gs-^R4Du=+%@*d{l1} z41Nnu!MLR-b-(dUNbde`hYRX>Nn9Y@B<5a%9R@wcBg7Gl3QjFF@vwkK?x%I=Sd6NC z%NfhLn9-G1aZQ!g97j%3c&Q|=p7K^o%4iwP(jHC&!v%KW3{Qfp>E)$Y12XS{{*BP> z7XZPlo!dXY$j(NE*7Gm9|4S9`CkAOcIk&P+l(KmD!8wN@(CU6dt0#Z(vIb2HkX*F>%TV-0kF6gWv~15IvVym{OiB&}P)+&0mhEaAwIJM{#2=O~@B+@6s{N24wY zBdy({6X8?@=H*7WqL%+zdRNMS-nq0HHcBnF;@T22Jih#4#rg~J8)5&L`|S31=dr_v zmv%|v2KN1vYNu;)Eob5BN=|Wxoswe;=F-^Cv1hk7vIvH-H^#f>ucrK|ASeH;`=gr= z9WYA>>VsV(X~{^0X`-o>CS>kw_-;2rjxKp@w*?%`Pg|%Sr2l@=WGBYxS9byo_6Chr zwJWzfz4(%}Id`Q#>hCDf1F1ct9WXUL_%v)~b?hT1qXE0~u;Tt`LXr(^C*~qbgq{Gq zs0c?UG{__AHgP59DJQbjzS-P}4&=tGDZL^hAS*@qyw^b^SRwu9u3vW|pWj5ohT*$* zqvTB{;5Spg-`nBi;{zvHc>5Sy>_J~lqyG+1d6@J^bR5CZ5DBNpA< zZ~YpVa??;k@Th>2WeBApaJ$fG)x@NXle^6=enHC7&%Z6YiAah~6Y08%K%|_6$H8_c z7Z@5D9+Q_Vq)VpVe29UQG0U6nE1Mm2!@Plz(un!-=i63-bnfc+w7a z#IfDxhUN{}Gih-1|A`3=f7DnG&)Lg{G3JvbcwCZYk4fhUQWUq^$VGg5XS(X0;7g&#fNnwF!dbEo$lI&nF` z-_PV!lxU0dfH}Q zG4=GEoDNlL_0+slQ7Zc`Qv8*<8!JMjVq+$(@2s=d{&%_jUI;(vukQu%3A=XfM9N*Y zYDBb&Cy5}1Yf=1F1eue1Dk*vQPH0yQ1MA}e9`%XKIgpfmCe^Y54?A!pAqBt{Zn5() z1N8)Y_$$)w-W&8EVid7kT}v@eplz3oVzrD)nPjou+GHI*LE1^;A6=3C`6Mch@~F4kXY z@1eY-GqT6P$kWj&S=p6ST0Ppql_6SMy-t%bR|Rk>NI7|(<8mVI1Sk!l{mQNG5UkMyN{;<+*Aa+aPauGk60BE)xe-zhJ$!|DYxiHbq<0_P zNp#Gwn_ZEe8hU`35e`o63AnVUGP*oBY}>7gN7qg3>z*Nj}eHZ0>d3h(yprtatC%IZ68H1 ze9gIelbz5Ik5GU=K^H=R1|QNT4#QC%JK4X&C`-msSA(vuY~}j)F6*y-&v&W{05f5Q z1-{NyQx(j`gKdO+=%nA!mJLNW&8b>iTES{FE_FdZAuA)_zV$S%g;w2bedf5Ug3fu~ zkg8(xzyy{jv5Sr#pkQ(bjUU3A%A9@U~v_uL6cIhYah zXBOEsnBTay3wIADc?!#T%i%Vreo0+lU)L_3-JE-A5V_V|C>m>~8swarAxFYYL`oc!VD!#hoM(svLXAh5=qxY4p7`pV6b7u zuUQQ@lE23xV6dIyB!#!e8YvY!aA38=(Qk-`|7$?B+3eCt3EZV&19N!W-rLbUR|`HM zvborK*I;Y)8*kwC>)50o4b*V*O+-pY5&}{{`uLUK3-I%I?yn8{F<&q^XmKQaa-lEh zkw6Zu%F>ieN`9-H(_w3cLI3seVG917@0k%1m^vOA5%pyIkL|I4{^Ss}vs=By`d@5q zhONxncVE^Ts13gX_M_6_J4wFcWI3^JZ!!Zx0ccC3C%=^^&`-kOC`!0v_Wox5pt?XI0BZ{>ghY_dnn|D%hNM`{huy4xz2(psM5`(6Ftp$F~A~Y4K?jp0kd`3vVQ+LUn!qg1Js=oDW`;mB!16P` zL;oe%X)~BI&M<=uLnyd4-jque)l#N_fEVZVf%DXuZ2 zqDM}?VA##?%5cMSfkmb;X}v}pzJ2?4S!>dh|48s#%x@C+ktzk6a!O@?HeoWTzwFw1 zSfb)4;+QxBrBKc_m@#|!^3U)fyXv|+{p?om1ejX{lLa&DEQ}r)JFmOyQRje0UMvzQ zNf#{s|5hgmUpFfRZnLj)EMO8k!DEc*ALCs12* z`pujJ2oeG*SLhCMrx(XE1}BC}>Trki`8$Xt4O3_&`}jf?kNYmGUQ% zaQXlOiPA{U1kuufTl}@-*PX1tZocKe(b>O43Az3!IK>o%t<3B@z9|P_z0s5%psLm5MAdyLZhRe1 zX&z-*(!ZYu3{Ed38P=VGn;zS}GgHp;XBs_$fMkQIUPOB}C}LEi2{=cX2&k-xq(D%@ zK;YytNF$iH&(MB=mRL}P0*3kr25Ra;Ho?bvIhh`_q-} zKd-)ZVkQD71f?{+EB`N=YH~(4I`Y=Lg}_q=g6qAvdGz0p<_lm+4@KxVf$_q^U}7+H z!YxQxr(poUqi@WGy8&6jFWOl|3<06_DeC(l0RiBxmcR#yfb<2i>mM8({0JDZ1UOQy z@m)wC$DPLX9Y`7^cjjQay+eoXL$2)F*4G!HH97NXu;$8_Z@gOH-qNA!D__87#hQs8 ze|?<%^5nt%b^&@Y{PQ(_UwGX|2iN+)h03YmT}@_XRn^e-FB9OXGD}rbSn*f`Fdb^Wk%13T|TPMfvS4ikRm7Im?8p_>t`fIb{}p|b%(Y@#IQTF-k{~^ zf&cO3_1z##2{%EqCZvH}0p>dhPa0(DYcoyZ%M+Bm7cC zMESku<9<(#{fm3(Wm7G4beiN#3p(+U)PXx2Ilf0@E&d*NewSo`zgU=$w zr0VmWpN1~Yr}ujK27WMwBhk-+WM2>gQ8FR`6Vq^BQO+8vywed58&1f9d$~hzzh9+) z2cAuiZYB(%vXb0?k`bngMZrM90D(UfZAoYC=Eieb++Dl^fh1KrxgY}}S8>W%_*+vL z+VG2=oy$dca_-T*Gu2Tm-&&nyBUXo_qP}mL$wq?r=)TJj07I#qxbU$q{NK0P0WP&ye@wLXe7mfl@9iA&L(m%>toB~=_kOgGVS`1H zlD;+IaS(|I*1sa3SKJIQ)hPK%h`=KWA67uRNkzvY4X@*c#o6+fB^{Aoa5L$qN^=$h ziOi6DFPW5XIA^}wNCAa~@4l#*uuDM|DS`ujSqYwhk|>PE(Y4{RvG2>d19_2)9kWr} zL$c3TwmvUS3VEF%=iK|3Bkm~c-?$xAJBf!u{8B+i1&bSVvCd&?YU=rqrX#MK69e`x zVSa&umo7yt015xCZ5d1o3NkWw5ca6OQ2KF392hRb^#>=Pi2PTSl>xse3;98TwRiMQ z)^~K45MM&oJC8=n;a|{?!!MdpcFiATn0|f5A^dL#@Kp-72$I5!5A=czL#gOLiD+?e zcU+o4+4?EdAImF3c?#o<^m0{uBwc58S!ap3jw*EYESA+TcNzr;WMjc%x@gs$kh z#UUdrJs*E zuRuS45lXN3>X{=%UH?03y3_jew!v(~_ZMM#;q&9eFSgqE+h$P2{~uFt1JCsS{*Qm_ zIGyPRTX#6!7@OQhbcc$}Xd7u5GL2L2?>~-4DRi{5u};ig4$_q4o;0Q`e5)lA5+)W- z>qg8?lAGM^ckO(BzyIU^c$_*;I`e+LU$5)Bp4anrUC+yaNXhgm&|X9F!GK3ls}*tn z$8|O~_JI(kK$<%Gz3*vk=jP0R!^mx70Qnxb z__77M{Qn+a4SnuLNRc5^e42FB#tyL=-s;9RV72Lm>%FdDL2`5;L0TtN2+}g}5X9J4|1_l-u`x>WD4X+FxZu~KuoMqVA*C!51t-(~R zdGAY6{B{P(NZI#5<7lD@=_f}EYYVMADk%!svlzJ@qBT-9PW|gx6OQU0M-URq8CxOg zy#jy+tC98A~!{7H@$y}>PI-Z(}HgQtZ)v>anJk?1PQ8TGe%b`SvN3hH-uL{3B zB^X}mi~o^T*ElzKF7|G_h?DsmwU)8{GwLSs_(wF6siN`QRt6yfcOiG}+CVGB7KXE4 ze%Jr_`p6x_9R-G*1+e`i;lD{7LhI^LO-7d7#U4F%+>>$~F(S9#5eBQA(dmS8<)~c5 z@3Yrr%oKc^$JgLlE7Q}p(!RKrHm{ZC5!uo}QGUf~fELUzS55+fKZoTk42zTDl900A*fF&5NI}ou^SZps#=ReZ3xu zO2^zO#I|4zuBkYbfXh``gH=SF&Q*L1$s*#tvE1*;ACU;n|0Js8qcE6UrkbN*SYo9z9>ZkhO| zyonZqeAJ9tjp*t2J&dioc!!_Yxa)%7;Qkz-QIxLmw1pFi2Y%&=iqC(+LJLO)-Ypp( zk6^kvh0I?2%?4ihWhL}u74&1qSEr{X_qV7#i^P8lX9l!-o2-TZE8$t|^bKj>@XADZ z+>d0h$FkK+UoKm0PW3+q%?{(!b8l1k-8-;pX9gPc%|3TgPmaH6ntPin2{mS-U2|WPJ<%B2JyDLe5tBXt!DW+jdRHc1?wk_7`?>p#wO-5Khlg zbS?_fiOgu)>8h6EA4EO~n0usQeznsGY8(cmhbNXkR8;K!@LGB0KX2x(Hyw0B+C9N3 z>_8WMBP5_woN*e&U?NpIP;3X-hFT8Dr}9kf#PI3_$Ov5B4HS@hPE}f_#v$AV?W|Ir zmk)Mu3pKDrjlCF^J34o^5Z=&f(t7*<-N$jf0@*es*K3F=4B~N$D`OudZjW>KAt7;y z2X+=5IXH{kz{vf}a%O%lkXz)_mamr`fBfp?ax^1%eb4LRN0Bp!ry7@d`KupS=7%8N z&++h}J|pjev~2$1#)IiaBpW0diA3XlG#MutfQGbcMDT*cN)DE*Ur#G@t6CK|$)Iud z$1j4cs*Y{a*Kdxa7+()uOLyt*Yr)$7jKl>%NZN+9}YB42iSupJxoL}Je@XQXnj zsM?+_xUx)y_v;AY3XbXqD+zHy`Coa*9OVp>{SYJxEskfzp9vOgjB9EP&T)K$>;2EtT*dJ6q8rp{ zow8W?rvWmN#%i={zjAE+Ws{B8o1T%WzUT)ZQFgfFXicrHYu5(>Awli)0~miu`We>s zTD0)la`l63(aUlbf`0@3Y>`JfmiAQFW+U;|LyCy(V!i1(i3)jnMd6L77{=D;Nb6zc zSj&wKZ|D_t9w5y(8`u2ola55+`OOO9PtjiUqe)AirKL0hc47E)GBQEbZH?S@f(Jr% zCbDBvFbF3?2V)%70~0Bl(geC&H&AMmsC2L;18Ym@o3 zE$O9Kt=@4*H7o*nHKO`mjB4?RyU$KptSnf{X0}f)kH-(!U-Y)fDXJ}1hh%<{d)?8+ zJ##HQ@Y>w32VUx&#oS@vqXWm5+;ur>S~3^ixRe*a`b4?1F^(Sl;pw^9$U`3cf;J&q zIGO?O+o;41_iBO6sn`I%ea+Uw>p*tE!*b#5kf^w|NW_CJcw^LB6BJO*csV)d`qb=q zZ8$->n{VR~M*sd1;uKmQ0N@D0<;#6dqdgF_L)I8;fTLAp0%B|s>aAIG@k{~+0jb_W z&6qdyaD%Zp5k9;E2kP4k%Vo5vJZG!Y-MBA_XTL0QUpgvoE52fvtGJo?8l$%vedl+* zv!eEG!O0U770|1)INu~|sES_+-p-HSmfZe)AV%s3?B^EFwldYN)|(8mfp!@5brJ5V zvm%qZ?cnznqSv!KQM6zPe|?-9hV5}gf;}E>|CFG87M0?rswj6;58j~u&7C05COj_Q z=eQO}S&iu>ptgLphpW#m39k`F=QhzVJd|}MLLN6_DVwc|yErx30Ey3`jn9bjCf}_s zzaM;4x|W1Qsyuc2UkpiQ?fT-Q+V*eW&pw?26dQF(95L@wcz>Oj&TSb=yst9aH4b}FSt!;XVrYZCEgNe~?#{}sxdAJ5D{(0cOz{n;zu7Vho7o))L-Vy07A z%oZ|J6c8z2nO6!il=im-@Y06~YCjpGmr9L0?W>bs^SQ7Sd_KfqS`y(`wJ1ezLMc06PqRb1jGb zbBmYjEd?^-?iCdupx`e5@Ij^QM=Ju6`(GT&*!ss;9L9dF39{f@d_F{ww);sa74j~{ zw3=-Wx(ZCthata6d~rC;Gkyx-Bow4iB<_xT7%%$`j3te`+hMU0OuX<46Rn9^vler- z8G9U$AmD=gfpe02U0U)OUY(QkA|sNHa~9#F;ENXrK3@*)dUNF-QUkLr>v%R>=IoZRxqGzRvz$*}vRW|)7B;U4^{p0T9l7Av~{Q)#_ z?|+_lWcO!BT2u|q&tD#Sq1GAZH*qjs`hY_Gd_OepR&~zwsOW*6YjA}?Kwny(o1An# z!fW^O%jx#nlcr*MXNxYEgc|#UEO{;ZY4&*WFF4k2k*PjuoRquuWRRO=FX>GB!{TIl zcWqBmiw+lRWeZR@qSMXOA7=4ab9x1ZHeE5g9e&LXnHn9afm%^_PR^;3A< zjQ{FD=j!;W<&UQ>|G27LSv8!~-|`B2A675*-So@R@XjLo;QX{BTi7(s>5*cW^vcP} zFhIwXqCr|zT6aix2rt{dxna?&5be3X0`F*5gLAAZ*s)S9CXxVs!T<`N3H8wfd-<`8(ShgC!eAjXrdaj07psQ+l zRWP;EG<*rlZ)Qe1UEQVzYRZaH{$@P88qGIRwe?grn&0QehB9d53$~nTm#?SsM^@AH z4Wqc}t7X!%?kEz@WM*jgE_@w&h?CmSdBunQXbZ`%wo~Aeh{7BZZj^eujMm^0JUCc7 z7{Uupdliyu-(1=AQ-v_B93{9=c#rh>Z(+l>Hnh?#OVZtgR>FF!_CADW{4q}!>1sVQ z+C^FT)2wT@yd;w>`2%J1xC|P@)1g*oqTYi>7r7Q&6EGmXV0ex*kby6YuB$s0zDqWm zA1k{Qzr08G{^sfzW!XQVhsdHinO{jf}o zXHabvOW07b_>}s5Ah56Aqqfc7FDE&4hh(Vqa7}>|gXsp?6LC!@Gad%?P4N5= z?~eVxFn!`C^f6e*$}Csqg@F0Xt@)RyE)6er%6@c)g@swfEslo`Rtj8O?Frh(`>E;7 zP`-4&a##>IW#=^%`?a?!ZsJ#H3vz3k>xC-kf4XMxb_XU~`_;8On24hW{2cb1O;lF# zCu;H>aLtAhOzVn)z3Y&PnUyY5^09XnLmn+5VtYTaAG{Eyh$@%u+%M?r9JaM0-4*Mp z8=<3ncSABcU&q1`w&(&He{d4YoHI~@zqjU}Y|u@AsKXsfA<8u~utNprLqu#32qD9P z<(CH870cTVb*7#4d`kvP`xrt)OjIzk-|^gbcXcoc%eZeJTiZW~1tX`zZ(shta2N6< z(9g+_lbvdktyr$`p0&5PgW})qV}b^cXe+jHzn|TMQ1R&bn5nL%srdI@S&a)%wl^+Z zGm0N=+kJWTs{G$x7i23Z;J2F4qEM;RrtHB(`%|++kA(6<_ea*(*EiPx1$(S$n`@&V zQEbDa36fmuMPt;mvfkQ(AiJ#WS0Ms;7beD=F-dw@g=!C46rD+U46T*&Iz+Ohz$SKR z9UV%ASt9Eji_;~)KwmCMdeorcMM-EbN4ErSpY}Q-7$8%;`&^w-^Pn$DIzypMlqzAP zSQj+X85rEoq@bzWLq{(ze*8Q+7gZtf-yOgFeYCxs2bweQ-n~282=W)7r_+jd4_m~? z)s@=$ar*h)d>XB3{!4@OgnZd|=ZE5Dkmm>*8{?+BW-s;i^%ZpoDzyYTKhQ3EWv^+L z=MZU{SLskx);)SYqu+!@A>tH!$%(c1MYu9reSNp##Dn080T)5`L``U;0TP(E0GAo_Jkz*=jC#PBFIQhntihnux$Su({ec=GM=ckC` zQydl#`VgVLUB3%8r87&qM;fqM&T^^G8SBOX8_n|?HOLNO}bCYv({L>lFULD%+R#Y}o zL!;3`#Zl&YexGYX_lraJE7f}J-JADJF|FT0JbAu;fZsj37$iQld7k956Xlx2kE%`j z6yh>a;39CEuXjE8Ahf98kDun_XOc=hLvg&(Q#skI{1-!{kB#CQI>FvnuA^@&(KH-` zW~!~0yJr6;>O?@drG!r3HdQJ)Dc@THu`{=#cARSCQYx&}L4qA9fJw9Z>}?y;N#pC( zNar&wAdtXi=-gb?&al48u!^Bs_(gYjbt@BAX7AhIipj?0FkjgV$=hjVr>W+=c6?fP zXtX&kFRHId+Wl%@+nY*$aDN5Ksa79?z+sOB9zh*h#V zMwTRQO<1Hv6!W%P9>OYNIb=yWDfb#`K51cNrtsI_qoWVAGzNn7k6Gb->Msu3)x?1>_6ONV+3gGa=gRTwKM%9iFWSHjI zYqt7mw}yC_tkdeTrt4^OJ<~4G4njG5I-yrn37o`Ha^eC7P1l(=bRGL+VMjW-stx+8 z$LOrgw+Dx4W8h8j7OM*%%D^F2@V6Kfu@;t)(ne*7DzQ|5F53k}hIdL1cEL+yk|ogN zPfj5C?7*F&2qr}@5DSD<_a#<5g)s*1Qq_WNp{t8dCaZx z`-fh74Iw5-n;HG`J7viGzoQ1n1iF;Q#z|EFijS1sH^J) zuYP%X`bBhf-9!**?4t%y+_J6qw(r`GB?`yh)J}@1rL?^XMDhTAfDf1p;`))iN;wiy zqkVBe`BzZ?;)7)bxCCZh!WxeKaY0v2JrzP)PPToNAt~2JvR6GE`h0DqJ9~~KTR%R@ z;sxxiM_F!zWu`-561*n$aG9NVCV$2brN}0{fQek{^;+qG7Gi+2zWq8x40AWFj8@@! z8h2(3mWomrki=vmS0})UHD?QtqW?lfswFLguN4W=pOi1>^bZ3_A-M$VJ)_lC$p?qhp{+y)hAkK5_! zMQp=Opcs3__I$IGc!n)$TrUe^xP2KQ0%tnCA}p^^0ZxbI&YCICY^Fq4IqCkyrtIL(uaX^OpheZ9Yu za-XBqHIO`Cz8RCqmcTKzn{B#58)S{6+$KQ+2M-*Dn4qTu(hT2gB75dDOUp6BUpuGfSO z#POthH-VFr(Q`XTf9|C32w7k&Da0#g)L?JNFOPq2ZRh!S4)pF9=>?k|NK<`w4!;BC zpK9;-uB?|{wIP(1?Am^)4?M#ZEx)qTfv1MSrPZK<+=ARU7Di!}3NumLxV!X7kF~4y zuM)Mh`}yPAn^FFOkd7(bkSi=kI2M3g`lEDlnT{_LLW72cEaB-!5MbNnl3Qqtw11U> zFF5O@g4RZr?1t?9c3Cg%(W*$cP}lhVlTj)nPjMPnMJwygq~hKsM^#LYqcGLKg4=g3 zn02*6CX(k-x{eahf`fn>2S`r`CuVMjh21`Oi7$Fnn^swyE>hJ(A+4y2bmMG%!IK!P zUTy_>;P-*gV`GaAC;G&rHSM}A45T9+^^Yln`eI*Qu=QT}&>%E!9G898B<+#EiKY{7 zyrB4!V$#2okf6T;|AYrQLYI#~ZOe1gzh8rSBD{p|cCDgq7IJp0!mws3v{SMlL_YMnkXJh9p9G)r@2*Nb8)qiMWgo$Y;Id`-$87v} zW!W3AA1vv_fT_@DSw}=YO{>T(rRf>gmxE*qq)aTzP85w%@3tNC@CK$zx+&xNTN^Sc z*{DBBD>?-^jA#&Cy3|2X-)yL6ykD>QdggsZM)q$7JyaSkDA29Tuy$O%pZ_RsYH@LF z@w|C^l)9D@x5tj_IzbhbCJ(sm#3A*%1~OQ#uWO5UFnJhE_AB&$-bu{@1}UI8nW-sv z?;z>~jY?o)TV^Y9inB_ZfjddA%Kt9+{sOcIS+aR!lAbp#X~7XQt@Qtyn|#lsZd_idP#(VY{mYbW z&Odu?khrOQ>P`d4pU&FQkB3TS6L zQbE&H^ZR@rz*9YB*84YW-NlRVt6E7?dfzw^J;F)j38;$e=J?X!3e;wQF2?7uI7Iv^ zBpZ(+o*(f{d5aCW{1?io5awPuN+P#A866ypOcx4tB>UxU(DR@LGn5hz)g`@wOH#}E z!C_fL{Ez6kPvP;Og1wG~dT1(6TL-$e1=VgS)UnzblvOAPh66SxXIqcqPiIo?bhsOD z_~1#!9P$$|o1C3MuiVq@iurwJp*3w!bhI>VFp#d;iqX_g(n~@Gp;#JGZPYTruhc7Gz<5Y%oaTEyUNzQf`s%C(jSoJ`UVU)31XyjWz6O#|* zDhy^U#Q*^59na0@+>*&C(wUwdxi~A)ZUUf`Iei;Reb($H9x`osGZI4@!W+4%8F0r*Wv7 znqYTYMWxFiQ4`^tlk4CHo`&BYQq)S=QCgNxFwshXK*sGPZ3HBmlsu5}TZPMwJBBz+ zTBsjCG)jCuqS!5gb)A%nf-G?}<4!S3efl^t<2AD``P1f;4V4EhP7q1USAv8#$b75k z8~!!FU*wBBJ8Km8;aAymzAQO@t`mgFR(ei(d3k+$<=nr@NmNHmoooi#FNUt(XfN!b_&lQh9u3-;O@WeSeu)<>=(he$?*^1yAQqzpH zO@sBTldjbU*OnEQNeVIIkfQD=b4YnlI+%Ft&@z)1jnhfB5|wS6ahlcor3y%YJC>_Q z;x>9Fj|k*RgAW8U|NVybSjn6K^-?5|A<$wB zD3W=&S9}e_FsE4SvywIcZvZ4qPNph1&4QgZEJG_!Ny!5DPH4=vGsJ9CJ zyM7Al%iHB@=cezkSxjrA(rT&Ly%Ed+C5}Ng2)to?(3t8C5_*NtnRNTdC^ql4i+;91 zFbe`{MbP6e=f2*I15{2YlkA&?4ZTkOg%xl>eBDd^TQO$9nJm_{^Yi#`IjK5NeorNA zY&+Yl&$2d_lt*EUUy-T6*A((`zAqrVJQ51oXp3pGtB#@~w@^}G(R;^`YPI~If*xoK zy<B3nv>F=t>uNp#gc;h<&%N!_hv@G6*z+No?nJb|UD_M_=>_LOi zq(wz+OH98H>MXBeU+Z+Hqh_OEOR-VJGY2fLqTCs%d`yo`%OFEB^$})z4(|nhgfLr0e#szZ~ZWTtq;Q zN+;*|uAk4pbr2u&&+g}>s%bGqhAe(R{os~rkjUvJZ`x~xv{FNR>Tqjnu5&iZ$yt7T z2oDQ>zz~=B{B%@A@!?t}q$_y@+}bhv4Q)@wSr%^PD+l~6i-&^5lq}iAa7s6;yEi^&-9JJ{S~@ly#)uRk1DX z?6@C;p?Ocq#fF8~XPw$?U;YRhcp66&(7PHc6km(69-x`B{65& zDdcjO!i)gl^BI+JH$OEAA?KMq8HY$xlufj6eua5(!?=U=xaO!z)jg`fkSj!2-x3-I z3M-Xr%igTdzTXZ@_2Gx2vaM<)1)jRvkc1EDkr?C5pvM$o4|F27k>2Tptbv}D0+w{T z;+G^9|LNm2p+^r8DN5WSLk`)0`cIr=J%U(_w>y*m7F*A}JaS~WMSRrKa{o-#I^?1W|N%+Uej?Ck_!qnGL&E39~0=H@4dQu`gY9 z!PzNh=vX~Isq-PEpvDikuM0!kfy*nTHNb2lV2bYM$iYKhwwwsaE>=*~CY^Ju-{$IA zN~)-&*G-Co`a;CLqhkPy(3t%!pIbyG3>G-szqJD6skay07bztGZ{mr77!s`b;Mt-5 z5KwvTh8zu0)6RrAx4A0VrGe9W$HH-^gr`%z0ZZ}l2S4n;IsYn3Io&ip1y6iT2}g!g zqs#UZF;^kY0?uylUx3&Jows9`$46gG#ZPZ&`qtL;P24oU6hE_F*8I!v%lGrNqo%_| zRL`+v13`UCHcCom!3U4s*+2t`llllKcv4<%DI*5!iD03S{xHicTg84{9a+J)-Wl`o z(&~7B`(*1$0YBR;R7)G_PprXZoU7<9I8t64R9D|}hzDGJ?&kxqlvo(XzU+Izn+%rd zaTOJl25N(Vo_xtPKENSU3OYn6yq%wkQ69e)!FgDGJw}&#NG)AwAch7>BGxkq2un$) z+x7@YoQGv`NAm{w0xTO=I0h`=ALG#2o_PUlphm;%TeB*yw`-Yfe&EcCDYVf|^IG^J zoBZJ43AE8MTHb22(W+!>d2VX;Th(%2Rotgwx4xq6=?)QfGUGnUt*erMz9N$`&|CWE znxQY0HxCUO^1Ba-`Pluqdz5hzjt0s21uOw0+fD!alunFENu*nb_nIAJ8=le9QuGtr z7gVV@C=A^l_oySiOi4YV71y*(biZ7 zDLu0QJG|W7mmenn1@1LMnih{Vt;EY_m1X0nWaD;Ci)B*!PBn{Kv!a`~D){F?xyxUd ztJGcMT+;t2bcsKx?S1TqvFHh==R}V+t4~5fMH4C41$Yw`R8{)JWcN`6KmYmI;jp^U zrirDYel!2}4p-zT`jwgSh(R9jJPID8E#_gR)Vx`RLpZk4T8GQNwH3`(;-zQq%hNo0 zRR8abiSAAtr~gJ7eL9Qbq{7l|HPxQcJQDZmN9d()Y({IfAree$CdX8NV+>1N$szaKBCIWI#0_Rfy{?uZc<+nAPlom}K)_-JcY#V$JDE@bY)z8Zs|9 zBb74(T=FFDM@AYa#JSJw4ZPi(nTe{7EbCEx*^lPFCgYLSCpY0f6Rxq|`CBYC#ZO&| zUpg%N5WjMD>hc^fKM*@T3O(WP4Du>=o$u%`A&~-R@4O0aItH3VMg5P?7-ntnqc^b8 z&i2;@sIgj95sH+X@Gu#ppydOy`L?+Q^Z>iAe!Yp1WV5oF`x9bNpS#V!ctW~`NC15q z8mvY7z)#JXToki+7YZ*ed~bN5*VTh!jVpF;?=YKK;)jdJbJO;D*x1?ar+Uy*f* zpwvtdF4wEQ#gooH@T|&`($$QAg>zh2?0i&#Yn-6M#;ahItW@y&J%bp(B7svkm`V51 z^S4QXjzp%_4MvM1eGJ2+2kEBja2;&4rb9SEMI0_qQO{xXAU{?-S2S^`F4Uo@#cHD; zb^^~f-q(z4CN%4|5;WCRy)P6yvz&_)F+TX_(=X~hMZRjyz(g$6BnYi{m`l-`TnnO0 zto^ZcJZAL=F7}MXdMv#-5noiA_`0Lx(J((WOmJE`t4-Bf$LvDkty|dPRne6Dx9?AW ziJy;-Kg<~h)!P3`29M@17a7HkCVOv`ZR8e$p2J_Iyu=bxQjmYUcWrau z_W2Qb3v3^C(qjq=wwbAbR%JTd>S!UOBevr`h{bwipo~_+iTXPTOj#>pJ=}TiJF;_e zV;=9-D9k8fb{|lrAAUYg;h{nUix{_fc{5RBn9>Hba>V`x2Te?bEh@;w9WAOvu$2^p zwgk4Zjuj5Clca}Mtj$K_YtuJpgVXGvHcqSR8iquh;T(ugjnGz1;ig?#)9am8$1^)?<5g&lmVT4>@w_ zb>?4O=aF}H6Qk#I@}!?0cp6D(>)F*zv;DYt*EnQUq5`}DMG2Z^Eh!}L-a4>84Hh0;|@uy#pkUUy82-}9#~iK`9!q;#5_cR*UY z)wONTwSP@rUOM8v6HfkoyllQsIm}XK*Yx$p@Y1Ss{K8>*Xd1pa{gj#SR4hQ1SSOFV zw|8`Gn+U1(n-|B)cQh=rqZ-Z&SWMgLOu}=tP^Cx%B7=s*G-)GAMY)%0P1r%*-Nzsy z3Y89ippbH@90a^1BYLSd96qIPT?Mp4Z$Y}l!99O>=jJ8G&g~O)!p^~vQE|8H4Id^N zg;WFqB@FmL!bi}1uuT_oXUm1w(tcGvCA^AKwVp(l5kN&h;=VE*Lbjjrr@$jy; z2ZQ|hg)Ym>U!VLk_4vZbeA7za?o|NxOR@>&_|fh05eC__qt8nqmRY%I75H|P)NGy@ zKN_+WB{(+7H#_Y8??)O)>;E)nT4N+CxMMiUHWhPzdld0V|wA|XSlD{%)u3D+GAlx5xp>}@P{I&kYO zC}2x!GYL9L0s0Y1Zn!+$5oK}IrTU4bLt^91wofq^Bhf*v5l`rvE zpj74=pxBfPQ+>lwOtA7ywwxvVVdn+&mml!s3h*c#%fG@}y3Zwz@Wo(n-qIGo5&qQD zyLrG{heiy&&on>qG?dmGN(s2!v2-izV`YhQ-i^}CtW)a(Wh*5t(uHG{VOkqhAKU;1 z5!vd>*nt>pWD$a^{`WaPJ_FJ$UeCMsj@TS=;~;pCM>fp_6}(hMIH;Oxj8j9`@PHlD z*iu0TDz%@JVrq|*Zb|h~^FDUQ@I#8J!nDBw&DCqd7jW4ipGG48;dD6zs_B9w=9Xd@ zdIKpn_RniBTE;VWu zw&!0I$VC=0HF|<0pzKMDl;VVn2pX(avXP>lW}SM^ub(9ygr(ueC|F9#%@T8$RI~O5 zI0;JP3Lfa%U|^2;)}k1K&Q)6-E|D=B=1_%Eiuh}C z_OB}0WZw&u9)75N6&@yyU$4c_ZjWCbiJu)_m4?ga!YjO639KHjg`WmGotcR83)Hv5 zn0Jkm&Hd|Q^Oxpo01QnyryF>ywI#!j^ci1&GpYg?tMW+<=?nKckXm6#mnkWPzY~$A zyGoTR0vc>w8cKLNtokx8#-h?8G_7VA?quShqQ>@E0B_>%i9_O4wl5nm$tWO|l8l3e zJXe&a?W?4ICzK|t(|iw_B`kkvCsEP1ZmvHA!K}}BBg^^Tm?10PSpLF`orQM2c;7)cQn1`$bPJi8__E8nv zOkV(=eQmz%_mm2hdI^UtCvm5N?ya#+#|&vsa%_*;5sIJTi@-Wg!rs&SoNxz~&_EWZ zU$AC56&08inT-I52sqAjsIGLSSCo{s$Y0p=3O?W17x(aFH0MLWJH~ zcElr(W)&fiT!e{BFzxMxGqOuUk;tQBA+pLwzxM#8cbq>Z?hfsU?O@&aG=jJYIA14X z%x9nPEyiUS(9+9kMyR!^0)jXk_n~gJ&1)q~_Mvn2d;E_KuMSmA7pYi?--#M?mv9w; zYsk#(NJ2W-C>;c#08V>i$zmj=hHZR~)+hZ0$)%*nsc7LLs>k{=o>+{L->^6rtCpY` z4872uPFQLYsmIQxP*`%y88qf7xwvDD7zEZcn@ghdK7(D(egFO>R6OwKzUCA*-XsAv zu%1_ledSK26xgLfznMqt}vKu0g~}?%|P7lIQJT6HYElz%m7mhLdA`pv601WgSOurJhR$@HbxLK z3h)IP!wWZt@4O@gxn~+@MqAEIHZ4BgzWQOlaXt$=;F^BqG+m^*6%h{hpsn>4pUUrI zu<1{F2<>LIsKh5e``RB()WK&j@jD$%uKJvAj%lb&G`{ZKz8(ux3wFr~ecDld?m^Ua ztBKRt-8YB23U=XiLB6!aGbnGY(DvPPYsJ~?3-{G3IHaQ39Kd4tLp}77pj(})%A#nw z6gclB^%?*ofCSYWQ-TWGMh`kNGFh3CN=Pupq_d4h^af9(w$lj=6c;mn6HdSE=FdMp zIsSgB)b91D1j`{+3a7*+##_wpq39!ScH!mh;I? z-|e7K^6=`Ikqo#V*`@rISN^b&^*2)VPF~-4jG_$nPF3eSVpO%%B8ne%#vU1mNN&ER zi`slz6ya{vR=?5ui$Mev4s{|#=?FbE>lB4ydQho?T=l%1QEGh*rXxD-c7~}~atpmI zyw^*3UK2XQKL+if-TYASOKN6d|0%Z$>0{+j@K}`db$=p3qJlYghfxs8&qFjLo-jE& z#vFRX86)X~N-H405=^fx;-R=n5W{o^V6Z5=LRbfnq-sPk z$yiDty`$re`BodaeRSPv?o<9DY*Q~Re|Ro5pBD2-7+xyGe@!=ZSl z;2wv0iX1&njm4M5=x`C-ZtjpKX*vCzYgua2!2FLVznn^(;Ir_}WN)&Xl~3uja`LPS zMhl4o^u)7$4B5#cmNQOmS|us9+EXO`WRi(Rb)Ex*v~&ZOLX8D;xAIFo)h7#FN=F+f zBE!V|&e(h&Rg08kYzU~yGfSFu@$E?C6pwe%HiZrSa!ctN;9=c|E>24#^JUjM<5y>z zW{xz?#IN?}$4Nbgm&g4rj$~d_BHh|bP5*6Jo3{%$*lQOl()R{-L{#mw@s-(pXntgV z_t_3zzh5~3%REJG5RI>y){sb#-^lOAcRQny;ch^%J;UYwZGNpj*}5A-QG60g8LPDU zpw`y*#!hi}_sx8ZTT2vgyooIn0fX{9gB14?eHa_Hxjg5L3)>O^09jxhYqspoqa(89 zNl7N1g-ORt3@!A)_F^fl!!@4C3;w#(&ie9zGfUnW!Q6HJHa|4SXfy=tD7g=BYqs9g z@Lb2E!FK0XgxW<=R#j2(W|y}uO7gbnFTEQXzBuxH#A5zO{QNKRUstAN{~V5+(Ko7e zyN6E2MnmGSkq7KI;GupcY%@;vZ%D?!w$kZ1a z8t+pj}#&IppKgL5swOC zVz0GMX$C=L+zeS^aer*n#4kNOzQbO%f2qYpNM`?opc9rOG1J>-w_QbPQ#+{MEKB-Y-5qJN#@26Kv(Sm(Rb6{@_=I-a3GGJ&JMt8YYn9b zC_c~Sfmg`^&|)FJ?n^VulGZD!MI>dRd@(}>41m)vDA}!lJ%|{pPxg|Q6|feiyVMi+ zfjm(-5Hykbzb%4de z``|_L&%zUbdSGbR{Pk}#$5ak^PGRBsVhhL;ifr?i2pu`kcc{+n$cTk<=w`ngc$GwU z>MS^dD({_k7|zU`PcvioK@z+=OzVRv)mtqJ}>=hDUaoD{!gyl6@v71|RO1 zOHD-dhjzzRB z;q3f3scAL8)^((!uj(PK-hKKL>Th2%ozL^f4vd@+@;~A+5#mw~S?w8-ud(gpZKM>_ zYp4?2fIK-dz$HmkrXBwaycsc2ek6Co)~xuXm|J-;#dv&4^jD<0i*$?Cb?at$Mu5WJ z?m| z_Cac>HS2WBLc7O0g^Bi7>+F#qgpphwBvrY>n!q+TY4$yK!Tp@5yJ2!B78?TSH1RbKhMv|9RQ~;X-9$Ce4>}s~$+x8NKISH=`fC|#dvRkO#sXxuC6vbHQj&ZRr(Zn9zxJ=yLiQJjd zQT~MZo&S*|MZ%CgNf`SN<8|`)X5_u|A#gEup&k7}T`BW%D-8^clCgm|ywUA_+f4?P z)Y=yNI~$h-UQ2_PKcG*oB0P3x#B%A8uK`$~8`WIu{+NbpMYIw!eRq1F?+CS|cc7+U zzdbRurms9)`h z63>C$?*QMdNW=};NlgqAqIqjs&B;FA+Eo=J%@0`kCmEMd6^9AVzQ z^L)so0tM}%M%?p|Gio8Vw6x&dRAp5g@CL(x78&?FzJ<|})%3k^7Xg^3ENt5Y!>n18 z%q~}7wJhI9KnwN3_u#lU$?nR&o`AS1wTzBME530^+?Js;ZJQ3tAr-difr;;Ex*LThfV=kp|1r?9VZZ5* z9S*M(Tg=Dud9Pv}5;HLg#U@%yRK-0AAX7De9Ok>qs9Sm zA6i!1Y`uYg7%+q1YuagfxKGd{bk2p<@_?Nh0YEV;m@`yT&8E8#$ByI;FHTg&O%6K|Z)4&6WFk8jbn>~e}rx4dwrI8FIagE5#)Bo3b7l{w^%IU60HXa7#qLZ;?b zmwT&eeae5o)Wk>1Ml>r0@5kxKKNYO}CmLJ=X0xsbh8K8ptTXhklSei-Dkw{SqX!V3*|}jhjQCi)rxJ#mgFj`+liP<$b#SK+ zleJN&0?P6tZyy@z{$x`27cwVE|5pL~4>`V6mi~S^G1n+-W_0qhMQ7l4*5!G@aPI)# zj?I2{_E`A~U)q1nyh!H^q;C1gmYNS+u1aA-6M*keow84z7OSEcWsZiC12r!Z*9jq- z3M?OX^`AehwZ8Rv@yX;=QTLDYe?<;l^9dL&r{M|r*%3!~_0egWRv|C%AZAJDx))XU za+|-Ne#fZSr9uu;RZ*0F;U+x&G~w2*8I$O&wxeDhITs%V><4M-Uy0+Tk^D%0uR1Ga zK%486X#BWxR)f+t(A!&{&SnGq>*wC==YG9Rs2w#6LmhVMPCwy5A6qdkImTJ1TG3`u zU$2@d3A7na*U|cygD5ND9UC4T%Bnd#(rGte7&ng}&F?7c9?a!o zaE(i&MUo6Y+@3|8zM1%;v?KB_n#kB7(f`HG#>I^A_>m*=GgYgfvt->yvggnP4S(u* zFNm+s4zS&~aUQ2;AT_UidipxXY36cc?o>zC#nIeJUrdr&a28S$#NN^OEas?zH|u1v zP`^S0l_UBUcT9(S?+}PS=TLF_OqkMG8+fm@(SRG3$BQxRocVO#B5x!Yt7VNT;JKXZ z>)`WTB!7pk^WNzFwuaJB85Nq5P8cV(5=77HD`7FV&9*AoUb1`;G9iPUu<)214g;vu zO5u!z$x}N!Y$XtL0~AFRu7vh(3(x75d46Swe_ku0xL(l~vCdXhUH;;kYVz6JMk6z$ zmm6zr4&}!!>Y_ON{8K;v$Fw~0nTc(eVcUaUN^n+0v;A(jq4RH8)(2NUvK84^%Q*P2 zr7^eLE}_&MjL*i}4Lnp{bC{a9nw_nRFgGvjqJ`JzTzdflQW9Zh21EYPAL{A>&DYbsh>b*rK7uht073Mzv->#-RK_j0Nktl>#<*p3%k{gmB+<<+k}ruiw`f~R z@jqj~zvsWZdG>k2yf5eSI_G`PdC>`+*DKcw4xYZ2-ufl%dio(cj71_unU%3Ef4uy5 z@ZOO)YI2aIOJYgf zGp`(B_Wj)Z_HB-Q%jm5fQKiiB_k?#RP1U*#1vm}3C211`)Qz=36E6nAZM=X~2Uj_< zIl%P?Qr+3*>$4`JfKrsLQrLd;tN`_FZ_P++4Rs$W9fJ|&*NS!=J;qorKzoDsIhCz9 zqieuxGH|X~O4GL*wW_W)(m*X^t}^8mbx;KE7e%rVIdQpC17tcCB!baVo!XPTj$@i7 zV;E&Mf|$MGO?)|32j{VWG!N&mOb;eK@3Lw#NG2VD$LHL58zVRnj-ic;FlBS=??q#S zCm*REXxhFZ;c4d1Um)zRdHsFz&#HpmUytsV&p1l+SQztw!BV$23(zhb?VGv5{>m-U z`FLQVy!ylxSsZqn?Se{KbNN18X9j>7bG@-hB z&FITe$02v$H4il{pPV`D#p^Y(z6&-rY^Fua+qc^>`SU%$t0i&z_cpftJ7th(K3KdISvP8uK^5Xd?)tuv;IZFkdmvFLBq3(l2^rI4Z8+}Z<=G}||%9k!x@SrL0PxZW)tJo#_wM`gwgNOSgF z)SS-A{+z|FDT-^<__F{iC@Xc6aG(#`H$R=)0lEP8@+Kr$&*%tnS-}LA^w7+*WAeu_ zunWfw$$T5QvIAES1M%5G5WIqj6abPp9fG%?@8@XOUNk8{wvr{EdI|DzIo-9_Nnto}m{Z>a^+5LdWbJ z?$9ST$>(Jk5I4VGf}rq4w8Cu2Ob905DmuSCbk{BLKsBy++FzO6edV(g4|Ew~;;>KL zJ+J{kj#a7s*!p`2ND&g_SnB}y$~oR^SIrwTYt(#028RhZzoxMk`VC>MZK!Yo~0*S@zj&2MhrBK%(y8)z2rlpZo6n z^j7%0{J_%-FkNinW&Ofm_1{7CKWIgKe*WposT-QhMK&&6E6sd!}2}n2Juhu+5L?gLCTL=+RzIj z_iIG6!e=Y(EO(jPalmSJgfb31Xa$*}NRkZU%I(YD>gtzY!XEs3r;1Rf z<8O$hqc2SERyntDuFaqcijz8Qj)PP2x3`Puv47baosMv$;foTO`qb>t z7xUe`$C444&St{seaazV@`_)EPI+Y$)lnTQ2fCK0LpBt`6z-dbW@D zdx5~~VAs2QlbO30zXBQcX#MwBKnDBr*7yIy9D?-4;q~e}7roumtbQ>RG-}#*XqIq@ z)gipZ3_d76I7{}JRl>eOe^E9BO?P|1-E z(ra9{9!fNzZC(C7McJ!v);?+Np8_ep+rHKdf3v|Mrt41GT9G6s$sY`H^TXdaw?w;% z(vWn38(_Z(@N;|ariXy?xWVs=&8KI+kb8qOY-vTlmbr`18e=n0W#)S0+=+}dI8BpV zPt#&;#$=_rG9!k+z08?m)bi+&Y70EmKMY2aZrosMK%^3M5Ylkn;L8UVx=x?_dcS_* z&w+)ZE$6-g`P$p{@@kuFfnBvEge@kCLvkDSjIyL|$L-?X2;$0aQQi@nytbe%Ky}RE zxTkXUGY~uJU2)H!nnRG+X;kK#;#Zdu7-V6%Y{e z4KYgyt8Ipu8b`DO+dqw*$_y+A_k5}yc+C^_*`6to?^cdmr9XUY;iWqN<<+g5mSd(=hp0o?S7w-r)+nB`R6qFz9XBBZWh`Q8#Rj{#1!Y;M9e*%dTFBk z5FHItb#|}GTK%R2MuN3t1_&W*U0(JF&5k>7WQ`X>z^|zhK8Jojiw{yhvBnr`PZY~^ zvgq6o?t*8TPe%u8EBj8h$C1El%6e7GICSSi)wN^ObK@bdU5uf?A?CKBoj?tk-&l>y zLJhog^uzYa1&C)1^7Da^RRovIDE-W>AtULU)0B>kPXPeNYM#7;tkn4Np!QdGVtqEi zCn~utw$WNJtOf5Yfx(R(`i|>bAgfe#`CX-2T2R<`Uk{*d`~Khhd8dd^FK@xPjosfT zo$6+eAN+~2?N>I;ieP&JE2mc+r9%(NI#rG|)>1^#=8c-A_d#+rTCAdNXr4|^7D6`X z$-(*xIsVtnT3`X*Cn-6nqewbr64SKJM97s6y@z|K?MCK{HnHyzpJo&r5Y zYv0~0YChT=v@_1jF}v{sW%_Jh9#jj$Yp$(bd=^#VVs8sy`=LUl8tk#t+G&Zm&U7fKY(AKP&nU&Y@T>-kG%EMD zAG`^LPi2jgwf-kO+a1|ISRjp>9sGrd5p(rjCGQU#jaeB|1#aDO{s$m$Uc576A^ya- zl;ZFS-{#Nv3RXIEWwm~PTzgSk!Gia5S_qZ zr9#}+I(^b~gHo$1teo0$r_rShoYH4Gbu?5iR-c|!!gg_shBE6MjcTAR9xWWLTh>q|t$r&r^t<|ME8 z+p;^xQ+rA^t~VAjN`px;v=-I3_a?6%Zc~MLKq%(HY1nWKJ2amw%I)B`B?-^3^6B>- zbK}B~>~NnR(hfmFY~sg&Q5L8DTUYd}ugxUm&*z|lT;hs!Q}!%aND%a($i@u3HcyX z!TH-iF_~a*^`>_maU^m~@oWvDaOTy(;t#pfMop2q=#S0J=&n2RAPY^1JG}Yw!>YBd zG?AP3pI*WwH`cQe~c74RYh@!h-SVJGiV>I$4@|$hr&2kw5vz zlamVIc>@CfCIhckTy2XncMd$k4Z*je%;_7@_|8}2etV0DvxcUL;3YNCToR7LMw?O7 zr>*_4=F`TbarwMla2tZnf(;4s*miS%#Y#3I-u5N^&F#h&IwdPDQ+dwn1rRiEDPjxg z9UA`jjFD6PA?9Qm@i zG4hw6@u>N6T2)DGh2q-khtnah-G|+3E!GZ+drrO8^y5nRvPHAdtp_uL##+fFsKW=0 z_`%1qw`vU_9lV7Qp#|UXJQP=X%(iU3zxe67?=FQl*@oHDWbb%0jm~EWvy5Kexf4gH z(>DtWoZddRYP8nP13wFhSs5S+EYn?i*1021-c=}DI9axG`T$(ljYjqCz|8-}3VMOG zqU1^X8>-j%kC!1HC>r}P0U2=jd_T6xSUgv&Aadh}i=v*X(mva5q80V_q6eMIV!PwC zWn4t$COjf+icSxf^)gC0Gp~vx@)@bod%5&!J6pT^%7Kb&HO!14tC4otElvMwQcoNS z_GYT>lj3cAf=lZt!ZF?uxi0MRB)J;*K%!e7Cr>+mwi+J_^mI!3AEuNCKqgNEGpB;n zOi1bY3Hlp`ymEZ-9wc;v+>~XRL{c@9?q~Wxs3o^A!T$D(#37$mv(D*`!9jexGT;7{ zU{jG&i?yoMtZRmeJt~1+Hi_rM{uvvPu7IkP$9S`GS<#NZgLwu92!}2Y>fiys>36RS ztA2CrjmhVwpCNKzci0fufqAGQ&o&g~_lnP(kIg#mx28e3`>abHdP=#`cLB;2p}l{e zB?r2XnD$S|PZyDK$3NKop}JhKq3d&q+*R_xlYjy1IHkN~VitFUt8paC%ne-mDtt)3 zLWei6x>Awmuc9P7Vq1+`4&V1!bd2SUM*$xs7e@%x*b05KR+}CRz zyr(JHtdUG%mwnzF?Y0w-6jJZ$e6VgdnbO{1d?kI_-aq1A2QOa z<$2Lby);n>WDAEaA)udrb^l?NW@<<^kIk>;iz2+UGKn7wJ(YF*w{&QV@@3hDPTbcJ zXcjeEn>FE`(`AmGdXs-9K0Ws-!WSs*(s@nKxLeum;}D|05;;n3f(4u3hHQu0>ZYxh z(N@p}#1-v@4=+U^%r)PnZ|w`KOn20#F?O+BQbY0)b*LtoN;2n&aX@#Nv?RMBypaq5 z`tmc?ThqwTw%AEc^`aa6&;`l&TN}F!MQ);5A1%3U1Fb|nB9SS`21B3^kJ5#Wn|^y7 zouADo=Nd7bTUeeiv+2*Xae`HM^1JS+_!zUj7j6xFEB|G}fxY%u%SU(`Oorh4$jYS# zO@|7`mJE&8&#nxTtZ=upG@X*#ctK{9XDR=nCF4bzy!h(t3voye8lp2k^{yDoB^Ef6c~MYF;~e#s(W&ngbzOD z&O>Il>E<O2WH(WW@=WfDKMpExLCdQEORTe5kflJBD4NZn5?Cw}Rj+t0+lCZTFbm8$Yu` zlAjLy4$dYUv*%)8>Eh2X6DNKoZl&B^h7qUKD4$#N$K?F?h1TzB?w93*Oe-H1Kh_e& zrKRq8mnWeF}?YhfqV6pby8^rP@Y{~KY)TDZB4a7$r08ApEa=fS`rFljoO zoG1y)jf%{Gk;9zG$PC=f%PZ>MX8SN~mKt{EAd9yTL!AZ^hdh}{Q*d>>2~&(lkYLij zC4Ndi#1@TZSyL$VL5>>vA~67qlv2Yk5Z?e}Ob9T>3_e|n$2`#P)grZsa*}0O1aT(P zX5d3_%4b=#=WeiOzn@zSsh{0p{wBpb1WB~UqdXm{iC;xppD-WSFhz$!;xJzmK5k2C zU-|PcmYu*s){sIxCnheO!65F88lSwAzt7*B-(F?moV=toOdXs!cAQW&|P zuD}<|$lF#AZj|q*&I})+|JSanD}L#hO*>RAe8@V9jyz--7R;*qiP-SV%%OpZ2a5sq z>SIJW6Zv_FGt8~9gdD6OuCEnpKg2_1BBDo=ZzfZ|p)WVWXt=8M$=fc%{oe8G1de=J z!{Qwr%8GG{;r;cpq-#gjm0e%Wr5&n)PyD$G2>!&d4vRur6RHLCk#%-||GwM^wGXkA zd|C52g_ue%Ph6I#Kax>l;@hs66Z!Gz(s<$FtX#i!B{DlhSyKl8&%EkT#YexuM)t!WR4ep-U2)Ljod3ql zFzPqdy9GD^Bt*8qqRJtM+Sl%WTU_i0$qiH4u54DcD(}ssJ$>N?le>Q0Qo7o~r*B01 z((2;k_E2TLJS_Hc{@Vk0Z(k`>@6W&T6T4|c-|KQ=mueo0LviMH)EW|&4hN6|x1~g5 zvsdD=^1;>e<19uc8CbeulhLTr(YAvug_Mt%@q_WiOdNTbPVWD^B;oe{z3TnKm2tAr z>Nb-RRn9ae?PE3k=(ocS7J7S6tLMX6>QEdyr83*VC;CTa_ryXnGJhbl-=JyDiOWw zBw;D0R=|-J#bb3A=&F3BE@hhD*WmMm>{9(%v5>qGhc_I2wFntIAZ`=AfR zfq)5Bgyk-&0(&k6{isR7txE_p`ReJhEGwDo(|+)6aG%dAq#Cw?`&VJe#D;G2$%*;q zE9%|PdqL!g#76`ihJnFiWRbQ(XRY{it3eC~h8elrY@<{}Fz!}Lo;tN#%9`9uyKs2!M^uS*aFZR(w&XvXQ6KY)U*CkG8-{cq4> zFdV}=!}aIpY*`QRW?!O+iKpQ&^tcxtgn{i7Mn@T9Mfx}W0FWsTe$Dxxa0!#)g+T*O z#=S;b7^#=m>C(R zdupJ&ZgPIdk19l-JA?mfq`V*cB4yI zWWG-q6EI|kPormG+4oes!WZYJN6W5;U051aPd)!J(cxY2*$1^X4*a=Gk>8k9S1{(- zn2cWcv7XuQ+Y=@&fs^rtEAzVGNuMym_BMOYT4C6j?uuhOxHJBKK3i5iU-$L>9`77+ ziej&LU7+7m@%^t%D!URjj@M>0=w>DgXxLoe6%bG&4LbwNU5~>t^Yl;j8J!61k}!#E zhjS^w3x=}nC?|&LXExMM`3`{SmwRp}3_4(BKJB6Z-sc-R+mET7vlhGuSg(&-1kCx# z`C}f!%UT!<1+l%sXwOJ_+rdCb-Y#XV;v0qq14fOmUn(6v33Yt<#{hSne$+lzG_IV%U#)@9neoa<%9}goyja%cc&X z6@3|fMn1s4LO~aGKv5|t&|L}h=R9WJ-b6pLV3{1`2eu34HfuUF!~DT;;~Fg86+LP9 z+rTVrx27xk1d+FoP>d4w2JQG2FI!txFD#aSOpNR=dV2uWuk$j6tANqZ?q~7apd(|n zK=u*H*=Dp9h6hNb%D$)vT?9}YUkEJDTCKX_D`IR_ySyrbbE+&VG zNwVG*ws^Y1BKi?WPmx;*m*k4L2Yur4<-!2J)g&hIq@pU!3$M|_;mDxL$*|kf`SQm3)>5203uMWfA1Nsx06sM1>yBmpuF42`ybm^hA4JqRD|ZO z%LArKwbT?l1sod5V>RTXKOH~d8~$x{cegXx5L?zhf(IUrcvZNk^b!X)i< zFq?FidUN>Ae*_z`x*$zG?25z=)gb7Y$`Cw3pJHda!4=Ms8|+#5ZKq~?xA^M15WO)} z+zs*|IeDF7bB!?Y$w59#lEvcnT2jS4oqbg1myh3-Kt8n2UyTC(VM&c5-pu5pz09Cu zBDim)yi;e8yw!Y%IqQpN(j{ytw}J62vb+{7BuKf-v3E8sd}5+Qr_pG6+XLb4OL$B} zy#~5ywrhO$d)>giH`CReg_=f6;&XgDuJvtDGCA*xJC4VjOkZ4vX*jAlKL7R0+uhxD zA&Id>J**Lm#;P{xnC5;c7jD_IMX(A_{boe{CZqqiw;PhO9))-kMqhHsxR~8}ZK;;a zQZ{!bF-4U|uTl0MzujHZ1#(@6c)z33=zrSpF_V=)fI6}op~W&YwJ?^InGB)Od*r)? z6UJ*7qmhvSaFi)|>p+t$`ZDgzBI(~&iua=@N#4=KhWw9Y?|Ww#b6irRF0oiVWFJQk zi{ysaNCujjiGED@A1we7PHCXKEhi@Dfj_re7t>&Y;`xJOiq8>Z2wM5O`T$d6=+9u&r<_g#bSF9U}Z9cF`% z#uJnV03ZW=eS;s|qVX(kNfV3->|t>Z zvxd~cZORbDne5G#qRVJeL@IsICK7eRSfFK7yBB9#d)F8}3!|AkQ&BA7dOF47aC+bA zf!a8eJ?O`OVM_oZ&Efc!U0vY=-8&MNgSFH;CP{Y^(**585J~W4`*~nYkxXm=>FcS2 zvIM70XCkqb!^TEE=H$x_Za=g*JG5A=Cz(s|L4-#P^}0o~x;v(LGeWetHD!?M%9cJc zIRP5$vJ-Wpn&$nVS=M~!TGcoCc6(RdK=+5YEGzsP4vz}k!~?blaFt+^&jpV=_i$KWu7I+Z8x{PEL1KS8 zUMG@Lrm^SlI}n+J^|!092>K%)sE79i5C)ONj7mJ4Ldh|J z{aebK0KU~IZp(vE5j|~=h7Ux1d6x!^3^cs9l23^Akemz(wLbKw917e0+Sh_CXkZ<@ z^C%zuHvrrzOW5U2Al8w+*B=Yzr-9U!x9KPU8)^*5gS=``VE}S#LXl%vCIy8-1pxn) zfD%$t2uF(}GcZl6f$b0)`YA^lNmz^w*f=XPY;NH)Sh3_4@5qU3V?|?At|U6kZgbGo%NWU6>Kc*X}b_9 zC9261t+!vaX5pU*=O^D)KWGC+rE?dT0zP~q>)w9fLOH<91I}^$pg6KWN}Isr3*v-+ zpFsRW6d&2M&hg>Wixty;NOi(Ipu^e6vOE$d%K+5v_%T(6{lh6NJ0q3Z>mc(<@0%!( zH+iUqVPkQ2f9?TqccJDsG3hk65S%R`u~m9QZq+h==AZva1im@c5u#4lhLlLG&+dY< zf6COycqt$(HKYc6e!K}d^}wzT7V8>}!Af}8At-TS8TA{p?25nxF0K4nG_fufjfKdl z@_<9z4a24kjtuD5Gg1}lAlByiXd=0~+Zmk0KevxLo9$z>`abe5CR5OlWIed8sk*-M z>#3>@>Ld!~?ztYo^qpf8Ir|ETd}(}X4VTI3%#a67ma-81BZeyk)w}o0;r30ssi;&q z`+V$xuSdR;vn(d|PzB}13{A5fuxG;$dgh+~T@A|;V@ZQO7U zm>?<(!dX86E5ZaCA44@G5L0oacR)j#!di*>qYJ{hE8totbk7s9T%P+yEr4)#&|nC$ zL4p2q0KJK^ifo=8#e>_>jRmcK)V&mRInQqgyYNq~f;s*K5LasHQwZ*CpM-;kqXn%J z2<#|?WI1Wab;X!}O1i$E^JspqTW-qGW}?twKZw*NJ~6338+twP4j2V6%wQP6&1hwH zJ2&sxxOXMqDpCTM)bAO{jm%gIuuP7|+ft@*!|@=L>JW9)W`u-dvKxf7zH&3VD^kW) z36AZbdA1sKl+Q|nXymnDg^AW(xEdrCwLkZDP~F`8o`MT26+nOoUKPuc$SgbI>>hM# zC(~y3MMhnE78TDkvjVDOvvr{O8+{AI3wQmw@G29!i+E7m< zVRgmCI=Ckt)0Pt}Rf!X2U3JANiZi)#GPqQn0$Jb7A^xS&(Rhb<7$S+bF(#3`jl2!@ zi!P1#^_Z_y_ceT`yB}kwK*g=nNB1|pd_B_Idb-aj1)&Zq;qlmhP?FxW8>38nOfQ}! z(Oz(^SyoJH-s9Khp=-0|Akh_(CbuzcRwn$h;(4kEtHZB#Zg!5hkj3C8{QmHZ3b5lS zN@XWy9iZ10C2h0}1dF9YT5Djpgx(WXyv*XAO^6E8kXPAJd0Y+#ZKE_|Caf}_L9Kkmlz4fg}+wiN^Zm*dmh4_5DQcAhPp{jH(pCvw9t0k&T@6L254H3RlO%Ew^7 zxM64d9NZO#si-k%#7W3aL1YKfpZXX+&!71p$SYChGU3la#O!Q+`Q6XbmkHk!)Dw18o&5%yjC8G;4y{Z#XH@BPdFD%zVmQj*rZbd}4(p~MqojhpM*94ZWWFxsN zUpmI~m#__->MLal;R9EKNS6Hg>xVpsbFj1-vV@tj!Ej^`KNrhUkfILfzRs^;|7Y@t zDtSZ$x6VtH={IcZ*N8T_}KfhuIEAx)P9<0rU$7R za-u!BVSZmtoQeI#WE?Dj$0IfXZ)Pbv3$Sx_OrkPdp5LrIRUwZZ&dy>mzb0tjOdtPWld-dD<57ODtv%Y+damp0`-@>&Kg z*9rjabSY`9J(B7PaO>fJJL{s>6%x}&M>*#u$*6TCOO_nEF0egG2BE-iOE6dgwk-VJ zk3wt!A@Ssf@FkC29JJXm&m@N2)4%H^k_e8JN}(HC3IiQGukEoD)AG2C#i?ecNUOeR zn%0*~CzE1T0kq@`6ZF~Uc!?&aG$}0}?hF4Lge755NxGsP0W#G|#^)n3 zq-0r~{^*9lcH0Y)IWb$bdm!i!LFCq-=x(tpIs@#817(myjUS{-BoJqNoIM?Ahoq+9 zgOw?Kj$-N|4WS16&=V8ch*7gDhbdOX0cwGD7`*^f0cyHEjAf!QOb1*10Ohe*!$|pq z+Sl^L*L3l5FUwo2qK-1Yl8GNSri)U+OYls zZrB(!%F8aiM zWdv@d1k&&lKr`}#hs>Kw8gJ0rea<|yfdM(`eev2_V(SnoIZKi1TC)s;NK_y&0t{D* zyw{{e#RNRwqp+v4bV!-pvEg=8zJp4hV(79jM%C-bcO{ffjUCv#$%w#KxaXpW!9ep2 z7LDZ^jD~^7_v15^bhV{)D>kB6E}@HA_na#FLF_if2lw?KGWNhL@y|y}Af5n-XlFQL zw1uC8Ebv*7N5lfpJqc1fGC?+8EXzXTO?Ty=Vi)jm@~Z((p> z@a*&EJn_{P4{%`rF(xALfLa_z-a5hYz72*x zNZcGD9^KE*pYY!>w*BjYn7dt8dzP7Sm>dkJktiIddkbU+Bhqz(1Q}V(wc?W!Q));U z7vV%%!uLA$ewywrzPAfPbwzP_fc}bc!Z&~;?%UC+FSZSdaFDOYRVf4v(bV?N`u~05 z(CNOIemDdrs;l<%hs%~Nf5|a7bI?&UM^zgbG`2y0lWDwt?ov_!Qmv7zF;vPWZitra zurKrO(qn)t7Mqy>$&WLg{qSf4NvA}LQXwP-#6+f8#tU?8GiiQwd1}$pGAiK3PmpIj z^@)rn^wU|L*KUt!@hw#Pqja_Lr;H(W&eDITyFq?~d)1o0!E7M9Jn-rjjEE_B`PeS|zk z4-!CiF^CqGast2vBnqx_JraTdr}=K2$jX4&u;EWXcOVgE;uYxPlQK5`wolxRz>&*~ zpCa;OWJ53TS?T+S5j8AsI2Gisr0ufMQH5MCiQh((ntoZ#k6Nx#v6_NE)T1F@uzo^6 ze6416ZbY+@WL>YUf5N@H_N+(uWS7eiC2prg(R45N&Ms&<`qhdasCHuX%Z*RhR#|KB z3D?n?E{u~G#B}n?y0XV?qY|xtXt`Hdu9lVL6I@(aI}luvJAa3(&0KqZZ3@hOPDI|N-Jjo91y}gE48Q}f$cRHA4BZxL{@mTa z`a|fm)XOrUMmIZ#`JS$*?h8I$UES~4>HPV<48f70G)uZywAbKz=*>}owdM^!E9_f*RU@Aph5oZxRLa$ zC2H`_Dxv#&#Nxte5ZzL{M`tdUc;$eXN&SOY~kIzrJPp&&@T5k9cYRyjN2< zOPU?~LU%T3>do#V-_tG=Uv18m!R^a)NBy!NrlCv}7W=z4o>ideA>{^3~cGcRHviu!+C CgXVAm diff --git a/assets/example_data/lingbot-va/robotwin/observation.images.cam_left_wrist.png b/assets/example_data/lingbot-va/robotwin/observation.images.cam_left_wrist.png deleted file mode 100644 index 68323f37c47863b981821ab7c54ee6281008eec3..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 24723 zcmY(r30zZW_C8*_474r~TnLLI1`rSfmnDc&sHhQT69XcOB3BYX6VOCqGNsU|hyfBY z$Px%@6#*3m0STs186{{4iB_me2+KHzSnGn5w6#%$|9OL*@9+N^%7o$iUN(X6lF^}o$OhTrciUF$#kXz53r)~}8E`z-Cvx$NrC zzbn6){d_9Uz9?xsFSXUOZ`QA8bAo!^+mm;WWKMV`zY3muH@)T8gavn>Cd9@}Ri27m zo*PMbuj_Qx^ghVO^xsIU)}<@N3Mxfczk*8X;kk*uOID|FFnj@xg3EXW4wf-n>e|i2 zu052;QLA>OQ`OXY93H_S5jEA`H5zo+=4)i3HP!c0f;}wUvq;BO(o@NU)5eVHMbRnN z0a=-}i<9$hxCd6B%8sVcX_T=o@}hm(4KK9&8mUT0DxVm@4t0 zW4@$PvX-IsLiZaydCG^@9swsSyhsu92loAZB6dtPwyD*(h$EM0y|dRk|YasC1Lwjoh#!EOs&^3B8JHh9Um?g5Yv*i8!FwB4P~Mt0!e2{ zqNcYL1Zek-5=a6zfkvV4rFHD7KZzffY$Cf`bIOXo!1QL!!uwN?a^K(CCcQ9xeztKs z^fHr8%Y6JbmCsZpCDX{a)%!8FH7WlT$d4y2>~w2Fl0>qFqv5jDaz??->ndm6{X;>= z&_@qtKCQuPc!?~@9<5o+IQ3& zExFV0#(uPF;20k*tYIp6Do=0y2K6OY*_a&#A1y8NHQiMVzl8!(i`%hiib>RHA8KBN zI) zz9ik+%lIO(ciYNlY0*Vkn2l9c#%J!3_PU4-(%DSeHk)QQ5<}Im+~2!l311X%rI~s< zKx6LHE*_89Ovbu14fU~Pg)V*3PW7aF89{R?!ni{@(KE@ZjH43ayF1$=oG&Jzg$Wu^ z3a9Nwbyid5F%O_wE%d(AN@C|y1coOC>G8!OBwgBPno*P_d} z5-36ic1n=UQybT@&$!|7@zgNbCg|1ZSCY&_AgWC= zFd1a3CWbP8;?PYn5GoU2B6AhD6cE4|k-X@#P?99cj>=Cs+)V}hB<%DH+bz2=dBLXb zUR2a*oDE0|oW^kB7S*RmPE}fLE>cC%VY_s3Rt%Mb&u@U3MMMUZmRwVrd@dzoR%K_B&YU^j*>uZ;TIEdT%Z8OIWbdX^ zi6&f0lm~T^g}6AYhZiexqrHN&)iE33mBdXfGQ$+xSo3Sb1Nh8BVU}r^I!gntT|*u9 z(}$I=F50af@^XNx7PK z*Mv9*52Tv*WGl55bnY*wDtEM^XfhL4n$JhDXw*4|?*4w$8(=W~l4}pac;KKrzz8w) z{@7OJ#`O6$%vTB*T|Go0-C90u7{77$?zp?IEot!CwGbsw*x=+}7E!0{zwF0uE{OG6 zAx6tsiM{(hOy5^faqT`*9T5v7`8dVoRDP555B+H;$Y{a{8?G0g4w`TN%LcaNG%A0v zR8_oUC!J=Owup*8bR$tPh{0eGC=_9MA%Qdjc@Ed^n)%z5kAz~;8CFAUQG!z)G2q4R z6{On%3knGg__FCcGRSFl=*yh@f-Gf^vZs#17U43bT-ut*==$cZc9{+rcX*GRmIN!p z@M~9wwqY$7S!q;{E48dL0VAz^F#D1DLK0PJ3P)fZsj%ouII9QTBCVV{kCd^gRmMxY zv6TW&2!ZT4k-`>=y&)7qRCG0og~wf&*E zhl@c}*3A*HhYX%6xkF}XmVqbWGI@L?w=3@p8ie9_ zzHiu!wSbk~HyQp0{~Kv`lCmo;SB6UC8{PFQUbaV_0oQRJLs>)E4f&yzK$6r(QzZpL zstx9S%R^OVyl}U&NNR)icu-rxh+pA6FjkONvvf1a32D<;4{^+dL$tQAqPjF%|qyY7@ZEj#l+tV2y6s)LJMU(}cDXRaW8_H!yE0k#bB_ zl%72ndGh{Mw?*#8qxmZ(V^e?h7Wi=daAc$MIwqJdM6i7j*dY1WNn$mZShCPCP#EZ@ zMRn&WXN_v2r*j2W8G($zl=wkXZg!9X{l{-6jFa;LZHUCeB75BlFf5KW%7@?!1Pp%# zLaVW1`{6N-cv*zBkYH#dkg~*V$lU}u^ycw9FTP7)Z61F#=mn_l-Q+VG^IC#-qad4g znx&9$P^x$lHX!UezCDG0KAw#4t2Z7jP;D$jR)mA~dc$WB2xg4P6GJD`zEe<@4hu-S zBmI|$N;mdy;0M}aMM?9pCKjzA;QV37;C*%)0D8fXDAuhaYiu7lT3(V9G8vF12FaP2 zRzPTFl)5pns%UCePbi7e`yioBWl?1O1kg;32I8DYnl>F7P1+)osw4$d=-MC@czNQl zsH0wN(K@hie~vkZPkj&hV_3NhmL*wZxTsdx;I{LqVnVOmp1KEVAk;lL z-L7PEeTUmlo?qDh-kzY`e|TFT_43y5fxHKn49+|#b47QAV}s}@bZ{In3W*T|yNNR; zE=X746^g$;qUOkvHS=Z?6f!=DYbLLNcSpVSZQeLom##?E!j0oQP1s58)RD<~gr|2~ zu7DNTO{*p9ZI!hwHjRg3W7#$BT+%UQD8npY&|Fyr3pO9tirgE_OtXMQQYy2`t_ZC0;IIe~!3)R<_aIk+Ecpysf>L#eL8g>ELf|m50BQz^uQ}V5b~rdIC97Fx z(s?5VV=oNzxddNV{^uH08gO&#rv|kL)yMzTJ>xwISQv>KZu4&PH|>hv4MBKvI?6!k zZeX_{a3|{S%AN(SgOmFptJ|sp^~iY9RNLk&%rdxW)Vei7s>zOV1-Ale;M%An<8eqM zbb}fACQ~6%!1_;H!`% zv^RN43QF!>N?FFH{i;iZR#A&71c8vKwpfvH7z6-CfYIhxW%fGdi2R7K9kgj!iuBrb z5@HC6%DfTEAMD)gi!XxHVfu6IvIgU_L*p8W%~}2YQu=8im{rReFF%K$S?_=x3$?ppAO9s1j>!sQ_qLH!co^-T*ij z*R6Z%PU0p)xdvEHnT5=?oFTJ^U=Kiz054cbvK*G;>S-dUf-X&H}WlI6qS+~ZD z1jsy$M4clJuGe2??uLu?5tt`@$yhG3a25Llm@zh*tOQI}QY*Z2cRXJCI6*PN>ZX8W zr}V~7m9ky|01%}h(^@+|7m7IHo$1kDgbUX=NtBxbl6h> z3Rm^$ES^F$_5gn1Zw`t8 z6uNd91s64lyp@R=GJCx%=pQsy>M1c^4;l4mgCu4_knNWPFb2zegwMmq{PgRP74QUs zeRkBR`ejs)l-|~)<+$#Wh(MBsmLPbbWwrK3%H+G|b@N}eM@5xW@lgus4*W)m3PB>m z>{d1LGw2^-1e(SjLZ`7vz_onp-2!$x-WI{_mggI1^;QxLvzmPkjwA@s<={W4j_wKd z@yGg=VgUF_i+sYo-L;MJ%2;K@j#pc`hgP2|to3WKXKYjLV4k`clv;n%o4&KQ(>rr@ zY{#FI&n`;82;g@n<;_G$li!0JAcCOcO|;Giz?zCp5HqKC?7fk>P2lBYwG%2pgft|^ z(~7Pigl1rXM6Ae-g4xlqvf)@tRS{$<(zai2t?2l#2Xzj;D4GQk07^z&drq%l)=SkH zXM!r7gTWX_kWV91nlVUJE2Nf-z;Qt$tbO9HhrM=p=?eI`_rY*D2%txqy>lE4hN0bG zBlN_vX(z~zVzID+3p)niqNr~#s9BMfmPTfbE?Sedh-BUI-wjpHphMoXgrGff*B+W* z?DW>8H4bIuzDu4u5uvUNM}We_j0^?>83VXmybl5g-}l3bS%m#ENQ+@^jE8Lpnb8E_zbD(rkb~bhA)yyuV#r@Zd@7n+4%T0?0Ki<3apW1Cu zP|(xsn73WUcT1qK26W^fq+y6NVgUz=L#%U#nrkXsqRGGhmb!-05*6j(LAfROgy<5X z4*ieVn*?9fX@W687|DcaytfV7MoqB!Jk_j%5g=Dlg# zuv5~a+8!R)7mv9hYFjMRZ8!}9iF4Yxu;X03$I2A$A^VMwhwCjIaI(%lx6cFO(SlY4 z(NS=uZ=*Lz$Wf1s6hu zL~Q!IZoA0bo#X8`Mvcy$QqsLU!vVN->_7-7*nLKj>_$)@GnOOw?sJv zSsK3lVAUwR|GoQ}gWH+$N3bJgM*?3xjRD?SqbfB}y6y)<1(-2rZD5kiCtbuiAf&`Ouz+}NX(De zPn}~tSNZ7pldsy`%Dk(3I^7m=2t`7$moIuZH0s}`%!38ROo{_0-OAt&#WQNnQ zA1w6YK7>e(j9<6n&cW*i;~x4(qDg9(1lKM`@ByCJ07gk!*xEV}H3JhAIoFhlBQwU#P27a<(Fh$HWw&WQBqwM(@6B_^pxn?zhK#SIZv&ZOn8^)XaD* zY~Fi@41gF-5VwFOG=be`HzY7~I=bzabw#m-9T8GHE+6+pY|2aE)VJbvQ?<4SfUzvf$8S>?4q6Nc{rO~JHxV8rzz_=`{hRHrD z5TzJdl<}LWGROGUuGVE3lTKry&_PK=Ov}a(oGL1K{&yQuWVtrw8hB;@ix)A1t4Y>D z;tl}ZNI(!m3A8$*dK(Mw#YM!Bv`AWn3bQ4$(q5gzn$G3=l!baAYiIThF~Q`ZPLz94 zlEF>%nwk4ZMb7>t>!bKK3Q&LO&cK#xtdQZD&>jMOut{rdrmm-py*G@#ONU3X9kK7@p{aen~o?m6H)9!St0yztQlq?Kw&vMB4d`U1fOv?H!fhjKMc zANEI=g~PzA*=6ENwGMa~C}W`{QuAsn#lB%Ra2=VdY@P9J3tTolzD$J^vD|x)zZB-i zRXLjg6v_prP@u<^n%ilIhVtUc`sZ*k^Z{QY{e1DOka72N2C4y7t)dQ52f%(HqY#w> zH6Okt4@6+PMT&95yRl=%+b-s@{d_cmcE-l|zwqOXF3|CS6GO!ce3uZ$_w5_usm9oU zzxOP7%UooK0-S{D+k|EupFY@qDI94rHX()(l@Ot(@~J-iVWH}j7_dzUl9)0M2Nvjm z%o@Mj2tDtCDZ(3UyanI5nx*gIlGkk*I! zB5(ENx$du*O-uHP6J*pI$T=X^00UB#bh4d4;7ZEklx3ML9hkfocGQ3c24>y`?J~GA zD$aCkk%qyBT=GTUAL4Ds{_zj?Ntg$$*7rR|lBuL1w8?zO#~ zSPGI^its#v-8Q>pDL;GvB5?|xCfR3)S%mzb4#C};3&!{DUIFXaVfPfJ_bI_IUw`s{ z7eK=OT94-qL&Q!8U7OfqMV+XD3x<1>H~E7=dxv^j#bdeqgM3TAXf}xq6hI9JK_Chm zanY!P7PI|lf*Ue;rAqzmCZ%=o!Z&YU3tJOnoOJ3;rr~%z6Hh}ffvm6&*b@RN1O4TK zyZc&HRFw2r4=z1_vA9wU z;acqi-3Yn?nJtcrarFPmG`KCrCi_naYaegW;ua8dSDa z#e(qc0L>t2%QuKXjNKP}Xv!o?(~GA1deL2<;@ii|idcg#I`vgf6Yk#X3tH3ORA!Go?kXt~dHVx1>Bu-QOi;xtV_$&`xzN+> z@uX62@Ugz&hC2jI2$@3HB+08i9^#AAl7U5N|k22AX2{8sGF7WLYQ4ZFQ@C`09+aSm{D(mEP8U1VBu7Ln%2{MVgqq4-Gbi~= zxF&Q6^?Qq37UJM-O6LR@uzE_DO2u1ymYf*r2Vxa{Wk?kGGy2gvQ1$Ntn!|F z-FfJE&YTRmAtXC29J_Oz^^N2&eIntRZVA#Iky}6UXi{dno^j@0?K*vWI3{NF+LP@{ zCBM}&3^>FHb%MQEwX4mI3KTrg4K+L=DM7@du-j3cO-hsjwE+tuUR<|x@6-lJ_}fyH z51K(5$OXj1AGQlgCKxEqoAMEO*Deh?KI~%E><5y4i=aOYz!3^0pl#vw;-Llaa_U@Eb?4*3W>`QXF7~{~Zx( z%>6VSBY7ts?(9 zFHPnFSbg{W*~OuUf{DTWU3;E0cJf`nL3xl{Sz+z$e6b^E2fq)~#Q zj*Eq)CPV+UO|?^@b133&Q~xX}1Yw~{8boP9%>kQ5l+j>m(!1#wH)mHR97{TN^Xdl2DF*8Qz&KC(SS|XGWC|xm^r~e$7{^{z9#L`1-e<-{Gya+oQ zkwWD<#~J?(P_0w1=LPh@zzDqYAsVVnuRyU4Wx5@70r}f)x!nNX9042=3^zfVMTC>b zxTwoCQMcC7pgKXC={akUv2?= zAfHr?Iq81RoqQI>X;Z}=-YX`SRR9%v@{1mG%v4M$sObtS>fA-rdw>9s#Oj3rOT0?0294qlAGbw>x4A(XfHz8vV(3lOotgS=xi~}N*mi|k zSWAGWMLE^A_d~%5KmydF?u?-jAO=FU0^5L$mI`m0DwouURiL}=w#GWZj?0a^eLxEF z;rwb!eR74{8OWH63=1*+RX1xZsVRLDo5>x(1VPxLcp_vTh0upxnoPMI!_%oXEp8p* z{4EGfAUXw^J4l|vvhJ!PKAXNVJ)q3`MX@L>vJwh2YS6_kjf^r23Iqw^F~m^Fc;i1G z8v5BqyidFj%F2BR@?DEaSETN=;Zl5QP%QBS0pFqaUbp;>jT%vca7m+!<^r+KbSULI zB_7nCl)kL2n0gwTGB4Hx2})J#HUavAS14$x5#25egvzp;;u+J39K9?~(i7u@NA4Am zQG!fP0|4qBVCXQs zw^dq*C|!D#ogyu$CHSz1oH|ZKfi8eczkk^Vv+54?_1UY5o*Rbz-1IZ28*u_U zUmWx|Xqw_c@)H@IsLHUK>&(-KlTSl#C?8hBRg6KMza{6_`Zp8r?VmYiktFSL?c;j@>==|J1njnSc;Jt zidwb>Dn4qYCQ#gv8j0FJL^veVU5t271ZbBaX+VgD)F&_%&{XBRIhyQ7pck34nBCAb zfiacllVjl^%844*!TbO0bY49#4qVnBtFMi>TKu7PH&Zjb!WV$=-!KsnOJ@4N&%Q!z z3<17CmD>TWTsq;kUsg?p0TK(%)d!DQh9g0=OgJuyjf6X@36Vk^$reAPfCrl2{*66EEtC zk@N&I4L3V$LjvZKbTxCW{q58nq=1CS-Rotj;Y6psg8bKA4Yl_xMJPoLW(aG5)x!Vy ztliI?>LfjKxB3JD|EgOJOOaNg%D8Z5v@0cuSVx3SGDDQEVmuNWL5tf==fLWtkq4aC zKK4G^qlKyp{+NkkA~eM@a&?_8bn&5Q&$lFH0PTp#Jbvf30$~J11ip9C+_RF4yIkOy zzKhl6yL2`2J{C}vrp-*?K;u;Ti2M7=&!u>e&?XZ#r7XB)vf0#MQ)32}v+e;pFtOq0P z<_Qy1X+#wAeZzc;xX4p+T2Sb28!l0uT%p?UgZgalsJELhTrOswM!1uAk%=tuqGTo} zVJz#WpCQPj@(gV(rZr#JVto1AK=FiMSY+#9wR=Xuaf&88{_OnYB)9~RMAy-s>TM1> zvW_c5_luqV4gGS6<|Lha94Q03x5(Q$?J~Jao@HzNj?gM|u^^fFrX+DO9^h0SC?0TS zT31zi+U2`Jp;BVam~q*fVO9%N9IRfh{v`dfo-FB_9*C! zjkT-K1~L`h9s)^EwKv^ejO*O%R_~MT)6fkgFzC7Nf+c(CIME770OWkeTI!ATWSV4? zj|0ce2a!PmCYgZj0%YwH(e4T+3i! zJ+2lc-8IH?|HTO2P@t-y8_J)^mHqw7*_e-kNtD_+&XnXl8)0q;D*?KaojX*k~RsjXbMR) ztn$$98+8AfpuaEiZB;$U=U&wNFpG@I;;7Z$(BsvV`YY+yU518do)e?ovsz2>_Mzl; zm_q($)n*gH1!dg0qwdmA3n5;qT4oDimF@RTLf3#{1>-X>mxK* zRkOuIr4y2fW|@xhfXCVmZ3+J#p7av5|Li2jQ(eVVZ=V#8!|dvAIp7rheDsaWP6)j? zxh#w0_`6fjdN00g^Pd(>e_xI|FBT-odtUXBsVpFC`?3c%;qxI}DEl)pjvuK!&_-2q zOc=pRuY)-;souG#aeVc94hdH8J+Js3W=&+G8FpG{F{f<8okM>NXqU08gqY zZGiB%i;U~y%2%609EAH2ezxV}WA{Cp-ISS?p0;2(dI?cS6~RCB-=%QkEB}> zV(#Ogv_3!}s)X^ZG2<~9SmF;umH^{opKPcss)VW6m`Kh-aO7g2uI)!*$|&#<+pkBN zwthTb+f^WYDrq6sDtv?PWPO8V+@kJPuYDG{B|*5JpIP6 ztClf*!;}v~mnTy6VbEG?Q>_DZ1-(t+kNW9BznF98a|(^$iE@`>HHh?w34CAuY(ZdU z8_a2M()8+R#QEwdgu4=9Hv&a1oZHX3Ca~C ze@_#!Ie$9oQdWpNs&V&i+`(kUz4ij<%$wzu#V`n|aVG_L%T++$)AB9&lV#k+x7GV7aQtRl8t17 zC}hZV^oqXhSUL?LovZf9)IEo(f3NB3ym)I>%jejyul?+Fb>6Lt;1U?Rf;`wS*;fwp z&+Rfjv@fIZMx8^SX(5QC%>p3(8mUWQIMQ0cAJkblXcvGZiH?Z|6^O8wWv9ymwJfPE zq~@Hn!ZWOEEB(i7jj2!}1Zo#>eCk>0AV{by09Qh5SY*LSHuR>J2l4~MH%GFqdGogfw&=? zp{;dj}|)n z*Nl{^Hafz1pHvegcjMAYN#Gt~8vc>l7qcr^@ccvsr9UBJR6?{Vf+tr<0RPCl@YB17 z&vT&yXqLGw$mjMYtOPr>QStojKqMS@2LKyB?9D#)J&|ziGJzK)TTGKQODu_axZxD-_P;Y&At{`m9y%BSG{PS{f#|4<@rJjd`ZX> zm`5sU5GWFQvR~nK^SevMVD~kAB7*^ZX+Y1sIF1jlt&tXR9jVHVbi;ijm0vy2JFa&l zL|}H?aTXvrKTdKaRL2ie^RZhz*7kYFhC(B2gaenI{ksoTcHO)juSA8Vh2hk%d$dVV zWddJJ)HyQpM1UnilqDiSc23xis>%m9dBOpYq~yG*M_p6TSNVF6&&Ed=NekV?QNOw} z^7mB$Ifi^%?=0*f*2eqc%nB$Q{6(pw-g>WF;eIV{7KRfD9Qy#4O51p2r*FNeUHcym z8A7{#Bfh&328Sd`F;wQLuRfV!cpjjIsRV!ISKxGPKMUYbY>jnu2k@z>0NNFhIgC zu*16gRCKZ{E^;unT&RRHn29 z4nbuQMBr5PO_qof35Mnb-9oG$V-9sq{dO{HIDN0Ia~VXMwDLFkB3w)APF+NTQISi8 zy<*kHz*X1Psw3GasJCx+y0640;Hjk+E(F0joJ~uh%(FCH3J~a1=ffryci8?F(9jnX zx!zeoc(`ZLs7nAh3Pj6E#rQ#zj;tdiY>I1-5=gginqw8H{YRB^ngdkwuj%mA&|Xo8 z)Cn4pSQZzR3D#7lvja@}O6|BkfrAn17m2xVd(BAoU%qzLMbA+KmZ&lVs{f%!9rJvA zIQHDhb8d%#SRkOE*2Y9)B~K~p3)Q-tJgN3aB2c2Q^jTG5WKw>>Vs3=8iAMlNX0%9X z!7f1bGJed0`UPeEqIWhUKm)&J<%cbs^?~Cdnc)m^`edh zot2@AdreE0dDK_q+D%I4$-d$7BSe@~R?G(Ui+O5ca?`usco?(5q1UPNJu`^Z{FFMV zy9UM4z)C2?RqV=*&Axg>z@F093gZNw{)>#CjF$|Grb~&~87#{vup-H_-6{3LJ{YVa z7l1CvSQO|_1}jPlxOQ#8czaa}l}xJ7KG8ui?AQ5FBaLgcrDi_UTvGi=ems5kb(YD# zHn#e*{fBhkEz(MSo5~8egGLY;26ApDgp_pcOy?%$0sgByVP+hh6EOZG28oI2sJoA} zzZ-NACYe%{rbrq_#kfMMVafff88ypif2(s@Ufv$nHNJv{oVHa{4ukgV@Td6jh zUD2VcA*@p_rw+?UObSy6gRKG>z*{bqNd+dx)Bk5ztgYLbj)#p^clMb_{L1+0{!ZILQx8<(2!z9B@JVMXFzWLHHg zcM;V?ig_XAZhJEvY$vH80tPd{n(g;4k!5e=|FWtY)U8kMUzF}yY}3vl+r5*2~d{^nv{W}u-!21 zS{lJkGF(^(8cZx7(5L2X;CpdhbiQ9mHQJlt;{$R!8S7dG5jw0sLLFE!wvV-JE2WtX z{=`?_p=?aomH=eGC{8!*9RKC@8YRpBjQsvh`B+-=Ua_<^+R+%1Za>UB=#|C2W~ah? zAEQvF?d92MM2}uP&6>zQcfm{XwSPyFsO#SF_|Ka}!VbsY5qa9~?Ll&S&L*XEmLbF` zzxV4hGrN2kg$@qqAM*CGP`Gq$SkfjA-H4MM9V)|gKL_^eqM;Z6Gi$^&>fb?l;yT_g z)})^{g!7xsPa_+%T|@B=HY!x$5=kW;1j zntQR73p{{bns$MqP&D@ZCl4wQhGwAgo=9O=76Xx=z>eRliv{IzU8o%es}`^;ij_6Z z%YfzWJpzE?S4q0)QICvQdwi1d^}qc#x;5FtoYrOOOWoSl0pr#(S+)m+Hlwpt>8vxP zx9`*^%Wmi7yt%tvp^H|Y)CgHH={yAUBg+At+eZR7n?$c3}!rGSU7k*gm$ocuG_#*2me&Lux<3IG*?Z8-#$(^#AQA) zx`1F9tqU9)X63B?oK9sLb~`$rY1hoHaToOja195#@8~*9540|(}`XA z23anA=@qW?4Z+xj(Mz=6$SZ+}wP5!i={_P%q{tT7alOiEkt=j-2bHxjgriNL+Cp-W zC9o@K3Qq4xpy(wft6=ryB_g*6LXTI(vg zF-yjOEaiGd>7pzRW_?H8iYZbhWUGwXSixcOwZLoVzR8gPuq)AFSyL#QYE}qH}<(-gMg@ICnp@8e{sXdaUJ%Vc=DV;6Y z?Y{Ng2vH!<5hemlRh$jS`bsMF;pyFs?!eWB=Y_P$-ZTiy>@U*tD~;cu?DMGa^v9Pl z%a&uUS84>njnT!u$kb{?WB7~AgKOp%Sd+*Cm+!hxFkE8MYbp!v!}3A%LAygGJ!Gc#Alt_Eq#Df>0k-bqK!XQj#99o!g>n_9Iu1Ah<* z!syi{*Vxpa2s_`)kzYmX>r#O+czH?KiwnXOMrog%9!b(2COIv@vd(r>hwF&FCxvn0@JzeySs3R* zR0V37xJcaUt0C@Gt(NS(lSBjfNhi(kPQX`%2b2jnn)$Ogj49bZ@1n@8d7bhIJlFyma-E0DZDPY}?C5p%St0HC3BkH^!};-<;xK3E2ohob z)(hXvz6w~KU4<7O-Y=?=9 zbQsMRa;yz2APSPD%3aD8_|!*qk}B;LyzvS@l)At%3u1izpnT+K*Y+#kL)GU1vk5z* zL8l9z9PqE7D;^f$_WYKl(v9F>>X_S)*HXiifQzk9Iv=m!bGqZ;-@OVCdMR9B5#Mhz=xWzeth%gAG()fhiUy2 z%|zGqTa{hA-Rc9S8iv2vbMnjrPKDXUZXfKPX~JS22YP#z9o=pf!-9m{@{^Dacw8?^ z5A}(BHs=R|2Fx*ww18tvH(U@6Twrn_diH>&tqbBg0<3L$fIUB?5!F_1#AL^ZTn6ak z>-Qew zL>o+dA6gUAHTKGkvW?h;%CmRGs~dV+`rQcS&vYAwPeatJh7lhE7kJkUt2|7Css*w${*afxGyLnu1TjNTcJf1GJChLDiYZ5r7!MnUp(+E= z3UTMmd|ITm!x7<)U^KNmqeQCm4)qlhSLDXD^v4oST;h+$S1&E5f9q}^`{vE>V*iej z1f6%{pA(LUwZ#XZC1H9R6#!8op=MeAV1lkBPU)fYV!N+bI@I8cDiblV*y%b* z(nLn?WM6Sc*YwnL|BT^uzVTLYi$b?b|8zUQldJ=p-wILxGjsUxWaJOR!USEb{#F7` zZg#SR)Ol1Y8>|}!ABXYNRa>Sav99#och3{m2M^ibNq_gpOO3-+Q&;wz_gRTu+$WiP zLD9m)_QsQQjiJCl0j7(41)M8HF*S&ZM8!gKc99J>3(E(HQZzhomkBM1a>gsR|M9KL z6nNrjgVIwyHZ|VFI`wx0gor5!5e~VNI+b`No$R0Z=bc9fwfC+C0-9VSs}A8aVe9}s zl1&X)I#b8VCe#PYl+$DK`NDfuQuq|6MqF#2$(uxGB;yDY%*)IoaR0Z*x+ebw;pe>n zLzU?>oc*{<>=ZM#y|My7ZF<5C_8$F8=c~}|?RCq&hONImq}xk3q{@IfmRaq*^Vq<+ zT{mao6Hkb%&N?ep;_Nr*H2%Sp1I1(0uiyJ;bRF(ci~3;QrsJgC?}^f657QQjPfu9X(t^OlpTK@>*!~4gc z!8z0K-+-+59#_Al>;-ez9JHSbtc2GV7N1V zo}sYXJzsV4kKE}=10=wKtg+lT55yV2e8p_n2#mLKGwvMxb6}2fJNNS*Ci9(wfgy);`2LCf3`2fWH0B6n9f5It;)Wo~nT4b#sA;F{_ip62JZ7M0wUy6a z2lS5U`rj5$zy4kgp&(cRvI#GOAYe$~i{emsz95>Py$sKRPHS$butVB6@=IoE$K%ZT zhVwv`FOHP*MPYgDAzlCKsqVn*##__AAQPXJ6;MlnULaDz%z~*i)GDHFAREGf2r?07 zS=e;^2)jTvGrtHMN!1?JrW3C`3059~q!z`oKNdUytOpU&n&}@qfzfctT3CgC>9ij_ zoaL1zwHx_qxCg@CJts*wJbsX6rnrh7fAAQc|H8QzR2)&29`0SHaTKkT>jra)h>Q}! zzx_j?K_CFiWD}sUPOsc+X}}!JJWIq`ogE4P07u0(Gs98OYS2Y}AsUc{!V&^HP|$Jm z+)81hZ$43l2f0EmHGU1rfXF>@b&z;vo^E#Hs>OF6N0kkHm^uXDfM%4xFf)iDlrINt zk#j78QVvfqfqh{3vNI=rIaOf|M~t_uBSCPjbYzPlI&`lZVD9zFV$*2kD@EOjt~ZLGxYxPxWDh{He>bp54Wy{kfC;_~)+uCG3xqtFK!Wq*C!SpH~yI>RDz?B%8Bfpde;w7s$Ugb>|) z{*?K)g4q9j<+$BD4}O;in&OGGDvY``uDx{$z_%gFI;Yrn_1rCTW&F@&2p^X$oM$q z`bUmSKh3*Jy|wy8SF-ui7MJyN2%m&}|M#e`$V-=gnp}4Ii~Z-k_x#JOb@M~9)6#~# z1JNO0nddG#F?;^4&&(eu2pcw@aFZtJQyzbu>~gt4@$D=JrC-8lv$Q372WEw=Dz>~! z`n2>@xW$PB3qSXEe(rhM+x?TjWqaK(X>j{KFY9x3s#%VmiC;~HoBP1Z@W;|T`2!M9 z39pp$4WA{uEX`fS_;@}2U*B5jY=i#B-nQzDB_j{^d|(OM`L`wLm!;)0IM7lb_|Nng zoXgd5E%}J$U8|2xtL7ajw|tm)@xmP9fq?1|?GqAQ_9;kdi&fr%p#Oad?*BLtT|e{t zuhnjG;SLP=Z`2bc;6=)do#lu&)OSAhzmMjBHCp3@whb#ql2No5#^W_sAm5E8uh3{%rU| zMEDNg2>m!@dldQ_h&d1Tb@j&j*(VwzK8Ev{G-29}>FB2KyMB&@W4etOlA!5myl*W} zaGcH27B6jymR?$hEYb3oGhBz1m{0=WKj8OS-nYs;o+E53fz|lfUHB2~;8u@mEjZ2h zPBTlJjNHofeQw?Xn~&M2zeR#BJ{HS99150d+WLnr9R1b_j$_&boNR-+!>+H)<>6nM zHnMko!LsLJw>ps8L2)xSvgL!^K73{79RJTBdBslQ8_?YNN zE&bGaExfnjuLVl}y77!$&;kb{E?Q~7=#7CGU>^FN<*0LBnB{jR-&>xT`D)&`Az#5x zK!fPN1IPc8!Q(HH602%HrMrM#!r?-`g^$+dedV^r((;oQ_q<@A|C2ZpwVaENC6Aw# z_c~{m)O0HJcTCLR=3K`X(r3*Yznu4m*tq)h|Hp7(#ajXJHO9wQWwSm>Ulk&9UpxD| z5*~=L`+@T;6gIvx-+b#R7z{EmBu)R8kj()rzP0@HR_wwpCTBDWv1z*@aaDf z{QjwB-hnhYGUz@8ZNH#&*)q$F-x+W$*d-<8RxwJy>7q?ed$EjXLDu^9A26?>rwfo|NO8J zR<^iuY1J{yrE8ysY~sl;{Zu>u*0=k^Vh?^<1Rtr~1EM0@-0ku#L;J7T9{zlm>9hY5 zdM`mh$ekN)^ZIO>+r-67wB=DZrgY+CV?izbO^2LzrFXoXpEZ2&&TWC+of<6ZUv2f5 zuAc^{IF?fOY2J;Y`|ZVnxqGI4I?lWwSkY9yn%ffFz#gcJ2%dawF)@1R&E#bEYR5U( zB{2&fj#`?|dh5=z%Y~*5t+RGHIVBg8zJBvk)5$e(-u>b5W#+ZFH`;P)AAFhkUUO&3 z`;i|Lr;izXY8z*N(ijli^Sj&U#IjiqCFOX;j@;i~lJ316Yd_p4`SRW??(?5C+m39% z5&)VE19mN)xw%l8SDg3@3O7LuK5K|P70CVNC(XMvV_kdi9RKp` zKb8b#Uw?bU>Ej1)SOX(j!BZV~y0~NQ*6H=4|lBFP}mYL z+4o>U+X(mR)BmrfYYj^(-NI^9TFqpane1qJnXybQ?bIlh7FL>>m8a272r5}a1{UTm zub|Cj=5!28(J@S#3@_leo8T?MvDC!UGR4Llo)k?<2@yfA=X0907)Xlt zD&&*}oj%SXLNF&ircIPO-^W)go|_}JlU=FFp4MwX?XE0eB*u-k<+sgy;vXNeaQPlo z@OsbK+?WC}oH$ic#=Y>@<&h@+y0!rH(k+|5XQJV#UJMS8j&F;Qs#;tr3p#0<;ikvT zhAyftpzA8OQC&_>-2ayN;ElH4E0ute$iJ{?k8aB9p)@t759~t2c7JYe&vjtC(d;W@ zj&p=HJkC!Iaen(1g#o$`6NdTc;P*3689KU5Yu21jzp_|LFEfjCG~*au%nb5$?FC_}va$AD=)3J-8{4^o9AUMPmHw7BGe3Y-$Qim% zrUq>nLqqS%o@fJ!MuVsY+nttNI-c%ghneyy3*g6n7CRuKM33d|znU82)N_+Wc$@c9 z?+zA`EUH1}NZQ>gK&!0;6p$ zE_TL(y-C`!m9D(nivz+?kA%kgDd(~P#@i=^>LL>s2L(sL{Xv8Shf7G{@GNdn2(c9H zC__6}hSX^%3>}G-Rn6(flC4fl?%C%AaefJOIAOi}ks-^0`-!5SM1-Y{zWVvT8{Y9g zgn}aBD6#?)e*1e|iR^qhiSnc6>+P>}BO0Vi?3%V9Zg79yMfGd@5)Bnh#1eF}vkvpq zDI^US?(&p>`lDv}&ao|+e0=Z`$LW2&7kXJn97%nhERhlpVmXSEMy|A3r+kW3rp8mE z*VuvCg>*lL36@-S_C#l8(O&Y@8+zvaw5UPe9%{O3dfuVaaUtjpUD*z~?R|WT z)JFeVDpIvJ-tb>}C(dibj?}|Jo4McJpqgC|UDr>=+K_Olf#U zEFz~>sJ=sl^6*e_e#V22^FLzHXf!5dWA@=(TD!=gLuxe!)pG=2_PT}Ha@)&fIS_`m zcB!B4$HcOQH*JvmHVeJ##rXDy%lnf)!iO$0AF|Ftkz}_H6dM!S%YxWWFbbc9f^IpE zx5mola({pS*x1os5nFJY{yx11kZC%K92uRIl@K2aiggM=@t7y2Zt|Gd@^thX5{1}S z^+P5#$c>GGMCxvLXfY-cT%0T1I`huhh9az~7VFpN7v#5%zAbDYlk$?gMybfl2C~vs z^rZ5(0Hu(9m%LASGGB#ssoK=R+qT+Jy0o98%Bg?1}Xj7WoD4|7{%t z2|Fhkrc0>%W}6bMYw@FSJHNnp=;N!gR|V4kxdC5_{7r&QzbXssXA*byAEb|+F`Hnqc9n@1 zfQ8Sz=aD7cO{O@CY_1}3J(`&hE1b#vZnY4^2P8iJh!s>Fva~#1aL0ZWaE}5|k4_>~ zSY>a?Z7<-@F-6+>Tbb6$=SMGEdF%y{qY)?T)&y_9x?1?Ya#0>URkQPOz)XLgL|lL? zE-Ji-55$T(r?4Un4F$1L5C*3Gwxa5QamKy$(7x=p(Ym??CbNaf6kmOpyDNb_S>oZ9kPlV_#L=&di=LEE9&*Y%H@@n$8K&yQbw9q29O50l8T|s;pC@`N;E#$y@N(! z0-QKR7^;9-7mh-N3g~phh{qW&TbK$NC_n~-YlugRqA5w}^ zn9Vq{C`yh_h;o(*i`X5os!=8!CZ_=7nb+!dZB*-rLRKX}N2r83JeVbXf!akRKzo+F zboy?Moo(RaB4TgcSn2QI;qQL~aw{yV(>SLoKc405%1n7F9j#bD!2L}=;$=|2y0aqi zVdK2Is;s!58!xvpsAk9JCTm%6HBSMvq^B?lJ$lP;6>(1!Oi6J_@r?f%(kz^y{_D{*zn& z)mY_Mcw$fXxWzc)!uv7B{l_VH??Bsn!a%z>$KJG+e1eY|R zo~tUmxp5B$hTJ$9z`95>n}vcEy+@0P&~HIB6M^Nw@8zej2YbXjZqFh;ExTM@{$5XY zUL34osKzcL)qhffKd0$GSnzijGk)E1x@qrk0>!H*JKn@kYG?>iYKn5EtcEmHpCBzR z$)s|LK^zp&W>A<49>jUVN(~@DSugWem>Kt7vd+kHEH&A=DAyK<3+S*AwEbK4tB4s@ zJC!o4vq?2{dPp~RQUA5$G}%X-T&qhKt_NM`$hH+Gg0g(Jwzs#c)pHx^wBo( zL;%cz-ez`Aw%6?K?{2v55<6O#(UQSrej}HUx*XuCTDlC9fHeJQrFFfI8xSA+1E3_1 zfby+9;rFfpm%M<~Po#O)CCTZT<~8jPI?7p6jj%>oGc3gHfgmCj49+@Mk5iMQaBJk+ zJjczoMXpZ7VRCz^KLasz%7e%-ot z8-B1dvt74NgaWTW;4tuc<*ruVx^;wgKbRf13lf>E9|}q7u6OLHX*ZI??QSvnv@)pV z`(M}RWN}`PRm-nqleK#{{l+9R-cNPX6S{e$rmLD_M^$!5v)K_|A|)}T7}2N)Y!%^2 z0?QWH5{1E&ZSmU((pXHK^6NUe&_<&qn&1~qkEG`|-VcpNhj)*UC@P$>@%{-PV&P*v8ZGL^Zsp zroGr3ny(nq?60E|Mn$v>aU%IFj86IfyauOyJ>lx2o%D?uy!(m0IkYh;?cIp1Lh}8k zK#U=7B%Z3hPfj}pLrlT+Xw#e|XHPMZmg5@xj>MQ+5Xz(33T@s59~91Ry0 z##D|*sNy%|*tiD`hWt-$88a?n!sRT}EZ2Lk0>5k5uLWN-F7Le?c|YgIk+Zjcc@$w3 znd6~yB<)PxkeAbejL~yt!B_ldW*ri;Oz2_4rGXCR&j^jeqG&&(itslOO`?7hh%_k+ zENIC~STT8GHdQ-?LUYn);T6*)Q|dgdgq`+;6^# zh`}pQzS(ZD{)$z8gu6f|aglq3F=HM)h?a60M3YP50+nvSlQ- zaI-E1f?0_Y?(qItV-W>eOy#k-iX$9PH^Ls(H_PKTro!U!!Mc!@_OiJ$*MMEczxPfQ zOgQx#*2ip7I5Vouz2KzS?0B2pRNAv6qirmxOm)kaej^k5*N^p|k~M8?)CLDt^*$=! zD7;FrePUOLO_5=%%&Ke;$Erx9W;`wJbWHh9w5plA=XR#3%Io4>V{VO?bUj&P_m#I! zT4wH#QfBP3Bl>=dE5+wnAA_&sm47?_c5~Wnsjx98!PFXK{7hL zjfACYiBSoS$CKL?C1z(Nbr_`^8HkpG?~gRQ2y3zif=RMTN|g?_bd#!|x>p>&{HaQ< zHhg|LYuu+@Z`Z)8UvTjCn*O&_3n50jB#Z*h-b1P*C%$Ayt7DVnfs8X{m*V3OG&fr= z#wTnw4*U9bqBAeSW=8F@xTrFIDDeb=Du&(H^kE}ceZ=9HLJV=$Uq>gfE&J6L9YVHX z^B&c(qa`s~(sqAY$F<-{a^QMo3*RGrimH6Apm~t-K04jbN|)d*C4ECHz;hd;CB9k- zchsK6`-Y{u5zI)!;{+qJD278}FLHMg?PtZX%@H`}d&wPzOAb2TTH7q#y}U>>w|;qo zw8V=lm)@VN8>r^HnjQ*YS)T9V^LxvE^on;C7ngVUPE=m7r?N1wt)q;NG?(u1*x?b^ zHqhbaKeP04X=$mTcq@Hw9$i_2zQ=AXgUCo!#>wIs5lFP)$E21I1h(nT&_0Fz!-`Q% z3xX-Db#LF+hR+JROobBy!Q@n9Z$SY-OurFNi(xTD1RyQN_|eh|lY)zy)MD9I zZ`8NL;m(}Uf~AH&MJm-1c$2$AbP|q;wY?yJcT$^Wtzk=fBz|F=1+2ENL>+rn$6ky| z$#&ByAS5Q-{TatxC-O6e0Kb=Lgi)vIJg0fM}Gg7KgaJ`)N`p0Sy>iLff<$3930Xd(t4-*-d=7zuK)gLyDM?# zTRc=eAPUd)TsqL&(J{8P^l_kOxY?$5xwSLTM(zGqy+}z?c328x#6Z{)2rfw(-Ry7! zBE@{vf>6L-I$@xZxH;8UM;>n=AFPMqMt*He}ctGY@P8D1_mFq zlt@5+2-8Be@XU2Y%Fhc}_f#Cq?;#D(q5!(GafK;dVFg*7R+zF`jAL!Zu8H6-uFu7Y z5T#S7#Kg^FrJJPP-nEaPZ4VdBc1<{(9UoUxQc8n`IXLuo^}WShz?ZR8AB%2@uQ?NU z#_2$7sp{qU1D3<+%2pm~CS(cbKk4TsY*kCBkALlhlVl)j+c=qsHRH9?&rZ=DF$o*} zWVBM;^-^qme(sUg$);Aekw@YwCL{!tL&HC!5E6qlDcY%2_6dRYz-;5->h0*mdkO5# zj7iDTjU3Caqtmv?dNGVTSq?7ZZaLj-u24)T8+FwZO$#q^g>W_n&lY_|QwP}msUqr) zVYrE@aHv_$wU&OJPG4P^>TNB!KQZ-IgG74Z#r{BxGOz~}@dBiD1Q5x1I0`FS{cfX) z&bmV@pI=#sG0kvhY|1p-3^!s-l=jr5@xw?@*>B?P!_uYC=pm$}6P~+OrN9{H5~@^V z068_#@Uhcj$lr-pfIHKd2N&b&b8wAnnZ`dd-4!@n7!0X^8}B7Y9L(G5i#h8y$YZpz zBLsJa0yc9gJUs$0LZz~(W`dtE+FZVL*tDm~@phYXtJloObCrQ%S7%mdlbaKS&HCZSE1zeZe%WYd zPYwGO!TF+&1F_5NaXy?cPkqK6X4=stnG`Lo=Oa_mMsjL7E4Z< zVN2=Dk;mmA0PEH{h)MXuQKFpt{+{UE#1r}Ii3EB_Dd{5K5r{!~PYXqb=41z}>_#a42Oa&M@n2WV#+lmaT0R9;?c^RI( zJ^JzExiY8eexn!R!huhphU-jLT0cE(&LH3WfDN`}QwG+LJY*Um7_pJETV$=2`z?hk zStM@EODic)8`MQXob<#agWSw^spgvj#K&waERf z)73Q<)wM5PRKLjd53RZXb*m8zb5QlOGNdYx%90+{EowU#r?+I?Q_afET%FJG*=o9U zVK@=zxs7AZ9)v$E4a;!194&j2?Wd0XJ*+OT{Q589GV29rZ5kf-9G!yMao0t=Q zIA(=8j!BEDVj}7(U-)HbalRgMMzSqBSTRv2F>I+4f)m-h;JNMfYaG*T?4Dq8&%OYi zRmXooT^?VezB|SJz*=yOCg{1=9W*tq3kvlGce-}{dd>abiK$$tXQF1z&?6n>+h+=I zmjH_SKkK;^fBV~Fg3a;itW1;DiB6x)LrWJ_4mMWEwj4$`RRaC{VH8*an^Ede@}Iw6 zT6sR=onqPMe@vvZIaWnxclIvad;sqZl($~XGAtl*vp>1_u}?7*jkU>HrlCzclTY-? zVZdGVQB2t8CU-a@;)pbcb?4Y>sI15tagN%yM5-i{TKSU}3(+!Qt}P>cbuBFJLs(o7 z^}Tj2IQSZ@Vru^0{`L8iX3QC>X2+&0ac9cnN_sBcKI4D&SzlGS;KRu3KmL>~k#L)Q zujE%5usdpc)K>`{>(RN&W(iE?KdVEmaw2po(MLTYsCSPSI_DfQ*@@r*sU;Y+Nihqa zE*T+t%VNm3){jt&^zDfr)KUN#a@%f^El>5r{+GjGAO*CHy{o-HMYB!q7)i>Kj7`x9 zPBwa1q~qoBFn{{$N`jtC4gGomo$l-V&oWN8-Os*(&!=6@2U;MNRuXsSl0AyMx4&sN z%dA;~50w2g zR>q{59}=CU99^Q$NK(m>x1JRtk`uW%Z3*JG1SWUxY^E{Gm1ph;sEBPuc>lKG2zu4o#ZoSfuxxoHIjKt~~x-%kCW&t{)sg&x_WfBVc1 z57>8bqK9o25XF%ai*+W~d1-lrVe=n4yrf=JR(BGo5$+M4x-<_+f~qJ1Vc9YZ!J=yiuEi*`r>^Ek(IC$|AUR%Xk z-D%8P+Coa|y{ctS9v6HP2>7tSD|1~vJ^aSra!6_p-Q8%)?mCd4P0_M*iyKmIb==wN zc>7G*4i6{~+Wh?dzPrgM*V^FarI8M=7Bf*I+(cpme#E#Z+VzRRyMciyM^Bo&YgiC8 zAb;WjBSs0?`HoMr(R~M^gdzTkCnp*XZ($I9qY$>N5q{0fbl1GSESkwX?w8T01qm45 zIj}yLhOXm8A&F}f#-PG$r)VwNH$66MNqMMiq8pO3&tPrM$6REi+_QhOL%9 zP$x)HdI|>eqn#MyZVb!O+>T;4sr4ad5r}Ptxi*V}xBFwfJ=3?rEVAZ`w?qR z#9HTIsYEk(cR+R~wQ0vH=WI+IZT;w$!wX~gOz&qI3uhY(x{KT2zTE{M#ZR9;EiU&T z4>8Iy)TBL0%8w&oErgwpYr7B!2wCQI{P?tS&=tRs<&k2WEIN2b(-3y}cgi&E0e~cy zTCLQwQ%5Mf<+UZ2C(E#7!$Mj@S+*Z(Opa9?1&8%iP=;?AX2{!WL1Ke?lnK1m7D|so zY%YRgeEFO0a<4b^=q7G1{Taug=JeUZPH4gJq$2rd&tSiAQ!9TYOOH+F&6SOt)&|Wl zgQxT~Ha3oSb#=|p2TWfYpEQ=k`0l@X5JJ$Y$!q`3eF&7?v&|0%id(Dmi62Fvvg z5eQf25f-cCdM-}3MQwZHW`=%+tdXBO$~E_%EyFNrY|r*B6pB?YcO!G%bHQ}aUjB)r z2qsu>BPQt-Y6K#gM3w_8gV#nd`{jt)jwb}sHZg!cEt!QWj%~}ICMToRT&u%YK7Wrq z-sjID`+fFhv^|Kx^lkq9zGuhgX0PY4v%}}gDuo}GhF(P$+o%bFWGASZY?9WDW4ns^ zSrV{3K9))0Q^AW{(qd1bz-LC)A_#NXu<4pKN)#6Irn?4*fl#v%_MmZS)0V@NEiT$v zYk~ww5D=@d$^X2Dw1vruuopy!TW)YBduV#Cgas@JMq`1n)CJFt0X(2LS(V;vc(&}^ z+jLpS%E)-Ks)rMZ4hISV*y0{OYQq zh5Pl0qe_mbfA?G?^dB@?Bt}wnn07#RwQ~Q$<N>e8D5dU*VaUEUor6Vaa!pT5m zAlghCy;nX-$JHu|N)Xy=$hF*}V*|Zz=husJ>3f2&U1gy1&{RJ@$06vvqN5I+%q7ydCl4Tr;NSP|^@V?3gO5Dd z@RhDeOUu9SnUP#6IF1#1XEIO1a)yAJ;!qR7uEX~Lz_e2~GiX)}Wn5B=wa#-4@i|J< z-5-_OR+`6sH$Vy%D_Heuk3h*zP~M)TErvL$ks3BawX*vD4zLycF+Ar@95W)u)Hki= zrJ1*U{XKzTp>0kHmN5UhttTTkHntV=4A8|z_qR@EhLKj-zRe(EDer16g^Fu(N3*4+ zrp&Z)$d@m#J`LyD)X~FuS00{jxALS2bQyh!997#>a~escj;`Ne3FVVJq;OTpl@gHP z7|2~lB94)I2d&2b1xpc&QqAlmAZ#UWhG>|Kh95-p1wP@iL7IetjWC*5AMxF?YQPq|NmkprhYd(TQGHRf5zT? z`7ce5O+Gxw|4ofFf)Z=YZOP2FO~N>?P0@vdp!;f3{*$?D6pPyZc39XN)j>_WPu8J-V8# zLm&Hl73o?X#OHNol$Dj&RoA?zt{w0$_kTNfugxB8hC_Vf&g_UkMH}ib zVsB{P5xM zFd*Hlhfy+_^p%mXU%z&|irn`1amwhAP+x(2Nitr~?1kLC=0xC4ey0doc3S0Peo%=a zP>kVNiH;0BE+3(W#htVgp%QZCb&0Nm6}9)$$o|-NJFQfJaRPxwdVM-63C)V%t+(dT zp%!IJY4Wl}$S|lk$f?8XXffP5ZJ@lz386_X@d&Q$Rg`KBdLua5WcH$26d|B+VQX_>r3+T_q_?V0ho8B&ZHNN6++uovUC>!dZ_Hb+*_=1BW^cA?|^Io7RR$*TCul? z<|weJ8jwJoBz*}?+x&lm+KLc5)nX~eEOpS*J!8op#A8$bNS`JCB$-Op1^pMB>)x)= zblf1r^nMM0nm^~D=UQuMT2)a|QT?Jev)3>KC{|heR7tY*wYZWvD3VkfNtDGk$HJWGSmHP_4i1+tco9@Xxo-T!>3X)mBhMacG!VbfJ@M_z@FgU=xsrb zt%U{S%1fD_*=COewy49hxArBF^ut!C%Bvk(?#*t~VR6`^#H7vC$(G}0sJDKKQD)Pk6)VJcdE^vqtmw;6PF~U9N5jzi|!wK$ypYJ}>nj;la?k{>66J z+7SL26f9}6kVhbqybCaTh$cfx23tC<b)|F?kVq^{Q}WVAxV7%VcHw z>#I*IZL^p+%#rnyB@a_xyAXtnx5)3;ufY%$E0Gj5!{bLqQxEP#Fwf<<(mlzxPoT0- z+Keu2Aog6NlB8hpuln_Y&Sx3tdY{*xKJn{lpr_GGW1-OD`7d$P8uV9*Q5vUgyIca( zat#jn68;=V?j)a*6~QxjhO$o(DCR59U_9S6wpXrWazGNGZQr;74x3+S_W_bVaJstg z#oyk6Rl(jafAzUPjlg~Rd*5Sn)4sH`Y>SYoF6) z@v<_o9Mt`p2)l(ul7-r6@Q*8NFiIs5)IcW3Qc2~&654T|#C$7IkchEVdhS{jmgEUq z9E9_SWO2C{1$y!rYEIbkoK)1XaP?Vf|7E>1g^GV}i{>`C9??wJk`VlNkO$V&Ews>X z(dJUb{3KvicB#_qLlU!lT_6pkLRJbY?vCEx8x|h2+}5h(BNQwEQ|@p4R!}e%P@Zvd zpv}K+=NO*7?%GG?PGyfxoqv?vIWRHrRTDV=b_}4a+4T@TNLW8~!T48d&XN?-TQWyp zM`+)q3|Sp3*CLFuCX{2G*fg}afr{W%Ee(6E#+|8Z{Lg%Uwi)|QY( z)XAvhljW#r^$56|h#`8T{NR59jHT|t5OwJF@Yzg@iQmJF$bY|4PMye23)%pc03TwV zxA*2?Ru+AZH`<$Ao_98HX4fvLseZ5OKcugB;c0Ltk${Op*AjpJs7Lxuk97Nq`3q%^ zMbZ$ZP@aq|#YQbos~w$?#A}$ywYY8+)mGFoeJEcSO%{&@xol(xKLULj_J>4Hg+1IF zhZ;v?zXzQh-Fr3%gvg7v%f;i-6vFA@ED~-ud8yjnCXT|k0pR+6`5wbThK2XTo4@}G zu#1s&J8Q|EOJbs2O)(4V5#N`poDEJ|jU<9e^F@YW3nT0(L9i;AoJi9f$gDOtHmX4( z7QC=G*jvB-bZ=Xke7XFV&V8yL20L$hoGf}*&{gkM=2R>#qc5Y@7w#7(_%g5(+4cAP z*tohQu(zo;(UcurS8pi0ObLmuBN*zaMGjD8th)xf)z;c(>%F0USCTh5S|ALGPilE^ zQ4opcSKn!Ay z`J8v=I2s}6_cL0GIFuOIvHgk+!4Y>d-N1C^^Jr_ire@yRvom_%4HaCHF}`LfMQYIH zO%3G7=Qnw5ia-DS@4|N(Z6ReYYN}7sLh|d<(ooH}sLo9ou&5K(PdAa}ypcrXvyZs{ zG4y>S%zRfZK^wp^R!v5C75V( zZzJVvt65tFtZc0*0N+7D!wNU5PVM{kqHHMCLftwh?*{4--y z<;(M%4nA(YgL-2I#FoqD0~3*v^9@mpSzFais~109IVnPAG!AaZvKhRMsizd`SY&ZV zm<6FOmu7-&v3-Kzz-%n|2G*8l6pI4W@5wJyxpkGBA9t^eC1Xh*&M-kH5^ z{`<@CJTb$GMRh74Y}&Wo!6UBg!gHs~U{J4`zf}vjnpC6q?5O#cao+fZcXr*svj9Kg zN*r3wit!i`yd<*51Fk+}C03N3acIBW!i#C38QDaMD-6^@YZW?dII4=GU zy;F^&qcSLjylo$>>75GjpIhKv@HTnjKi;YdjsB3JpeqJgqN1;^$&o1F5ds}C5=aRB z{BAuLh*Cpi<4{drv(46l z(BGVjEG8A?)Bq>49`QDc6fI4~k(ctsN|mJ*OQCXng5Iwc2)p2%Png*k-8q&i=8V^w z#hIoFQp`BTu(uzwWI2OB<5U>HkXTdoZ{1rP``RG$C=8AH|Bu0ayFAaIUy+;2_= z1g@_3FXo*c%*w2-t9ajbN@cDZytDd6^_?aIx}b3;(*$@+eMrr(r(lr{V(y+^q&kb7 z4+hr*!JpH72G;yJT)4{ne4#4Cr>oC>`yQ&8>T!>cjz>K-5TKVBTQ(xBkU(rJ3>mxC4u;+?A@>#okukp@I7k2aUh+WL!%ET3g;KRl*XpX&EG%!vBd%U}qq{=ZDkl}y-LzPE2D|_}i7A+F zY>~K-4fc$(eq=jlXkwWd?ugV`Ka$Tvix934H<5 zi|n2%ydG~%pw`be@WF9i@4-jr{hCnWDwwqY@_1#iw|C~LH+B(hbfH&|b}Ao7gSW<| zNJd&3%`-!vJ7BF@s-^}n>b4$9+R( zI0fPY*#e(yXtG`WNTnjXI7>)&mks@|Ge)#1Bl~WVk4-ezkGa*{m6k?Tx1bQg$ZCJpMa}iHqAW^JY}QZw!2L}hvT`5r zo=~4~(0udsha8}3si>CS*vwoa*l7(4W^KqysqFn5XBS&C1zYwCv+=kQO=}{+QAG~a zprDh>YC#L)yR;HblO)n`4CiFZf@fxM__b>x0jpDigS9Kuy=ruW>xT~~W}xH3$urw{PP=yRl?5Zz{qWMb*3?Ltr8HP=XlQ6; zd_COP_j&(xz?^rjz5g7BZDw}W;MNcKn+Cv_PZl|3L_!t{xpw93%2G!|UURcd{Pa^* z#eA&C`+J+R@_5CX7S0<*QParE@_&L>6#6oeh0hmgApet8K(Z zliD{-aNEUoNHzvq5$@=_lc!1r!;`pQ>4jENjQM`Bg5j zcc$*!=TZG)n}K`x@B5TjDIUBEsk7_z75^(0hA@it^=oW1d@RmvIVSBXPNX-H2?*K? z?@5+ny4$3%q-%}-*@oaXlcU=(wZFY;Y^-uDSYoFgCF%y6{Er{6ujrf4M!J57CXsamcs2P(ofG;{Th=U|pc_)vzVV2!;m_ z-fQ|`nPV@dEsLn9V$=~^l`xXGBs*U1ZTsrC3?+2uGBZy*yEtDC>>jI0&rqV}MChYb z>!`;`sb=NQkeY!V?+O}>GMW?AuJgYx4GeW~exrE1+?~ z*-x|sjRkFBfy03M2EcBM6KR``P5JHbwzRzS z->O?x4^@ON_9PE$!i1A$)DgT{+(qlKY>s9O@kSpW)02F0ZQ0|EvE7bKbfXEfU7Pd))I1u_K;AKB>)O^k=fMYLGmiC#Lp!Gt_l zXGdh(_94I4tba7f^5UVs0zv$M!w5tpoYfKIu4B{Z)hwmVvnLvsQtpp7; zWWesAB7ct^0_TC4sXA0~#9#Zp>4CGWQ&~a*x^N$17;w}>@pFxs3yo7 z6s`u^WL-L-%W}l}9TUS#a58sjBpMSJFisk;2mW35)T>W2XQ6U|*{rI5?<&;b?fdoj zIyg>7$Csox?JH_h?sUA>`0S(&0EtV#;R}Dk($GvM=#yJFFewy%5<(!=?G#a4!jmwrO$%ZHN{Tf~pv$-@) zUXp?2vl8zn0jHE(a}Go~>7;hf)m!aJ305Nhl@|k7r&kK}ed_3ZE?|4>=;#?ng+8-u z;A+qn=%^11=b}sxfq&PeJId>nkEBQ7eA_q_&*GO9MY1PK-JpF~a}Z5t^kA6lxc0nv z>i6E@%)q(6L+wAo|9()l^Em{&eLVSCXUIvYFWW+(oA9Oap=F(sNp0v=4QjTi*KRqY z@|_kRbBYM7VjEgGP_x0CQ4?XH{$WvSiH&T>G4eOM!DTp`qIObckKSy6DWAH3g)M^Yi1)d4hgy~^n9 zoP#Otf+_bvkIeCYRr#GMtWXrnA*gSW4V(5I1Awb>QM+`&GEllMNboVxHOzPMAmR)=lc3oMUr-0HMn{*4)81PhvsML_61#DWhRb3@$snDeT{8sz+8rSDQ@H@W>>Z?D#MXUZ8JP0B< zNMsCjiloI7jcSamX}C*H+hM;W>~M!hs`d?v_WyuCh2pJX4LJ}bB$1s<%$EccOI(xk zUKf|s*T3}#3Rl{?i(TqWj6=)smmkDNHK1kiri}Tqw;lKMTvdfbJ!PSb@sAl&=iCU` zT?o6z3455pO{aWtoB1jbz>I{;{c8TuSHWi<@bg~pVh~WO`rpQ~v#oG>L++aZ#Z;3$ z3Kct_ITgzs$S|^uve|09%zKqw6&QbWUmD=H0&*cc<;dzN6X=#swmhUQ7;;du+U9qJ zZmR{HB$VY>0DT@gbrTX$FBG7Fh31-R>`pW%gg17#XLTQ?3N)i}H=qPdc?rBn3#iZk z79VHDjS4B_E1NR}Bjg{oKLr;RPW}y_+<6DBMIzWH#$D+g5R#xZg39l(8t@SSrjv z0ZoNjO$Zp_4HHFj!gMOe^ zE8x9)*eoG2!6?-X)zNwrfrWp~b0=B(_treRt2J3aFK^tT`}f}dfU5qTW0euvR$@)a zy~p6r`vIg6LOoLKFwwRAcBsDNe#5Y3Lgp5=!76_yqGH7Wk2@*(L=fuFXmJTkY$>9y z-*!a=mjhiMTPw&eAo+9H7uN#2G;vKAK(iEe{q;FPa9d;egM1ZnmhAd}l7LC)V8Tku>)B;QjDu;T z%HtvBf=_OczYj=c%y-;3g4hg$WnCSYj)j(6VuC{GE>OwATK3F}QJNyCx4^3jYClUd z9rKe;3bde3fKp0e42j0whHh&%*+{Yh-Ro4Fx!`Ox5DrMk5Q*F9+YwBqLQ4@kfT0Cq z$;&&@7fUrODCcZmdX`Up?f=x;ttnjT>h>9^t*L&Y>H!FJ^48EA`H?;#0rjGB=<;Oi zu-^360hRTNj8dd$BPO+m^bkk;$)>#>?8j})KwUL04Jdj*=J=F%dGEBMueu_}#UmZa z#DJqAP$@{RLBI7B+Cwn{MWeA1-EZ-24w7^iM!m6^QDvSwQ5K`V`dW&;!;^^p!&xvm zgh9rKh9oNtk!-G-LJ*p}UP@~b>s1I5kCZNuP>h4>_J;_3%HQR;CB{0t{)KUPT zB7Ij?uznB!v<4;$lW1q+fl#9}FOywwEXAcHXcQGTrgv)LTiKkCORZq080PB(2Z04c z(X$_Fo~8>v{=Yf(LjeiNwl*vz>};8ebkD}W3pC3(gm+LgtNZgBAo-E3gMPTTST;jY^hLQ9ImaanUp! zgF;pn>`yf(sNwd8LN_G|`LC)_DL-NdbBz1a|ku;|6NvWn^%9V zi%PZ5c{3-)dH$!h&FWHfObuRV^`RXB#Nb>-s+Wd4XosEHsIf;JlIMrw;$^?gVJ|;8A949mmXHx=n!D zT>N@ulcB43px#^8>kA)eUAG#qE{_78w&}bAITy5{u8^8^X=&-{{+gAJ1{qTW6T?G? zlnrpI1{*&n(w?@l-uOqdJaG41?}abG$YIaioP$zYZ!3tfX9lJh#<=OYAdelYfSSiH z{&KXSA+ph{=4Ahi>wuq;QJT$`5>cgWCJsAd;p|&4!`8yGWpO{OX+&23v$}8`ON;bW z${aRoZL7?@funWbYzq9#y6CT%MI1nPjy&LY%RmH$6A>(pE5u?20#jjKrOG+!CrgXA z3}r2}nzOC`(dzqFk5~}ilt9}9k~^DINvPd_MFub4rtP~^733DZ+T~@v1OO(%l7Q#*@mOKz%`BhdA;cteyW-|0;4gxuMN# zU1KUmvFSPXj~s!9*htLoBkS#+icv*oFCw%BrlPaTYbt*K9$Q^ReH%JxH5nc8dcPCS zcNt3-VLvK=OpI_ZrgGSCcIiP`70&;3HsN?y=IQFn_ig^8SN0dZ3kB;*>q~&} zwA$Sq9&|-G5OQ(@f=M+_z3A^iNg{KCWb+TJ>BYWSk8|dE31uw$w4hT7~GucA+3k_D^R{-F-Zq z>ADz=l2qkrbQo*x!tN<^ATS}(zb_y>l~R;u!B7yMz4dcTj7`j2cNKbJ(;XCC-iHS3kv+59< zETlz2M1rHh*S58KEb4`T2!PfpV{@j$vsZ70TPoxO34-Vc1Lpu!fUzjJcu%RiG-|Xi zV_&=A$RFK9|AlxCmdWy zPN9u79ojp`6*RhL!i>70hhNa$J2i^(nLudxRlRScec0-<`L%LGA>Oz~Ah_|LFF1v~t?`cx6E63zAdpNiR59|5zl z%?aI17%StZKLBbb%C$5jZ5v##MXMxo{x}=H>fhE__XH0WCd~hWrxVQJP41|_W8Fz+ z7U05KjnqQJRr1@bzO!Sqb7e3xS0`AW=pL-@@ebT|vjAA-yV!=dWeBbr)3BuvOP*yX zC0}2`9@L>6X5HBIWuD&(BX*|23xcl338*ipynU+rrxoW^74Qdd?or)!Gqm$r`eVtp zSe^LdR#p){^R_`kQ!{Y-se|qUMnr_TW#4c_gqh;dx)G&2{0-685r_f-$9@Is@V|2? zID{H)o@<}aLLB1my5XcPnTP|kLQyK+L-t7e5k)E+w`;eD_6@DRA}V-lDh20qXp zJQKPGHnt!4~|y&hhrF zL&o1ePZWb+2V@4KiicMX5Rs@T3Ck#nI-}q(9W(U6nYZ`5F344qZnRJ(#Y6SpwhP;b zE%VN{zjZx*sXQ$$tv7f8kh#BFNo4hTjsXlWkZ&G*+|@DEF*H=~Ra1*bsm$^CnSuVo zcd2-tQnqm=QDg9DzSxbW4Nsi42?J2)F#5ExBT-n$$V;-;&dC@|197*;?mblEjchZ? zZHaJ1GAOEj_3L=$UDs1$7%InzNg17ylSNMiLZp5VTzG*tkI?xqL<@Z(i<{m0i}zTa z)xh*vF-*399eI^}_AHbnVWIrruT-GpET6x8`QlbndF|>~aBE8$wSk)J5YJfK7U&`U zA`kS196O)ZL7lZSS6o zCgexMm$?7V0(fCDwqcfcVqF_DTFbM-`P|u_&~)50F?)oF`64Oni?ejwkKJ#=kB<^likZgzfr6W5PzvUsNi8 zREK6=gbo^mGWPv5<~f3ao*8sOz~PCCv+D9=Ui1xn(LB@&w5zqV6J2v{2nc+Ckso(m z_`9&XpljK!%miF=I7*GaR3#CWam%BHsz{|>_&U!o0Q|ZS4c(CWWxvqM{^_y)Y5-}N zUfYEsi?L&*$9V6NYJlyxG}_r(Wa--i95`6ke;83FYho~ zL>SIVEcW`^;U3NK_`9x!w%RFb|5Of1Lh1_wWv!^R5Ek>0O?Yp-!9qTlt1`BN^&%-HG|JFCPHNfptL_ zIXKAAFKlu8!buf;zC5BuC`EMb22NP*m4elqt!mpnl}AhukESCqaUsXv-`uV&3VqK? zdr4BPo1+^RgO?PMec+sQ@kR`9XS)gi=N()ntrTmsRcn#iPFNy1O)d=;7gNwu6$bwl zL1czB{cpa8)1e&57 z-5G5cP8M}XO|(URTU`Cp0S()kVh7#4AF?aAAO@Wd<9eph^IaD76vXPURUYNN=pdv1 z%mwcRay9v8L_8{^T}f|pynG-EykP?3>6&n$KT1=xIdHnrLD$2EV}<=SX9tF_Ek?p} z(BgndNTICqoqtmTH4VpRXFV!SyeQ^Z^efBHJlfohfU~0LK%>ONAQ1xpRThbUw*~DA zqO=P)BC&Ry3)EC18S{t+CzdfVWKK-v8lmGnQ~?h7I+=!Q#xPi(Q}Mu_ts)7vj;zXQ z?{TH^ad;ZVx5?RIy}ISkJimz%RBVF9!%goybnuV>>V+9K`qi0ffO(d5dZdbHRWJP- zeKifpw@El1=zw3<_vM%OwzbiAhjr6bkAMzfaEttIqOHCjQ2zRFPHKa7-vC5ci~>R8 zVQ(&_Q79a%k+WtX55lC`eq(I0 znlYW1*4YV8&Nx`q6!N_1(z^nfh=t`rEPF#^h=b-$2Tdzj8OS~PB7o9lm#OV3jzrEVm-)L zpc4X)tu=9X&HjDGKy`qUN50wV4t;BBbbPLDb|qBpo~qhlW%;ItKPjFzX4{^^XqZm6 zfVNiXiDMY>WD1A4UqpO&mT=Q(6d33fnXS>FNIBu|7$zfx&Sqvy0(sJSG;!NLaLzLR}rFw~|a%qWpazyLYLP6Sima!okn2n7YD9dyQ1r2~2 zq*VOP*b-XQ!J8kh7&P6nhlwh%90)7^K^3*(GXqcPFzUAV02d{!cC!%cK*2f*%x4_ualpI!CybN7T3L~?gMr2sK@>yv|W7i*o*YA zD`7lgnBW_@;`>VnL~l$5eCy)#&*q^OZdK5HNL5XBA55wC2Mp}gd79sl6HiVYqFr^) zXnN-2qZpMj4<*m+Y(E@jo9=&6mcCu|tV#=2yE(DtDD*`~im`s%c4M}|u|VKGFhe8Q zsMXEMUovMZH-Iml{Ib^z^(aMAaVTDQ*j?2^@*`|a`OJ$YnuFQ|P{G)`?#>tMg&)+E z90`6N;$Byq=3-+lQ9e)SjmD-8RqQcrze?OJq9BYErsE$uNjNZL+}6EqgULeE5WnS*Uq8WF3%Wl)z&jIqTLtqV zJw2Yk@Z$Q!M;RGY!BRuQOn;5>@ z_J5e#$34DASroXL>-G6AIGuL9J@fiJcQnJN z_UTwJc*}m}dl&0SH!S6sJzY zG3jdt&QgtJ)%DH$z~KR%D_8{TSq94C(?U8FwZ6;TG@ z5Iap08VL$5uInG!t2-7uEsEZ z780<)9R&zD;D`oU2=Mvys)gN9H>0wO@IyVoHk?qLsDs|2i7~1$j0TisEC$RSDpLYI zf$g#B0UBjj4WKjRQml_=vgLO}MZ?LVYJH~A8M^p<(;Lo?gPu@0`if``;^7QbjCrF> z#Y-4thnx&^2q?4R!vRAUr6n$Z$w=&v4|!}h8v;us5&xRZw!+EFMRS6sxvoe3FAxZJ zxLFe-JribM-eGdy>niR-a)L7%ZtG(H2%Bx|ZLbsXIyyDawrAD_1()<+hPfgbtwFI6 z?ZTP#QlU5p@M^*dt?jRmKMui2aXSg z;)|G+IC%^@?fcRSN8?|HNfBjEDmuhLU@niCu8QgmJW6EaeIMm7G4R@oibs+*GZY4G z`Fi;PG)NN6IolIBp{VSVh?r7Q;fO==HlqseQJ=QBO#VP5U`e5eJ-6GIoU@Q1h3;ve zYwXoO+g=y+b?Ix*IFR<0kjLc5Xsm*1bGLdQfB`k(&~QS6_ZIqUV;VeNOE8+|3XKUA zW(V4)Ix@R&HspXvbVOxsqW1Yfp$tk+#@-EeyqaAZ><5cZoZBS&AOgn01;?b#;eji6 z*%z&6AKHP41)zx@fWhUUD`e8F;>_4_K?&%0p@fv>-;#s^EBu(QvdKz<@D3wl38oxk z)A>1U+=vm2#7T{y4Jfc9Ff4ea#~!Md1I&3nON8SXFpxc~_tde4@>H_){&<>_Po{C` z!qTf`)YROJ*2>`h$I$To0|UqKr6CynRm*a9K1An@%<0tyeOvH@5sY^KZh~7J z>Zow4Hb*+j1IE~30AtIg1Ca8M$0w*k-ZngV@>uDeBhFube8M3ma+UlTN$|ub=Z&^7 zM~zR#z~iJSicn%R*(`ixIp)z{_!NrNh`v5LdKm1mBQ6XcF(xZ(+DN3yn#x0$5MkqF zC1(RsLy8hN!jnvZX=|a>8j)luw>-$^x~9=S9DQ)ZS=N24v&*zg?LpJfg~^WOv#wz) zUHWNhuYp?YPYkWIPX;-*Dtvrk)cYI%|55ez@l5aU|KHE&bWT(!_Aa73%$DV*qTEC! zYm=?8$%d(%5R#kY2;JdxYO)#0jLJ>gqGq!?n!8B8=NNKCsZAk#n{G}M&MG%y{H{6Q z?;pR%@mM{kv%Oz;*Y&)f*YmnwYZi+)N8i1RxpC|LgWbH~6>CCPYzV1?P1{^~tyL?j z>KOs=(9y`o$wAbcs#^M71dKl_^ z%BO2VvoiTME^->8hV`v83}g}nJ$OS<}d zinYn9qb4yuu-!Hmt#_?UT9|y(9r-wKubJ^R9RnTp?zFwSE8gy>?cJsyu;f1FTfY7f zIAlZoVU*J4{u*3M60}p$Vg3i|A+7G~>+qVeu?}h38gMR0IB|31O6;kE2=2bC%$E? z>y)2A08)Uxla~66b;3)S#+N?7v>Ze&NA3GY{r>*nU$avo%J(%TX!YB3-+|^80=T`b zy{ze@p&(90%YH=t`@0iA{2TgmnBT->y1+`}e}cMN!fWkfLuVKrSlJ~|PUrIPf74-I zAcOD)G6;JB9T&z9W}fJ)a17C5|E{GaI6%s{)ZY1b8r}&~vmq4j%T z4evQGV>gx*Lz=LAO;Ad8VG@A<%o8jAds292aC8t>uYdHgB}M0VNOf&WZ??U3`AwcJ zFzvuzLCqFj3=QvgJM9F~;lkwmpY6BlU;<>Zc*5SjW;gdPngaZ<)=X{tM-Z`tDreR= z<-kf%tFe~-Ggg8QCBrJR1==U6!P%&-5}{c}=Ac=~Xm;qj?-!RX&l z^a5-qh&DN<|HZAN9ltReZ|BVy{YOb0jUm1|xtY&I$}b_C-JXym3S~A}Uo^|X?}fU- z&vmZj<?e!L)P}+9B## ztUL(=L!=Q^Ldupn>G{)-)STSf{bRUu6jTy$O?R!8B{8HrlE znIA?hg;$w~a6m{nfP-;}5wehmL>quhYJ9fV@Jb+_{YVE{^H-CbXX3A2ggPLFU{xiw4T z>;zr-PF1OlbYVJxRK=!ie&Krsch9{@T^OfFclV_Cxib-t04vWzq;*s*KM&|xphtULmxlq4pi+Nys7MjVODVelxVHS0&74Nti~ z{077N+UBbHm~L{EhJ75hVpP8I?%geVelK=Z3#yIAu_2kEnK_SNMNKtmQkwds9cKsL zcZZi~R2_FppgBz4x!*i}>~~(Db0G^vmBMv#`!$)6d6IhMn#r>>bh32tc@b1JEWnE$ z3JQjmT8+4FTKIctzA(G$u7ad1f#+r-mfAUoxfd9S2Cew??q$jv4-$^*!|~69&*~?O z;MX`l!K!<4byO8ONHK$hAy-Y@Ga&23dl^+B)r=}@kYKkx+hBF`TlM0u2tCspAGaw5`ZcIISGe~(tMuI0g02j@g! z$r0=tJ|Tij$90nh(WTMca!;N||FxEFV42_QEle~mBj&m7V@sV5u<3Xb4#o=G%sjdn zYeeN8V z{M>}&(-LiSnV)yj*@SaNx0}`5imMXr1`o6bxN|Fe$C9L=V>$EN(2-qd=S*&-oKZtr z{-{H)+bygiRVvD5)p+3#yw*v$B}0?g6y-g$P=YQEuZx!;Q=|ylxk@3oCX63OB9w}l zw~LDiOuCRkl%dgg^R5gn3=Dlc&aJ}1l)#X5vA)Gj=jGKp6RH0}YO4_mMOtqTs~brd zQu|I5%&OY|)kb|Rz;*JknkIr|z1i7JFIZ;@fK1>ToZYLJp9zIpXkH(l6R(jaA zhG9Z7F(J}{@vhcp&Ad`Q9JE`bXbCSFYfE%0wDwV%SBgqdCL$LEmTByeK&Z!LU>QHHXs8PS;SwAuMOamVFDsi$Y(_pjO7bY}8XOVUtd3e30DK}l}; zi&MKcx+Fx}b2B?PnTO(TV*k1sghuKn4g6jk5bA$MJ@ z7tFAIFp3;ex|Uz6N`Vm==2&iy%wNLCFyfGKEDgbS;dQHZ+>{jw`4KCoU^H+NHeE8w zZCdDz)e(82jnzllv`$eDx5^-+h8r>BE+9PN3Ij+&S_K)$fIqAwQRJ#BYv*SaL&Yg4G zc)Kg&PI=qMS0_Z=IBw{6cWxXs)(vauhrEg=X7=YM&787bt7)D8D%HFy{ZS%gMkvrc z3M{^ul9Wc~Q7o`wW`mpn*zT|unmkDM_?B}ymi(9lRfwc6-)VWm{wF_32y7y|PR!s? z*!h%jUE&k80weK%o7D6zYl{3NI7dbrDeb~5vLg#k$5xRJFQ+9j3-V;5CHz14I(sG@ zU^hLYOl?+gznxK|Q;k#euUt2E3p;cYW&8d0zS{9ELGyoWq-!9RG&iOY`;Bkes@Jx! zznt@t;nixrXb3VhTeRxSnpw@{o0O@yN~K@WuEy!N($TQ(3}yil)?b7o&)wq?=X@p( zDfg#Qq|bD!Go-#K@{jjV2X>*Zq)flKQRXMO(?9zSx?5He_pOGb+fNvxqqa zDmrF$HCW#hx(WIIax_CX!8`&bWv%1Ota%jW#%h+-qe>={SsPY+o} zjQ=(%b+(qp+OcRS^W2K7%c^w}e55rfDa8lfgy za1b^ycZhAN4WCahC+T9*(gNB(4G-xf8K6|5K2yf*)@>`K`l9CqA$xd7QC28T$8;~qvC7@ z&|Ni@sc&MbUWK>8YrYSr22)LzC7AE=47pl( zDc{*Du46CW-UqMS;u~&zAUMaLR?KAhn$x7J3#)UGK19`cji_&IF00lJak_rn>-Nj5 zSDzkzbb20o%z%)w@!Ztp=K+P{jnjrZxnLhRJ$~$wnfavk6spt6P;rekB644#KEcsAI0yWy9B zgIED1(rZ?Uj|D%514lRfn@Nyn4T1Tmz8jk!QG8IgfhH*cL_ycO2_NL8(t3VbFqPvk z4^V)md4e{qw5%+6oqvpujyOpVrYYS$i^1?4B*_IUq()}@m8PI3^=AR90~kaoF_X1=0++DXsIyJqP_}yD>zmj z)~cGfwF}>@q&kwEH+Rq8W9B~-3?8T&qbc|C4dV`2lV2l_s{K5>P=VQ#BIUa-OBz4`f`WW!!#U?4H$;fLhgSGUzZV zKi~!5M(-_1iOfGKmjXm63x$AJhXUWSxX^I=P^yi5Cb?=6Tm0{fCF&w0ja$H2Jf3*A zo=g<8xzdtPKZb_w0}dxL|GE^BZ$L5n{@iwH>XM_MAFzGXlPwUo1~r|V9sB%qg40I5 zwibE$A&w!8&e&XP3;3!~Yi_hBZ&uR+C5x^L%I;?G#1I!e*2frKX+ z_$XyaTock!@Y?SGv;bfb3@evm2DMc&;j0nEdr&>rfw_je3 ztJJ}URkH$K-&#T{;9eP;pRuYXUZ}&ajvnyAv%%esX%W44KXZ_&)rQ8z0@nKE?L>!}Ut(%Po*PV$^nE2Gf0h%uv+LbrdUcH(w zZdRXm+6d?$#@y|&UDw#ynAG|h%+2DUcfX1?8B1D|z4gYPZ*$W0l25Is-5#u5R=|ZN z03q`zeY$)MYaKhAIpe5Lez+#*@!JZf-8ElSoZo9h)m<*d!c27+sl_m!XAt*k7ml54 zist?^urRFwr6u?%8wn$8bn_yx3_fNI=Y~VW-u5C#BttY|4(CnHcizmW!y=BM;ACje zIt=R?_HG)Z#YTOU@`rqLxBCU#OV>Qx`Ar+DO1!$arwm(z#aGz7;3SM_9PBm1lQQC$=$yuXxEv~&>>>fh(*|T9=8t$RTolF5W(iTVHvuYXlx?5 zKOs?gP8Q8)+Tr*lh8E9Ks*nV~POh)H&csn1nXrSfFU?)F#H}K-hao%Xri5;j$A(>}Dy?zd<(|)btmQ#l!0KLSigGr2 z;C~nKen1q2Zc$&_o0kGvdp=J6?ar(wam5NL!H!$Qhwv-UiEZ*i5Jr0Dpc}V;0HxZW z#z;fROn=`C{><`C?2{M&IeI7yU37{U zjeU~c6Hvq~pdFBrltYJTojx=ZQk7v2I6|&|oX>?{P50>YB48Z>6B*QE!rX*ia!WZH zk)hSOc=w7wSKs@`ihk;qLaQ}s$6V*U@$Sm>VfD{W8}H;IoF}1L|A#g%gp!%wQE-5h zx#_b{OU}=?wB&jx>ecLHT_M;BFudu#R0tc{GVda*}0`lxo97>gE zmCPE9gl{q7HC1)lc7>~iPEEoXb|X?&fWRShH3iMP z){?~vI4mw;t&1~6(=1S8M%D_H(z@L1;e&^B??LcTlKe#@ewnTf$A8g|_(YEn0KS}- z$42H5@(n~e77he|X}5qjV`o0Mw0zJ@nvG`q%UI_yjBqzxj{lA@K9&}TFfF??h`M-^ zZZt4Ek*MqJ5{RAI0ah#`?-|e=_%w7_91tXB@@g|!LLP-vN6+$aLErrzL>{)w3t+54 zo3Jdp*IXuILV7|LMO2r8@#j6mkTFJPV;MB|x`e=T0pp+zEvW_FbwR(~M=&!Dz|ZlcGHaA5L0;?UH}vrI{Dd@C zdsoSCPDeMuG1U(Fh1?5!!2`V=fX=mO2Y}(yz$n+*w{KopXw9nfO-xfx#1jM=uSSUQ zeRSnzXECvu$)@lSaBPq`plIeixtu)JyKQG;rCzB(0~`VisT0 z6VTVbMmX2XP~8qsQx?u0WLm~WuaKduM7|139|7Q1(b{tNKC<@z;LuqfufaD7=(biF zJ~XM|P_aB6Ej;B@OLI{KLrf_$B<4}Z^Xm3Xl$U!1dC!1kgo!_KghOj(Nh!vCe^$PL zWxm&xJbC9{)BOCbUwW_lbb|l-JHv4WeGnMAg@uA`dKDF}#RC1~H6_V2?N@`KBeK?) z=(TRM=3X%+Yghm|$*)+F#EjoTQH{1S`VbN^g58NGr6FFT^Df zboo+)pXp@DWzUvLD9E2TOE=*|D8Yu#C@G;uMCZ!{VOe!$0Db>7VsJJ!`%?J~u32)@ zacV{|+V_VnQ(uy`D95tgU6di9G2KP*nP~+rZK_g3YA_A!#pPNad+w&|DS+rMnrbf& zkf6hWbO^RbITP0IJ0Qv_(8;2DKKFt>SKb`y&wc%?JItl^zkfC)3j=^UvnogOaFvl3m&gZ;xm9@)A-?9%J?z3BRDyF#vA%+dql*2(^0ydLAcrY(Vh2F*fFf4_fq-G~D{-6f zS}uPvr~z7^jL-XNTi&x}ST~{Krs45zs6T%~^L{7b_K8YYT<0-yc8v|os+W~X?2shb zYIXZL1!WoS)(Q%C2hYRa-M;G&!t1T_Xg4ap6T$uy6Tx@sbLwrL|GEqiV1%xLtm!Db zWDU7!Nls+x+BhHV?GtC!14*DPYFbE7oo*XH`bdj0cdge;Z;mTi**3gm!?|EX3lhe> z{Z(d8WpQ-P?7-OXF-dW1*WQaS?H`D~J5UvJcLR!Br*p+N(d3ZP)T}FaIi`Lg(YVa# z>XXdI_q{pBc#j`eDAM}p{|q@l9b1w*|7{K2+;A(sL8HVtOt@1EeU~N}mS5t+k^mNQ z*5!AhtB~V>R$s}0Wo9CUzB$iyjuS|Egznxx-f5uA__84$fem|uE?{bTTdEJ1Co2Zl z(e7T1zN!LO$qu^9NTn>O0chZk^vjo>)K)$T&+v~CW$RCy1g$T=ahP^+w1 z!gh(X>e|%ZH3OTx5ULEn1r6=eO!kkAj1&&X?Y=v5*fGAVMq1{9M|K*A7=qX1DMl;b!pqWXYQ{z_pRM|M(EW< zDqv)-lOk%YE7_hAxEy4h3;xF)h;IaVHsDS9Ko3(;@LVXj53`!eGVd976G_yoXR>2 z>LWm8^V0jquKAAIp`(pw6CUo|_~Goyt}f5K8tzYkD~*%#2f}(cOwOKzM+YHh^9OkJ zmNh23&e!#Cwfs&ODHK$n7y;TbKvDLEFGm5kBv!lW&^{(q1NqGQLwH?@erCs-tDtS0 zfc`Qi(6UYJ=hrqIcO*N$Yq-9vt9LsYV{RPY5mx(YvO~Lt(g)f=>^t`*b#|@;uB>>I z-oZtjqwR|FUY#P#fjN&+1j6aF=qAfg@@2p)7N3W`$I#OO#(tLQUuKX8=I1kEVS4OnDsO^vZ);Hi~DZ)bIUuw)Oi)$i3GHL@`OsE zK7H8xz4OanXuh7rW#z=xz_oId9Wk|^RyGVAYg(8d&YmCTxX$-qO+CMGajK0K7;SIN zu)r$m_B;wYhGx#=LHJL};zdN`mts?XUG!B*YU=+!T#O>1Na31+j4zw$01|l(PQ9Er zy^cKhBNk&OC*k58XvMOV?40nYTNX6p)YQ3Yb<2x8ev1bCJ)* z>mUi?CY)*Vu}}wXm8Y(O!>^lQ7L2u)($3_fxK4xt>1hbKc zT+!@BSPBM=$H&f-uw4D7g0HIEOlOXi^sr2}j^BlEq(0efk#Y%14TN1Z^K@}O9{jUU zfsPFjmT+Zy%La6?M-O1uTwMdLftp;;%)JHiuwp{d+1Xitz1I1IbGWEDdPzv;yB?>2 zm5`ea0KK%P`AzQ7u9?pru9|s0ZF%avQ5@JzcF!htJcb-G07*I+HhH*&>ZRXwfAOJ~ z;}s3$xoCcsY}0&)Y~4gQ&S;<2;^8@nCk49NxIuCX>u{`wiCAF28~!Neq%M3ZDcSW+ zdyfU2)YA_Gw07}Rk@V!tN0%U3_pV3Vv0rn4Xluu@Cf84T*?W?|zPp+_Gq5n_IyThQ zlnGo9mr~V*N7)Lwuut|a9wc}nl&2y&e;UnSel9MA_fdrrBhlDFoe8nd#CKq7NI12W zH=Xc(NJPszIM0GSg7;i0W}cxRpMUxm=ETfb!ww3Yvt#+?JX@*@d#LivkxC!FH9_!7LOD) z<`M1(YOk!c9uD34X$nBXZ~X(V&{zCpEF=#*Ah2neof_X7BHP6C@cjXNgt6~193?$m z_u7!@bl%{4#ccvx7Uoqbn=-IbGw@M;A`?Gl76tssX+L&SUZy_z^+z_mm7aFik8MIz zVpf|&Hb9=O6Q(J}cmej;P2fft5J^w!>P}B8{DO9$nVVKBdtuMT#m7tUO6$9NyAry3 z%efwj+BW|ASh$@Gp0oMB%csey@qyW|(xDV}i;3&3W_;z;_-Lysrke~|>_R$DS+XDD zPJhTf6HgFsBQYb>-0{(5VJ;oV_WY~W8{7&2xD+6c?3*@Rw9xUxl@Oa5_<~XiA-U+payUFJJmOwiOO{#vOTS_rbYe`)qLYg>h~EacD*l zR$}Gsz_HpUkkQ|PAgs+bB=J*Qpu+kGNitTXS0A8dz#?7vh3dfT=KT0rF zk(XPGDY5c^hlcg)ePm%EG5P{XDJXW_NFjy+V5w}PCi6`ea>^_90J{WSN{QG=eIOH^ z5#VadZZcz7NOA!?KzKN#YIpSBa$}DcJ8aB;>8OQ1@h=|06w)2cp?&;g|HKfmWdrvc zYKO+J4nRFm%4~0J)3kOs4#bWRrN%}fRTIS&i|%E_Iq73}8Dwq9Rh~Xd6Q79s+CE>MJ7+l;=8~5Hjmlsay^zQqxS$80CHPY)kjKsBTEl zl}TzbfLeJ#4Z!#eayvd;4N9E=g=ZVIBKxYEfiwN!!5*-HiHR{iJNpCB!!8|0aEIfk zt1z4WcTq*K5ISuxBiCe?%m*Vmc6tAlRsaqpW>R zT@#QE9Fl`>w@>#?z|G7VuaH%-UuFH|hgQAI*r;r<=J8>ByC}$yT-@00L1ZDo{ zUY`#*j0z*QlzB$AY+V1Jp+y{ty-Z z;qZsPoh~Q{FhY$xUTN=~STx<5m?Wq5P*Rixru$v^!tfLnfCc5F3VmIo5oK22j`@Ev z>kUvthXjD7F!Q>{8v|AqivIMtc1;i`?)sx*d>teJ6+KV&W0zwKs0mhGs##F)usi`? z5U9ihLK8#Rn@e;dubm_E3`{M9XHJ7>FXcG_va{Nu#K;@^+=oAZ{^mFY7vhMulYMm& z6m)i~oxFQ?y12aT?Je&N%RF8bsq6!r_r!Q>SK!cAx(HCu?)Bu(Gv~ zHep824TNH_-40LIvOVVx0yW+U@vrt>}}o8yCGo?RDb77*dCBC?wEk< zpWtCn7bSuBxdTukN6i2;pgn5+qe%@K?X#97PxhKL&6PBP5!krv^l#R;km5u7;d~}c z@5)~;OvhS(2}=+c5QKsRtJ1DNqKah_*{0P*F^EQhnxwkzD0jii*YO{PvtZ-!b1d0f zkp}#3hYv)byl2kK4c3BDwurB%5mt?ha;L4j!A*hFAB0@952z%t4uJlg x!w`VX zt45m6Yp3~MOiaw>5mJBegMbf3#VkySHP{L5T{&7jsh#PLH$P8*(%P?gSDO|F#>bnc z24fs|&3b+I(Q>+Y8&5A5M?rPi=s0K%I4pAx|xjA$uiT2<-VBxguu=vu24|&0Uer5Xmg3Vs! zZMfi7Dr=r$f??q+OxNwut~1}1-vCZNN|~G9x}d(gc>mqV&JXo>zj(R0gj=Aw$1_7= zzB|ChX=0iJ6Vo+g^Nq76sh`#1O`1>LxAu>xPXB&!sou(|$&OdLWrJ|8vVv&t==rn? z$Nfp%XTcy*ioVb%Tc>6`chCW#UggCI&&x$D_=7&KxJy?6q1%pB%wN(`dWp0fX3&tK zOqVBR2%W?ww>(>1g16)NI@OyQN|#FGZt{D{u=;ddoVs&ierRE&F!sjLU5$?(J$mpU zAiFK$Q4g5uo=v*LJ-^uHAFmBcENq?!rETNV@!fUW0*$G`a|@a&sKkX()g<}v-)GaC zk6GW!eN`GJTbC8&ZHFvefM+IS$_s3cA>jrVadc%xOsDgSk39aWRf;Un<1Dx0(zWW# zwii5fFTKL};G-?&-pla1p#IQ8iD|E5o>1y!6=%qB<$K}T<5jm(_Uadcf@dFDrW0Rm z0lMh3-+&vJhF^VJvhdFJk;ZjFow{Xf$&sL;_1TWvFazd&^l*51cwk{sW=;-V5^WSx z8vyLn{NS6P$A7kSfY`eMYWq{ab~g|0I{WqSTiJuD3mQ$zRa-`RE=9l!dTL~*oq4M) z9>Fp45E$@s&mtbBQ030)>WQ*{5^U)lNM>Uowt^hwpZe{rz>67CnQrORUYB%^Z)4|D zf>&Aak;Q!jP!7vrdujg;P9H*&BE?hNbucNxtCH{>*j>8S*GCV(1h@~Phpjh`>}mk? zG57gh$5qLLyH~eDVToF)jR&&}6N*RbYmR(yJ{P=-g6BRngVgL>4Sxb36`Z7Sj@->#oFbZPAx>(A>wD+}QeVSj!mV`EV`3 zKX@sh;)7*ea4HYXsXqk4A7U|4ODh{=B15_~Wfn6(@ow9hO18p%<8qwV+C+P!a;3Z@Vs!8_9icnUL8P+I7C+Am`jD&DsVfI;JLbYV0 z(37N%I|qfHouNHiS;uFU zVj^yLaN?!R%D_qX)q4(GvqyGqjO_^o@N-vsR)B%)uC%7NHnAh7sei!bb>Da!RMmsZ z?-wu)Q?LzR49&fNvG?S*(*M%}c&C2(eW~8~kZb0OZe(IPhFZm?@K`Uoyy-Qup^X8X zb1eJFRfN};2^PGr-ZMV&NA4sRgxxVNU%9Ue&>F-)!MTx4!!lsPP=$mQ&7`bXE0PfV zd@Rb$7`-HffOc>b*nkfSW&Kps`>G0?L63%z`!zdPre~$Bfcp%>ZNsajqfycBk00x7 z4!g547RCvU*FP&iw$!c}+S7DCC0V-DC$|cT&doVqyZqkWqtTb!Ch9i6)Q-8UH&FT` zsZ|>kb=tw01M-_wQeT;@oNa#dTXyJ@96R^So6?ZbT9;ktr+>dwJ2?A47tf_P8-14` zmJRK2*TK%WJk6U3BLW|DClF)<^p=3@g%eKw5AKeB+TnQ{k&z%wkp~D8EV{^o>dhCO zmnRq|1#bt_{Ma3TDL0d6hPQ(o-3d_P#geX&kp#H_vhpbwsI|37Vb93~A5x%i=ftbP z_^yKZ{7oSttK#7-!eu~y_wL?sO@&sY-@blq`Lae!Yu_8mau<=JGU&KUYnNXVfHhDk z;-uhK>$?I+4u{90K-m4ir&8;jG$~21Ho!2Cfk*PVV5R9q?L6~! z@Z9*QUV6?B^_NWBIZlxENGF;7EWjDdGmG&o{Po%)7nculcaH!l+G)WkCq;2jOGIAy zM(r|!Es5m`n0-ss#g^S9tB_*R$jbHU>4Go|tRlbU>R;06Ke8H=O4tDRmZ6kCiGDOMpl+8C`JfR8mS=j5#BSgI4yn3`jmIXG0DPnPFjn{+#3 zrv7KXG23BMd= zwu%tTj5u|VpbfQ35x4#2a9nZx?&~8((sg~Cax&riC%DMbw$&~%q<4HOec;Wa_b>{X zeq_438f1E(CTEw9H%_%%z*QMvp%V1`*Z=t)7{x6l4fME5b+ZD;XK}o7rw`VdZOW@N zSi8f`#U;CK7{+_D8aD1=!33i`QYZyRj&e^yk_9B+0X*8e?ICsa6k=h6B8y^(MfP^~ zW2K1F#f%+lnsdeWLMF=C9D~YnI#48&Q1IR5Vd3HZJ0bK-*bSLrZs zfs?B}UuQJ+Ko)`wWG#3_GKh2V;Q8oqHoM7lc{R_yl%)>MB!~u>^!tPBi>sVxO1R^t!pLu9 zF`_g6uQ7bB-(Hutg}Kq z0$Vw}`tc9l+}q{ur4kpc7vt2$;{lkq!a}&x6JCewHv$OS{<8PX^Ho~-aNOG5xZ!x{ zKYsSp6!=-abj@S^J_MA@-=8Qwu@;q+>6w_`ymY(?ZhNx;mph|w(;FXXaLq^EPRb(S z*|h*{K>Y|mAQ{0Y+=}y&>wxr-IC76y_W~i8%8CU9VG}$!^1D%&b9mz|s0e@(4HuFK z3Y{MIz~q@OkrdPdEPGJX#|Cemz&%lah#b{7V^v8iFDJ+v5Lb*A^vN5@!m|h1x%IiP zzyuW1t9_ElE0WObP@xz*cP?xpjCoAw&e+RsSGOKL3{AozEBm##QglilEfO{qJxD#hobwOb%{d9z6>LX z!NB>Gfl}tZlEFT5z|jESL;CO0stYK5bYDP{h)KV;LN$xD8@2npbMq(PpMH2YAsK*u zORaYk+(>mEbnhMF2apP}vj-eaQb;JP6&^H$Xhl?XUnKPPc6GlzayB8M@opPzBF7Qn zWqKccY$YM>9<3)k?tjW(gK0%ZgZy8d=7*)uZ@ zx6GNO{N}p8wRKbV2|$zKar9XE(?b>sVx5FLIVyd$oKz%FWfqK~SRJG!LA^O1dDoiR~gs;UQ~f z$S8ydDaz}U-;N9W-=;%>EsQ^-fwXTyfdDDg`+|cOY)}(V!rcm%#WBhwZ*w*acjHN4~nx z&b>9Jcl7GJ--d=#SVPFLx?H)yf@pEHcW?|1JwmSt7${4EZmA9 zC5RUYMYXaku-*>pyAYqmvZY)~tgN`EDyGw`*h>UCC{AL{U$P>pP-K`OE{!N$Q7D3c zy_TPVSZL!h9N!^A7?`9r7ry`n=5IAIS|a{xGM z`JyfFI1&^jY1`>Cyl`jbV8N$@tG?d6`(eOkJFVO|UsQknV(f2s+MG;(Haf32_iywM zU+&rd>(`5io*w+KHAZmp=>{H|bmrA;^7Nx(mogq%5x)K_lr(@~4h{niA1J>?5AOcH zuh;VI&0hExN!){>NKF2%QIaGnZYvVPlS=i znvBrX@-)%@yr0J2_ZQas4Fv&}c=pTtna=*C+FHkkS&$CjBcg8zvt?9j70@C!jIY05 z++()0H^#fBf1n5^V`}jqX`8|hq_|EM@BYoBYNt^)Xvz|!975s~e>n^72<{JfkSl&+ zK67Bxloyfc4OJUMiQ8M@R@QIeMZurumdw5F*xCddpYzh)#w23}%foZAGB05@a2w@F zhL;{9RSIDnD&2OmoOBXat8j9v@D>HX8%4T}kk4E4ko{OWb$uqXwwxQ9nmTUYj1JB_ z{fr@}ij2B_BnOFX+NCN(C>HVRHn8an;IGWEbcR0_be9q8fo<2LZA7p@v|XS`RvTki zD$S$q4Vy1U90Eki=})Ok*X&$oB-hY zi4a&-B~@gMPQHB`+EDUkax$^@)p%0FtmgIj>(3f5x;M2@{jVmKQJGHp87Dtdr1e zBI!aOL}C$kr$9=#cQGh4tbP$d3|w9IVrgZ@#mEgU=#{~=EDCxL?ShV|SaR0)V^Q+# zcR8W0G7ye5xQK0lvE;*|&ObJ?{xu1SckTIY-#^ zJ>S`p*sy!&&f;Q6$67c$5WxVkIXSy?YIbLL58CIK9d`GPL6EuKq5m!1R?Lxx_vhw5 z>v3zwaj`eHyo&1b@-6|`oqO-me7kqkqw{g!TGj+bwQXY4EAYFQVN*`g#C}E_0_n(`GWat*Qi7-$j4lG;Rc&!90szg5^J+pT9iwld16yfJ1D(wPG*M zY0D}f-5)J=?5%iCtZDy!yG6t`h6xns>rcD{#P08;@m^2pfqjuwF~VbAM92r({sa}< z=inhWO>6@oWnyAt>-^Yw>bFkETE|*g`>qEihfZMW<#0-b^yqyORk<^c?BYVB*@(!2 z4j{+8(KR+6Gd`9$)vy%iW9&mw{cjnj#>P913M-8ZZCf`DfBO2hS342WrIC5I7eX;y zeT>|4k8nzP(K#Y5G9y%2IZ#x510+%NZ|?*xd>tuyRCiwe>1$NfezJguT?rGwbJAwd zp$ZP*?8dYiQy=V7aN^-I>V`BV8>tl1HhiB4IzDQ-NhC9pumlbe)x@~{Z($xqAald% z5^X?ty=@m*+hfidJr`w`KozTQ8rtv^t6M52LDRyNr zy(-dQMm-*Jz{W1mS2y=m*Vt6U$VgJt+-&M}Iw(FG8?SRTpNe4NyBwx%^nqzw(v`yO`LMlmcLO!?juIYcV zD*N;A9!qpdps%y(W|RVS7A{q+rYW2r8}&xjJbelc|IUA#cFFSlN zZ9iXjk);RBG3YZhW}s(;Rh%!r@v46nqNml|xaxBRYwh%Zlu)vKu?Ceo5`xv`3;tkY z_e=VEGrP%`Zt0J=ph46})r&M(YAzSaL<|Uu1(y-5uvH+T32W%l<+KV>b;N;g1Lp%D zimt^Aq%>*O>b#68hPXzCuFSY4u*_N}A?6hn4~$PbCMEd|L9S(X9P+C}Lqkt@2F8zl zuyzitR{cup>t&qcb>VZ(}Cg_cPL%D z^%l-hZ?81>g^jtHILE^sy&T1n!Jge^WPlbDH&hvjx(%`2eVGbqZS)QLa5SC&1{q#} zUf`#PVroKGWP_+p3>x+Qke(b4o;~ouu-(mOf~#FCs^KTylbz+q8mHHT;NhG0cJ{W@_yHSm#9HxK;-{Gc)=A zc6?3o=qF!J4LiTb?c>D&YH@V;_xuPdExOy#rrt+dMoefb1h`V&Gz8kf5^4S3G;Zel z$FDLw7k}ck&N;DW;Ktj%7-QqltR?Cke;K;;zHn)lui_Tt6oY0C5r}O~|7h<==rsQ2 z>eZ{-MtLnQ9FWA+^u~nPwqCK-hq^CNBe0+1`1Y5AEaX%WEqkZUcvNVQT`-2uHxTi6 zz~_oCE8W(Y&7NDT=yG0r8NXh^B3BJA&$<{XyLi*Ldo%ww@q|^CsEVfep0?@;-%xtU zMifbHiTe1J5XjjQQEqV`zs-;yvO;Wx4mAt*FS}t`CO!1Q`u*#tMwf<%l0}D{Xm*LsB z&c-^4ZUsN{*hj^aSZ}nrW|xQkbIoOnSYSvcRN<{|uzK7ikyO#;KZ;)5itG#EbkR3F z31B#f!~T)$k^7E79(|Kt#<(SNSH? zNSDTsO~n_)17e+Qcu<%$1$6Jg!}|L9v4(L79L5t}5}pa9^ztc26;|TA=C40*36{&< zy1%0lakD#`XArE$!_9bG~XJZ4qGJ&`bjI3ll_lCb|gT z#>b5POs5Ql!O;SnXdARU+L(>9wfn4WX$hJeF444?%q zFB`aqxYcep4E9W(C6!BWLGv;79#98*2T%c??akzlC0$mz<>eEaeZ`S??5cRq*9>W+ z){F~Vys{{kH-gENcnh$T-+Pa|Im#OoAbo#nMsU+@k}s z+NI?+E5kUwXP$LE7#e+F1j-hG^+ONOB_$+lI$|Gzq%9#eH8!V!5J-f%#@p1B0pILT zpk$Mu^@yrHPVwVi0o{Rd9?%So{nSMY#HWP<=FNm^(B7Du{@l?-(BL~5=b*TzOXAtAD_n!sHVNCp zPgrzwBR3N_lPR3`h&z`Am$@Fsa%cgCHm2I4jcttw4`9sLOU~@Q5|MQcV`%$N{;d>A zct*FuOH^zxTHk}Ar!6UC?_+99zJQRW@j>#$%!DI!Lk2<4*jP$e`KlvAMyUwH5Hl3v zt0hY-S^jzcdC#8r?)(rl_8}O~~Y!vTk?0I~h|6cuPhlh)~|l zwOrFZC=69D-xj`#gYmF-ZZI;2?p+Gk*^bni+>-fI+Z}qN+_@-Bx6vL04kl7T?%PBb zc#tXkVK%+7&wWSyL^NEn%1wq=HB#7ib1Q#Io!vtJF*@AEifThEE3P7r+$^h(+-(^x z5%Cz+hD00*FpW9O2xfxa@>oBZ<%Y)V9A^>H?jAY``jOP7c{nql$Ziktq6}XWE!pcp zUWBK#VXUEKc4tEnaL)aO{bK0-oie3?%(|naV}r)aCA^$2!EfOCg3JKC3U&`ymVFJ_ zkyB$Iq9!~dy8q0nXfxbc}$FqUq7Ty zW-m0S{sLSoRQhH8slzoh1X|mCCDhKM@w9z8;8KLLi&t!`HexPkc;T^o#2hxMg3JOj zGR6>_68IqboFroNd+TDlBJx0BOT{ziW!x$|q3lp}8IMjVSRrOvKfD^T-QC*G-N?+G zP-?fW0&g9h1BH&9e8RfhAS0R(8V2ZO1lMeYYLC^?2o$(-L4z#uuj(6{n(cc^Y-kvI z`k>*#gEPsR_X?MY zM--XZ)yp0zim$C}dhj55b_`^dFCpJ@qzNLy_=chJDnoXUTXL!JqM&aX76yrt-x5WR z+_YMnC%lMcupa-5J54}Lv|?G)Hh)9tkrCYKO#um!187eru+T*tE)$M z=Dc3#bQuoW4SWZnTbgZ(fsZ`bu(lH}++(AKv_?IWW-r?pj#V_?-@pYcaiQ#sVw zRID|QkN1>KItZ7ajg2kO%>f;KV?0)kQlkFDMVN_CN^{nr5R%T{q@YSHH)g?%FK&m4nD|iY9E6=;=LSGK( zessLW&_{`>NbNU2BM*L^-I#c|uzE&S_OTo7fZnYWY&4C6PKZ5%Y$M#iMc2F4oDcAk zSQln7+3xBB(7~!+eeOY9m<_;+eDit^I~q3%ZI9e#>M8mkr|Cs2eQUja1AYB{HC4VV zKwd0Z$NZaK=H;_;)-@dZU;3Flq4Vz%bn5P`=j9 z$$M_h_6{SvkG39V=`bF9Yjd+EB&6*1YK{a41@V8(4*%i*eCK&x_CK&Y+9Y#bF+c@`dm$={xbKRgZ>WIER zkgAZJ5z=J<++1B_k{Y2+#N*AUsD%B-#Aj48X2{5_Zc=HZGCy!E<2rUsRdI7-@=e7B zuK`4vN`4qknCgF!faBt!Yi9#}}ja6CY(y)dW?NO&AlB2+CW>dI-op`*6 zH&C5Qz>WP*FHw8UXyfc^GqO{b_2hMTHvy(hiA zC#|JLg>m9cO&I0h-)Va79d~F$Z^NO7Q0lR9R2XB^U&hwgnp$2Ud^6YJO1duGyZkZ3Nk$0 zMj)aehv80UszyI#0Fl`@4v?y%0`BG4{Dj8zvO=Lufot<{SC>!jOtGZf)d2FlO3{wac< zHVE`|4=G6MlC)LO-=M6BTkrKp>n93M(9HZ z5NH2aQ3KWdw}0Cr$-(4JC(I)||Zx&xSJCQ}}6q3lI|=aEZlEysWKPQL6B zVj4g+;}z-umU%(GcT6`puVVEdLAZcxX8&WJ)>&jVp^!Y zp|Cd(v+E|$#bSdc4orH%OO?$b;~;>r+}oCOgmXgOybkE zkJDND{!sS?fe_?5g|3(nn7j&TM7wnMb!;oAS;$LpmrqH=;24h;w$>Pfi87#T$KdKXyOK-0 z*X+j^bW;)3-YPJ8%TQ1Wnew@R!DbxYeZp>* zq+M(v%-rEltbuZ?F|x#PilG)5x+Ps-O^V~ru~l%9fLI6S4Zs%TeL^#5pJ8ZdsH>~r zb^o#s@CsnF(@!aGfU1p~?rgjqLva_;-Ca+0$VbaK%5cN?HP5|gPHH1Y*cfPd>E|lY zw8$AD~*SR zKnfxZpLFV*G{>sdlfnEu4)zY4P1*rGD&IZkstX=>VsfF(rwp)GXD4{#ClI#u_4fW) z(50WvSJTLxGlD@xS1q%|PARr2%CsY%NyUv~M!6<#iKT8_d4)g1OY?CaZ$^h_hv8L$ z3540$F!>!ecLw=+dwcsme5|fEw!SWxy zr|MCQtu#0%M4!2R6lG*vy}MWhMW2B%seUA8D(qpKk2Tph$j4h@N7SYS{@?p}HYsk}VT*BXYWEuo!ocHW_~;y3qtW4VQi6u+QPi{z`zC%ffQdp2HA-UYlTk z+>&|LMmsv}p)8~_EZPgO5Ql|DZ3OUWscTn!YRD+xz#=8K?qUUu&g!a?=+yljR(%C_#=Ou#w*IHo*fa%5|MERF}88ZsB zAXv?9f1I;8NB&4ne$aL&X26oyt#;KSW*N^tc6a8Y&qAZ7VDM}3N~hpUu)uJ2(onEe zHV{ye?w@z?{OU+Ef5>sIJ$OZvUlcr5zjPFgSE*Y}YSYp7jG~IZ3I1iDYd-$hW+obY z4ue`h0bA?iUo~bd z)DMOkH!$q?MP&fY{sj{2-HoEI3>c=@cm$yt4^@TOgySTQEfaCv&E_eKJCkFjAb3zn znVl%jHNk4%TRfxuhZR+SOAS!PVsqQF<65GuLcgE<#W1r0YSHa@5>8N62V)Mlz03lX zzgDjqG&NJR^snIiz-oF8|KFM={t6^F`aDk2ri!xGMrxM1f>A?3nqYcq$@i{=3Cx~g zSlU5h%vx_F1j-XZKEc_Vx4YZod(4bngAU z>F%32+c!Vn{EKawvDH;^KISa1{+MW6dV%1ZaKSG1!eh0)d(*D$cw920{*C!-=OtMN)h#*GGdRY`iK+kb5SD4<-%h!)_I4R&_!sYF}$kG zzO}t+nv;``{WZ_a$|{PA^c6~cnVtsB+(w3YPO&d__(J8_QCfQTzpS$`d+*d6?+^zOb-`lv#>hD86K447r(EUw`iB#?4`Go z=~}}d?u030UO|GD=AAos%=U}#Il-YC{0xx{7!-G0{4^Q#<2}DRmVITT$cT={1NDOE zevZ6l@9TbkFZGTcdwKQd&9%?nU>Tm_Fpp>E?JgwnA| z9*t#?by_I2FZW)WPWh1LbHm3+wr7U7RM6-;qvKUX)X*R?L)K{>M^w*{oan+Hx$ z^GDDlr$vlNDrC|twZGN;+qERPPB!=V-MM{dv!tZt^`@BE2Ia;W%Tqlvw%f>*{QB*%i)YYj_8Xow6t znPrcSKNzaKZBLHG>*@M52fyj<<_623a?JJTAQJFjnsu?2RT6!#mS?h-vQPsZN-Gru zD{JQZi+_iaC=;d=7U44I<~Rmz?sBKoV8c#QdSqm$*A9EbBS#F5G3G*kvm`kfjhK;| zh@_jFtja^8&h1g;?N4v?xp4#T3357CevCPQ1Qylh{eRzS1$kH}+?*2sdHq0~!MJ7>e zJw*scM#Hi8=eii|cJHOy7|nn$AXmWL*uiGKwP{n$!U)I=*2$bG)aDjA^79<|y#uS` z0)C`mu1~U>TXK<3qLPfgLs(b~V=+tHZvRcMpW5$()cKv;mmPGEytpy`OD|tPH;MN@ zb5uG$4htWPxF(}5+ZgLJv`nZ>JG+^_%&N7uwY}C)eoRhIj*n}Uj?VDv1Ns^1B|6L^ z9nu&BwidZ6J|%Nuz;k%I$2ake!a1 z^q(By&kY1E{}a48GwEy2j8ns533%7x46KUcEg3wc{VdPw!^oJ6ulFA_-CHU>)g={WWx?ND*GuYMS(+X(t?;*t zsqYB5pW4!`&TH&0XzW&x>yXDL$=#D9lUn~~$tkj_Fmu?5ik6?=V2Ti&jw4=nQ?nGL z?1cOR?$_}(7?buPhzJ4il}utqiJ*PygAduTM&1ma(t`?ZMLGGtTpgS-^n)r#g`nXv zpX!WSzB?W9ZkcxmsH(DnjV}i>QWYSJ>WS zi5{7ik7%F%!}q!UE`^;tclA{Utu4QbgAH9t{TuT{Y4L}2c05ygfM;v(SHvqaNVZlf zIqa^0>Z(uuJv}`GHN`c-@Bo9oyu4gp{hyVz8yDFs@k(+_Qvb(LDn=)29oR%e#Aq)h zrINJ7LI*c>GR}jX*QRD=T#iQ_LJTWdSQYXoEXo_SRyHmH``{)WgGHef(Gr99e@nS0 zvk%rz4-ZEg@}Un`kN@esbDi-wtchY3TqN1dINlQe0#(3_LRbSsy zhg(1Vo?)8a>>5|xVZ-F@H({l={2kAVe=2KiQhQdyi^I+h!S)*sO((O=WEA5%ar>~f z5gzE_O|n?BG!`XOsD@7hSK)GsCXdi8DKYGI_jhFJK_8cb@3*ne>iNZmEdPnc#awG^ z%#nv35eD*ph0l|_3OpOXdg}PYSK-e#8>|Gce$QHcuPJCPXw25aC*djJS>qFW8`V14 zq!74M32x9!J^iDwANSZXy~?ZO%OzpkBOmSFeM0%qk`2ViICLEqBU^{9ty2pj7A*xl zE4hA2(3)3ey27le@bBq~SqkXyKs0zDC^N3*X`(Ty6^SF+h$n%I`*WQLAtT}NS*DXv zvK}k@`v=4tC=a%2v*DPKi!dV`*t=w@1*@G~d-= z=56rQ)~rB5VP&vE`avlIeyeH@i;&N1A*qN(u?pyC+O1z-fwUb&E&&1FH*ek?|GXR* zw*Bqyx5pJM2o|^g(oRO7K!~wu;-8Hdb#*J!_MPn>s0kQ(&u?<`cV+G$=w=jmjLz6n z8PW_>hCBwdorVxHE;-?h2hCon;?dzhFMdEHllZIGZ<(P&sb8)mTYuT1TdWKWn!q|o znvm6DKb6JewiFXd##UGVIx{l(YR)Toq2i9-Y(v@og>R3qhs)A!?m%1YSHil(K@UJobBZ#z*SLl&apDB z+=%w4_8R8qsk=WU<>chRqS#%%NA%#T{|L}4u%as`p$RT^3?wcdvfC9!RW-pQ@7G(; z3)59MWMk?*47_-a&HrOC56WYS1f|3l65^D6Ma2wfOh8*8QYi9~t;46%QnkcD))ERv z+9>y`Cis(IR|}f$BqU~g>TJTn)t`j|3~y>z>2PB)#|i&x%vuq57XsTgaf@wvg#?PKu!72=8G}) z&i?-X9o_EV1S2)8O@{o}1Hr>(L7)T|wrrqKkL)JDNb@;aZqsY`qagb3 z-H+ZrdfQoeR`A_a^K8x4r+R1IjV>0c5JNmPM*aOO(iRH!7C(ortgQI!>dq`xHg>z{ zFg-gidUkYsbjWLHT*APmNipCC5hUb_DcKE;7CT{Tz)ZN{^OG6fHPPu*Hu#&>kq+n69X04~D z%ErU?p&SaM1X~o0MMR{e5X4CaOD2i@neUe3JU3P)dE9Z+hj#DlGvB@~znaThovr`3 z^GR05zK=T0oOZc2|DNBt$_QKg)rN*D%%D99X*f#oKn8aKst4d}EoDZE zX;5_*K7dQTi7f18X$$}IxK%T1;uRf|K75ZYXT;0t`%(e zSXE3}$_b#a2oEb;{W;I%I&4VORGI~#4;z>Y_|;3iZoaPa!p zAEy|LsftJyQz|aN|G-ZHU>6hwe(2mBwA>ogxp{NMhtg$o_bD2VQb=lq{7r)xJ^N|l@1nBseJT4;$)CXA08tMKDt@r6=R*+2yDFeNcBTeRp~qeg;hKcs zo+806Gw!(gh{`){n?%DOit)$Ew~&qQq9vpVGOnltn1%?m+U-hmhh&|I>M`Q;X7uf!{B17oPtJ{8lBD%x{kIFZ1EZ21UihRNhj@!~`Ir z2@f6Jqeqv17Os7*5so5Sd>aJRU7DI3MyVWck7vq4!~s@ZJQQ_6$^yG57tLXX%5+%J zj~g*HS{O?4>|7}Z^O6RZN=8Ib>}!J=cqRnSwW`DdQFau8{9+^F|IE|r1V-j_mF;m6 z=O7NnfGz4xeXyn94DhSfAr~2V$}Y)x>6Ey27_xZV(;W_eOL>p(`z>cjD)>#m^K$V2 z-mN9UxG;CQFni&besUHZtO}nOsI{?}tXQ0^T?~|P0@UdW2=exx9+V9FwpZ}g+F-sy zp{J*&4N;;UQTkddihA!!{uJL+rzItWx+O-7FJ9;LTPi~tk8YJ`@wrxzL^CiQm13{$a`L45_y5NIy&PY>P83Zvf|vduvPZISo+96EFkZA4YfX*@ z!CHhTR!`NziCI1RkX1d~nl!LDs${JY1cL0`(VPFy;zE{XndIvh|2RK-;gi%0wz7&3 zSp_s^S2yFw_=H1u50keu>wdtuFoosTwCcs^JAuxE>7WKQq2OKll@L}q_%9$VJd%4K z${nwZUx%TQC?R9@#6#*hDF!~IV2XnxvWzBVok5#aF<(*MTSI=zy78vKL42rXd9ds7 zxl&lgh-k1_1jX$lkJ;DAM|Kx#fTIAb&4w?6qDsd@=qXK5RI0~PV@BMsFV~72`VKM8 z^=5x6H3zJGf4%)McY)IiuYB$!N3xyy?W_jZ&ixcxDnWY-LiBw0sk^7Ir>8Ho(e>@y z-3rT}H`XlXX$Gx+QGjwR6RIgnq${2#e5kg2F$*}A1154Qhp?7NS*~&e(M(1Ixl7qZ zAS?CCs)&^1@kA#c3vp5+)l!cpX-U;`wJNnC_7BTV-TRwKZNw%dX9F%?90h)`!5BMy z9amdJi&sI%U-MSMyCxrela#*OFlg%J9ml|-1g>Q8SKR^qm%jFva}JF5m6pi24e<376p1h#Ix7E3dP1hxD*}Lu+3JST1&)49S}pD644@} z*?A$ADj`4rGC_o4MF~SRK9Pd1W#F2Ks#}Fm;uXG3lnp{sCiYh99NkJxe5x{tB3NBj zY2gC?H(dQFso=W^w#$IO_VvF0(PK|&lO-Sh75o&ITl`b<=G96#6s`-{VG#n&)K0w= zh@<%%K0O{c0=%bR0dHBevij9<>7BX3&(C2Ryzqkw3ICxYL`39Ib`RJx07j8&1QL^k zBdC5&e6tgNwNI6_E&Pn!2|&ZYs}KkWaO~8descXp+b=uS_UX!?Em6-C))6eQIzqMr zWbi=OsSeIz0Cz-*NR_99eM=ycNJwZXoB`5iaov07+eX9HSM8C72Fu^hH?Mszhiv(C zzd6oW6~~Xh>9_)&UZ}1=>vws)-xIGsKyLvEJq)T@bh2I$(mo6d#o#g>qqIv1B3Ot0m&|Ds?p6@H9MH zhFvT>qK#o6w!-tKZp)f)gyTA6Gy%^Zf;+gwPZqf3gHFVRgYYciP4p6s;QJ#^La=2c zE@y!ykVwY#4?~ z^Imj0GUdG@Pd%U>dg)Z1Cx|1`pe1)74PYA zhFNXkJ7R1ic#50)=vpp@TFYRe+3euAzdhQE2Nn8tb)?$8qAnWv99A!ZJrwKFu#KQ9 zf}1b`K4ow-O->Ykh}rKs5EC!LepSP%l0w7fZ%WO9f{nZ1zSW++JlC#eSqSpb=Qs`* z=#y2(Rz|k{A$CGYnY4}u#1;4r;IhWY$2rFgj`4pM1}=VZlVvXe5k}G^6$`L{7$HDAB=w9KQ05#KJ$VQCR+ZAJfbNmqZXT z6x*ac7>gyqyZH6e2(h6Q7e#i$f1U%Y0-pTdfm?DYb_pw<#8~&tgcw4SOXa@KNpB9E zEI+9bIGhk!SpD@uuDC-OUI}k0 zz{_iLn$Mq~Z_+BbZEha)r89WugX2=`+n}$ontqW^4BQW? zsey_BcSBOq80;sLJ_0G+^v?5=RPA;Wf*MRgU5fL^*4Fqx{wDWrk|I%F9CHsUS$Ph!(O0RlfAHy|pcAy# zU&z%4t7r_6Rtn{?97jeQp^K#41` zjs8pZX|Gt_kz;}F{tBEztK;;!F}iz-6^1^7hSbG=4}{>v_FR*IU;sa;Bh+|ot&l&7O+JWuZ%iWXqt@#0@e#WG?J9%U zrEC9H#7^vCu5O&r*tZdpQA(h_I1Yq#**XZ#EWkQYkHpC=D^Vbc}=U`S9ee+`~;ofM2LG;bAex{yo6 zquE!y;9^9;)DLea%PFg3a@34;P`S#Ag%76H*ese7ej({W21r7{Iv7!bRqB@^-|V@? zy&Vfd5JJE(g|T26(G2zcFYm~88ugjvyGTPsuOfxbF1b<$AdE)uVzjnK@`qv!1Ba#x z*9r@R<{o?L``)}_)BX$sFhD2Z*C&rp&dXo2rQc?GLqcd!XN$!)W%yE|iK(go8=a}| za0aaI>Ytvw&>lAFw~!_1auqC^3x>Z;9zMZD^@1K8c9TB?ouG+ z#-i%CL6C`Az{dBWK9OI!m#5;1Ql)LKD4T?nQEwu85C>0#){`9KK)t9@*vsMow#uSpgg*K>Lk@!;F2O)Jk@}#w z{--gG1jCK7wG5Jw`D#C!gEsjSo&v=U!^Wy$OhOYal(!;I&c;*}nhjnSk`9-paVmrT zY^`S{B)j_=dJDah3Vz?Rs+WJZD;O;Fet&nVFgwDa4ZLGuKF0xkJjXi4YA*4)jCxcm zi;SbjqqjVjk;ap7$|8v^jP%A);7Wunx=pb~RsTx$Lg0K+F#pfyRVZR9U-)0?s1(DZ zHU5yg5qq7JlE`rqQkeE$6gZu`=;6zXq1b&0ShGYeOvCmMKRZlK5ii%NKSX6{6l0X^ zhT+nJ5$p&z&WlaLy70HjqxX=^CY5dwO(Z z>a9#U3k%hgRl#%C!86|l^O2Ljo&Md<1G-8S|Y@w z4n`6pDV@sfbt2^ope|C!>#&zG4Wdp&ukx1Go4Oyayu7E4J@r(Mh5`FTAZlcXz`tUl z^1!pg4a$NTy)*=m-ZC8eCIWH@2QbV{$MhC{ToZf|@D;3UmRn{t`7YN3#3WvO%gtq5 z7xsf2dC92iDCVHyjezC6OLWThHj)cJ(*Vr3am;vq)BJo#z-8w-M}CUqQcn%PS7Ei& z@a95Af8Xe)a5jQLi>rofiK${*;IkoeVuKCA#-qV93ke1w3lSv|nuRd&44E86E-~V{ yRQxq~Q`?_QL9|;SI-)}`>UX#OGX7y%u+A&-WoS@@qVqcVp`AEQc|txH`u_m=PZOO0 diff --git a/integrations/lingbot_va/DESIGN.md b/integrations/lingbot_va/DESIGN.md deleted file mode 100644 index 4adba40f2..000000000 --- a/integrations/lingbot_va/DESIGN.md +++ /dev/null @@ -1,96 +0,0 @@ - - -# LingBot-VA V2 design record - -## Inputs and references - -- Target baseline: FlashDreams `8fd97fa38f04bc32c288760fa0fbf5da52464cea`. -- Draft integration: PR #312, - `f98cae4a18ddf6c189a6cfa2099265d6d570e337`. -- V2 reference application: `integrations_v2/red_screen` plus the finite - `color_fade` loop. -- Upstream inference reference: - `robbyant/lingbot-va@7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb`. -- Checkpoint snapshot: - `robbyant/lingbot-va-posttrain-robotwin@8c9dea8abbc5c91cc9e18bc3264b8915083bbe70`. - -## ADR-1: session-owned destructive engine - -Accepted: each session owns one `LingbotVAEngine`; the application owns only an -immutable config and an engine factory. - -The VAE decode cannot fit alongside the full DiT/text/cache footprint on the -supported capacity path. Generation therefore releases KV, DiT, tokenizer, -text encoder, and streaming-encoder caches before moving the VAE to the decode -device. That transition is destructive. Application-owned reusable model state -would promise reuse that the implementation cannot honor. - -Reset closes the current engine and clears the loop's finished flag. The next -step constructs a fresh engine lazily. Close is idempotent. A failed run closes -partial state while preserving the original inference exception even when -cleanup also fails. - -## ADR-2: generic typed action artifacts - -Accepted: extend the generic V2 result/session/sink contracts with named tensor -artifacts rather than hiding actions in LingBot-specific files or metadata. - -`SessionDesc.tensor_artifact_schemas` declares `actions[step, channel]`. -`StepResult.tensor_artifacts` carries the tensor. The generic -`TensorArtifactOutputSink` concatenates declared chunks and atomically writes -`actions.npy`. LingBot code never imports that sink and never chooses an output -path. - -## ADR-3: one honest model step with deferred decode - -Accepted: the first V2 version generates N dual-stream chunks, releases -denoising state, decodes accumulated video frame-by-frame, and returns one -`StepResult`. - -This keeps the UI thread independent of model execution without claiming -per-chunk presentation. A streaming cadence can be added only after a measured -decode path fits without invalidating cache/model ownership. - -## State machine - -`NEW -> RUNNING -> FINISHED -> CLOSED` is the successful engine path. -Any exception from `RUNNING` triggers cleanup and transitions to `CLOSED`. -Calling `run` outside `NEW` is an error. Session reset replaces the closed or -finished engine with a new `NEW` instance on the next model step. - -## Memory ownership - -| Phase | GPU/active | CPU/host | Released at boundary | -| --- | --- | --- | --- | -| load | DiT; optionally VAE/T5 | tokenizer; offloaded components | partial state on failure | -| encode | T5 then VAE as needed | three input PNGs | prompt/observation temporaries | -| denoise | DiT, CFG caches, latent/action state | accumulated completed chunks | per-step temporaries | -| teardown | VAE only after transfer | DiT/T5/tokenizer references | all KV and denoising state | -| decode | VAE plus one decoded frame | accumulated output frames/actions | VAE cache and each GPU frame | -| finished | none | returned video/actions/metrics | all model components | - -## Fixed contracts - -- Robotwin layout: high camera full resolution above two half-resolution wrists. -- Video: TCHW, 256x320 high-camera crop, 10 FPS, float `[-1, 1]`. The VAE - decodes `2N` latent frames to `8N - 3` pixel frames. -- Actions: 32 steps per chunk, 16 channels in order - `0..6, 28, 7..13, 29`. -- Default CFG: video scale 5, action scale 1. Conditional and unconditional - branches own distinct video KV and both branches advance whenever a CFG cache - exists. Every action denoise pass attends to committed prior chunks plus the - matching branch's current video KV before its own fresh action KV. -- Cache attention window: 72, matching pinned upstream Robotwin config. -- Checkpoints: local root or revision-aware Hugging Face snapshot with explicit - component subfolders. - -## Deliberate exclusions - -- no legacy V1 runner or `flashdreams.runner_configs` entry point; -- no application-owned files, MP4 encoder, threads, or model components; -- no multi-GPU/FSDP claim; -- no speedup claim without matched-output evidence; -- no root CUDA/Torch policy change. diff --git a/integrations/lingbot_va/GPU_EVIDENCE.md b/integrations/lingbot_va/GPU_EVIDENCE.md deleted file mode 100644 index 05931f096..000000000 --- a/integrations/lingbot_va/GPU_EVIDENCE.md +++ /dev/null @@ -1,99 +0,0 @@ - - -# LingBot-VA GPU and parity evidence - -This record was produced on 2026-08-25 from FlashDreams baseline -`8fd97fa38f04bc32c288760fa0fbf5da52464cea` and this integration worktree. It -is validation evidence, not a general performance claim. - -## Fixed inputs - -- GPU: NVIDIA RTX PRO 6000 Blackwell Workstation Edition, 97,887 MiB. -- Driver: 595.84. -- PyTorch/CUDA: 2.12.1+cu130 / CUDA 13.0. -- Upstream source: `robbyant/lingbot-va` at - `7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb`. -- Checkpoint: `robbyant/lingbot-va-posttrain-robotwin` at - `8c9dea8abbc5c91cc9e18bc3264b8915083bbe70`. -- Input PNGs: the upstream Robotwin example files and hashes recorded in - `assets/example_data/lingbot-va/robotwin/README.md`. -- Precision/device: BF16, one CUDA device, seed 42. - -The real checkpoint contained 841 transformer entries. Two obsolete -`patch_embedding.*` entries are intentionally dropped; all 839 remaining keys -mapped bijectively to the 839 native network entries. Strict loading produced a -5,088,872,670-parameter transformer. - -## Upstream flow parity - -`tools/compare_upstream.py` loads the pinned upstream and native transformers -sequentially, then runs the same first-chunk video followed by action tensors -through the same cache lifecycle. The explicit acceptance gates are maximum -absolute error <= 0.07 and mean absolute error <= 0.012 for both streams. - -| Native mode | Stream | Maximum absolute error | Mean absolute error | RMS error | -| --- | --- | ---: | ---: | ---: | -| eager | video | 0.04296875 | 0.00751040 | 0.00951632 | -| eager | action | 0.06250000 | 0.00875314 | 0.01267146 | -| compiled | video | 0.05468750 | 0.00970979 | 0.01229867 | -| compiled | action | 0.06250000 | 0.01116651 | 0.01560142 | - -The comparison caught and then regression-tested a defect where the native -action block loop received current-video KV but omitted it from attention. The -pre-fix action maximum/mean errors were 1.015625/0.203186. The table contains -the post-fix measurements. - -Reproduce the compiled bound check from the repository root: - -```bash -PYTHONPATH=flashdreams:integrations/lingbot_va \ -python integrations/lingbot_va/tools/compare_upstream.py \ - --checkpoint-root /path/to/resolved/snapshot \ - --upstream-root /path/to/robbyant-lingbot-va-7c6ffa9 \ - --compile-native \ - --maximum-video-error 0.07 --mean-video-error 0.012 \ - --maximum-action-error 0.07 --mean-action-error 0.012 -``` - -## Real multi-chunk V2 run - -Matched GPU-resident and offloaded runs used two chunks, default CFG (video 5, -action 1), 25 video steps, 50 action steps, and compilation disabled to isolate -model correctness. Both produced: - -- video `[13, 3, 256, 320]`, finite BF16; -- actions `[64, 16]`, finite float32; -- different stable chunk means: 0.179792 and 0.333868; -- a valid 13-frame, 320x256, 10 FPS H.264 MP4; -- byte-identical resident/offload video and action artifacts. - -| Artifact | SHA-256 | -| --- | --- | -| `demo.mp4` | `df0c193137a673f4f8d6b2372b4bf7afc01a9937c4280b7fa5a51912b5e93c1a` | -| `actions.npy` | `463b307b667c1ca13a47bbbc5a17f68604621dfe3c3a10fc5860077216928d95` | - -Fresh-process engine measurements were: - -| Mode | Prompt | Observation | Denoise | Decode | Total | Peak allocation | -| --- | ---: | ---: | ---: | ---: | ---: | ---: | -| resident | 0.240 s | 0.221 s | 4.735 s | 0.260 s | 33.220 s | 43,329,760,768 B (40.35 GiB) | -| offload | 5.055 s | 0.776 s | 4.744 s | 0.429 s | 34.009 s | 39,804,415,488 B (37.07 GiB) | - -After `close`, only the process CUDA allocator context remained (about 0.031 -GiB); model components and caches were released. Compilation was separately -exercised through a complete one-chunk engine run and returned finite -`[5, 3, 256, 320]` video and `[32, 16]` actions. Cold compilation is excluded -from the table and no speedup claim is made. - -## Remaining experimental limits - -- One GPU only; no FSDP or context-parallel claim. -- The engine returns one complete rollout step after destructive teardown and - decode; it does not promise per-chunk interactive presentation. -- The VAE integration intentionally depends on Diffusers 0.38 private - streaming state and must be retested before the dependency window is widened. -- The official checkpoint currently carries two obsolete patch-embedding keys; - both upstream and native loaders ignore/drop the same entries. diff --git a/integrations/lingbot_va/README.md b/integrations/lingbot_va/README.md index f6bbdda03..087fe8145 100644 --- a/integrations/lingbot_va/README.md +++ b/integrations/lingbot_va/README.md @@ -10,30 +10,19 @@ This workspace package implements the LingBot-VA dual video/action model. The V2 application adapter lives in `integrations_v2/lingbot_va`; model code here has no CLI, MP4, metrics-file, or action-file ownership. -The port is based on FlashDreams PR #312, with its CFG cache ownership, -checkpoint loading, configuration propagation, lifecycle, and output contracts -reworked for the V2 API. The original unverified 2.3x/1.48x performance claims -have been removed; only measurements produced by the checked-in implementation -and matched parity harness are reported. +The implementation transplants the useful model work from draft PR #312 and is +otherwise self-contained. CFG cache ownership, checkpoint loading, configuration +propagation, lifecycle, and output contracts were corrected for the V2 API. The +draft performance claims are intentionally omitted; only matched measurements +from the checked-in implementation are recorded below. -## Install +## Install and run From the repository root: ```bash uv sync --project integrations_v2/lingbot_va -``` - -The tested model dependency window is Diffusers 0.38.x and Transformers 5.x. -The engine uses private Wan VAE streaming fields, so widening the Diffusers -range requires a real-model retest. - -## Run through V2 -The reproducible default uses the official checkpoint revision -`8c9dea8abbc5c91cc9e18bc3264b8915083bbe70`: - -```bash uv run --project integrations_v2/lingbot_va flashdreams-run-v2 \ lingbot-va-robotwin-i2av \ --mode mp4 \ @@ -43,89 +32,201 @@ uv run --project integrations_v2/lingbot_va flashdreams-run-v2 \ -- \ --checkpoint-root robbyant/lingbot-va-posttrain-robotwin \ --checkpoint-revision 8c9dea8abbc5c91cc9e18bc3264b8915083bbe70 \ - --input-image-dir assets/example_data/lingbot-va/robotwin \ + --input-image-dir /path/to/robotwin-images \ --num-chunks 10 ``` -Use `--no-compile` for correctness debugging and `--enable-offload` when GPU -memory is constrained. `flashdreams-run-v2 lingbot-va-robotwin-i2av -- --help` -lists every effective model override. +`--input-image-dir` is required; this repository does not bundle example +images. Use `--no-compile` for correctness debugging and `--enable-offload` +when GPU memory is constrained. +`flashdreams-run-v2 lingbot-va-robotwin-i2av -- --help` lists every model +override. -### Checkpoint modes +The tested dependency window is Diffusers 0.38.x and Transformers 5.x. The +engine uses private Wan VAE streaming fields, so widening the Diffusers range +requires a real-model retest. -`--checkpoint-root` accepts either: +## Inputs and checkpoints -- a local snapshot root containing `transformer/`, `vae/`, `text_encoder/`, - and `tokenizer/`; or -- a Hugging Face repository ID, optionally pinned with - `--checkpoint-revision`. - -Existing paths are always treated as local. Prefix a not-yet-created relative -local path with `./` so it fails as a local path rather than being interpreted -as a repository ID. All `from_pretrained` calls use a resolved root plus an -explicit subfolder and `local_files_only=True`. - -### Three-camera inputs - -The input directory must contain: +The input directory must contain exactly named Robotwin camera files: - `observation.images.cam_high.png` - `observation.images.cam_left_wrist.png` - `observation.images.cam_right_wrist.png` The high camera is encoded at 256x320. Each wrist camera is encoded at 128x160, -and their latents form the upper bar of the upstream Robotwin T layout. The -repository defaults are the official upstream example images; their source and -hashes are recorded beside the assets. +and their latents form the upper bar of the upstream Robotwin T layout. + +The measured runs used the unmodified examples from +`robbyant/lingbot-va@7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb`, +under `example/robotwin/`. Their upstream introduction commit is +`5ed0eb32046b34fe5c14f929d81e87ab6ebe02ef`. + +| File | SHA-256 | +| --- | --- | +| `observation.images.cam_high.png` | `78cab76d394114ba912f882ac9b00ddc017f98c946482cb9267c87de82486b72` | +| `observation.images.cam_left_wrist.png` | `fbe55b713e1b3d4505fda6be3b00132213ee1725a78cb357ed2bbd5b3a3a7a93` | +| `observation.images.cam_right_wrist.png` | `b9e6821b38073567232f6dd8f6389b123d09d3d1a70758c27d388e547e228249` | -### Outputs +`--checkpoint-root` accepts either a local snapshot containing +`transformer/`, `vae/`, `text_encoder/`, and `tokenizer/`, or a Hugging +Face repository ID optionally pinned with `--checkpoint-revision`. Existing +paths are always local. Prefix a missing relative local path with `./` so it +fails locally rather than being interpreted as a repository ID. Component +loads use one resolved root, explicit subfolders, and `local_files_only=True`. -One V2 model step returns the complete rollout: +## V2 outputs + +One model step returns the complete rollout: - video: float tensor `[time, 3, 256, 320]`, range `[-1, 1]`, 10 FPS; -- `actions` artifact: float tensor `[step, channel]`; +- `actions`: float tensor artifact `[step, channel]`; - timing and peak-allocation metrics. -Each chunk produces 2 latent frames and 32 action steps. Wan's temporal decoder +Each chunk produces two latent frames and 32 action steps. Wan temporal decoding turns `2N` accumulated latent frames into `8N - 3` pixel frames. The decoded -T-layout is cropped to its 256x320 high-camera view for the V2 video channel; -the action artifact has 16 selected Robotwin channels, ordered by channel IDs -`0..6, 28, 7..13, 29`. MP4, JSON, and NumPy serialization belong to generic V2 -runtime sinks; the engine itself performs no output I/O. +T layout is cropped to its 256x320 high-camera view. Actions contain 16 selected +Robotwin channels in the order `0..6, 28, 7..13, 29`. + +MP4, JSON, and NumPy serialization belong to generic V2 runtime sinks. The +adapter declares `actions[step, channel]` once and attaches the tensor to its +single model-result channel. Backpressure and presentation policies selected by +the runtime are preserved; the fixed video layout, dimensions, rate, and +artifact schema are validated. + +## Design and lifecycle + +Each session owns one `LingbotVAEngine`; the application owns only immutable +configuration and an engine factory. The successful engine state machine is +`NEW -> RUNNING -> FINISHED -> CLOSED`. Calling `run` outside `NEW` is an +error. Any inference failure triggers cleanup and transitions to `CLOSED`. + +VAE decoding cannot fit beside the complete DiT, text encoder, and KV footprint +on the supported capacity path. Generation therefore releases denoising state +before moving the VAE to the decode device. This transition is destructive, so +the adapter emits one honest long model step rather than claiming per-chunk +interactive presentation. Reset closes the current engine; the next step lazily +creates a fresh one. Close is idempotent, and cleanup errors do not replace an +earlier inference failure. + +| Phase | GPU/active | CPU/host | Released at boundary | +| --- | --- | --- | --- | +| load | DiT; optionally VAE/T5 | tokenizer; offloaded components | partial state on failure | +| encode | T5 and VAE as needed | three input PNGs | prompt/observation temporaries | +| denoise | DiT, CFG caches, latent/action state | completed chunks | per-step temporaries | +| teardown | VAE after transfer | DiT/T5/tokenizer references | KV and denoising state | +| decode | VAE plus one decoded frame | output frames/actions | VAE cache and GPU frames | +| finished | none | returned video/actions/metrics | all model components | + +The default CFG scales are video 5 and action 1. Conditional and unconditional +branches own distinct video KV and both advance whenever CFG is active. Each +action denoise pass attends to committed prior chunks plus its matching branch's +current video KV before adding fresh action KV. The cache attention window is 72, +matching the pinned upstream Robotwin configuration. + +Deliberate exclusions are the V1 runner, application-owned output files, +application-owned model components, multi-GPU/FSDP claims, live RoboTwin +control, unmeasured speedup claims, and root CUDA/Torch policy changes. + +## Validation evidence + +The evidence below was produced on 2026-08-25 using an NVIDIA RTX PRO 6000 +Blackwell Workstation Edition (97,887 MiB), driver 595.84, PyTorch 2.12.1+cu130, +CUDA 13.0, BF16, one CUDA device, and seed 42. It is validation evidence, not a +general performance claim. + +The official checkpoint revision +`8c9dea8abbc5c91cc9e18bc3264b8915083bbe70` contained 841 transformer entries. +Two obsolete `patch_embedding.*` entries are deliberately dropped. All 839 +remaining keys map bijectively to the 839 native entries, producing a +5,088,872,670-parameter transformer under strict loading. + +### Upstream flow parity + +`tools/compare_upstream.py` loads pinned upstream and native transformers +sequentially, then drives the same first-chunk video and action tensors through +the same cache lifecycle. Acceptance gates are maximum absolute error <= 0.07 +and mean absolute error <= 0.012 for each stream. + +| Native mode | Stream | Maximum absolute error | Mean absolute error | RMS error | +| --- | --- | ---: | ---: | ---: | +| eager | video | 0.04296875 | 0.00751040 | 0.00951632 | +| eager | action | 0.06250000 | 0.00875314 | 0.01267146 | +| compiled | video | 0.05468750 | 0.00970979 | 0.01229867 | +| compiled | action | 0.06250000 | 0.01116651 | 0.01560142 | + +The comparison exposed an action-attention defect: current-video KV was passed +into the native action block loop but omitted from attention. Before the fix, +action maximum/mean errors were 1.015625/0.203186. The table records the fixed +behavior, now covered by CPU cache regressions. + +```bash +PYTHONPATH=flashdreams:integrations/lingbot_va \ +python integrations/lingbot_va/tools/compare_upstream.py \ + --checkpoint-root /path/to/resolved/snapshot \ + --upstream-root /path/to/robbyant-lingbot-va-7c6ffa9 \ + --compile-native \ + --maximum-video-error 0.07 --mean-video-error 0.012 \ + --maximum-action-error 0.07 --mean-action-error 0.012 +``` + +### Real multi-chunk V2 run -## Lifecycle and limitations +Matched resident and offloaded two-chunk runs used default CFG, 25 video steps, +50 action steps, and no compilation. Both produced finite BF16 video +`[13, 3, 256, 320]`, finite float32 actions `[64, 16]`, distinct chunk means +0.179792 and 0.333868, a valid 13-frame 320x256 10 FPS H.264 MP4, and +byte-identical resident/offload outputs. -Video decoding needs the DiT, text encoder, and KV state released first. A -session therefore owns a destructive, one-run engine. Reset closes that engine -and lazily creates a new one. The initial implementation honestly returns one -long model step after all chunks are generated and decoded; it does not claim -per-chunk interactive streaming. +| Artifact | SHA-256 | +| --- | --- | +| `demo.mp4` | `df0c193137a673f4f8d6b2372b4bf7afc01a9937c4280b7fa5a51912b5e93c1a` | +| `actions.npy` | `463b307b667c1ca13a47bbbc5a17f68604621dfe3c3a10fc5860077216928d95` | -Only one GPU is currently supported. Multi-GPU/FSDP execution and live -RoboTwin control are outside this I2AV adapter. See `DESIGN.md` for ownership, -state transitions, and failure semantics. See `GPU_EVIDENCE.md` for exact -checkpoint parity bounds, real multi-chunk artifacts, memory measurements, and -the opt-in reproduction commands. +| Mode | Prompt | Observation | Denoise | Decode | Total | Peak allocation | +| --- | ---: | ---: | ---: | ---: | ---: | ---: | +| resident | 0.240 s | 0.221 s | 4.735 s | 0.260 s | 33.220 s | 40.35 GiB | +| offload | 5.055 s | 0.776 s | 4.744 s | 0.429 s | 34.009 s | 37.07 GiB | -## Real-model verification +After close, only the process CUDA allocator context remained (about 0.031 +GiB). A separate complete compiled one-chunk run returned finite +`[5, 3, 256, 320]` video and `[32, 16]` actions. Cold compilation is excluded +from the timing table and no speedup claim is made. -The checked-in GPU test is opt-in because the checkpoint is about 23 GiB: +Run the opt-in production test with explicit input images: ```bash -LINGBOT_VA_REAL_MODEL_RUN=1 uv run --no-sync pytest \ - integrations_v2/lingbot_va -m ci_gpu -s +LINGBOT_VA_REAL_MODEL_RUN=1 \ +LINGBOT_VA_INPUT_DIR=/path/to/robotwin-images \ +uv run --no-sync pytest integrations_v2/lingbot_va -m ci_gpu -s ``` -Set `LINGBOT_VA_CHECKPOINT_ROOT` to reuse a resolved local snapshot. The -separate `LINGBOT_VA_REAL_MODEL_COMPILE_RUN=1` gate exercises the cold -`torch.compile` path. The upstream comparison harness and accepted numerical -bounds are documented in `GPU_EVIDENCE.md`. +Set `LINGBOT_VA_CHECKPOINT_ROOT` to reuse a local snapshot. +`LINGBOT_VA_REAL_MODEL_COMPILE_RUN=1` separately enables the cold compile test. + +### Final stacked-PR revalidation + +After the PR split, the same pinned checkpoint and input hashes were run through +the final stacked code with two chunks, default CFG, offload, and no compilation: + +- GPU test: 1 passed in 38.56 s; +- MP4: H.264, 320x256, 10 FPS, 13 frames, SHA-256 + `d462cd1f0ef5afe01733afb9dd72aff1ac9365780dc078429223188c15ece4b6`; +- actions: float32 `[64, 16]`, finite, distinct chunks, SHA-256 + `463b307b667c1ca13a47bbbc5a17f68604621dfe3c3a10fc5860077216928d95`; +- peak allocation: 39,804,415,488 bytes (37.07 GiB). + +The action hash exactly matches the earlier resident/offload parity run. + +## Remaining limits and provenance -## Provenance +The current implementation supports one GPU, one complete deferred-decode +rollout, and Diffusers 0.38 private VAE streaming state. It makes no FSDP, +context-parallel, or per-chunk presentation claim. -- Source architecture/inference reference: +- Source architecture/inference: `robbyant/lingbot-va@7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb`. -- Official Robotwin checkpoint: +- Official checkpoint: `robbyant/lingbot-va-posttrain-robotwin@8c9dea8abbc5c91cc9e18bc3264b8915083bbe70`. -- Initial FlashDreams draft source: PR #312 at - `f98cae4a18ddf6c189a6cfa2099265d6d570e337`. +- Initial draft source: + FlashDreams PR #312 at `f98cae4a18ddf6c189a6cfa2099265d6d570e337`. diff --git a/integrations/lingbot_va/lingbot_va/config.py b/integrations/lingbot_va/lingbot_va/config.py index 0605ee2ec..e55576d04 100644 --- a/integrations/lingbot_va/lingbot_va/config.py +++ b/integrations/lingbot_va/lingbot_va/config.py @@ -78,6 +78,3 @@ shift=ROBOTWIN_ACTION_SNR_SHIFT, ), ) -PIPELINE_CONFIGS: dict[str, LingbotVAInferencePipelineConfig] = { - PIPELINE_LINGBOT_VA_ROBOTWIN_I2AV.name: PIPELINE_LINGBOT_VA_ROBOTWIN_I2AV, -} diff --git a/integrations/lingbot_va/lingbot_va/constants.py b/integrations/lingbot_va/lingbot_va/constants.py index 80d17ff06..cfd077f98 100644 --- a/integrations/lingbot_va/lingbot_va/constants.py +++ b/integrations/lingbot_va/lingbot_va/constants.py @@ -17,12 +17,8 @@ from __future__ import annotations -from pathlib import Path - RUNNER_NAME_ROBOTWIN_I2AV = "lingbot-va-robotwin-i2av" DEFAULT_CHECKPOINT_ROOT = "robbyant/lingbot-va-posttrain-robotwin" -DEFAULT_INPUT_IMAGE_DIR = Path("assets/example_data/lingbot-va/robotwin") -DEFAULT_OUTPUT_DIR = Path("outputs/lingbot_va/robotwin_i2av") DEFAULT_PROMPT = ( "Grab the medium-sized white mug, rotate it, place it on the table, " "and hook it onto the smooth dark gray rack." diff --git a/integrations/lingbot_va/lingbot_va/engine.py b/integrations/lingbot_va/lingbot_va/engine.py index 25fe4e7f2..9f5d72fed 100644 --- a/integrations/lingbot_va/lingbot_va/engine.py +++ b/integrations/lingbot_va/lingbot_va/engine.py @@ -46,7 +46,6 @@ from lingbot_va.config import PIPELINE_LINGBOT_VA_ROBOTWIN_I2AV from lingbot_va.constants import ( DEFAULT_CHECKPOINT_ROOT, - DEFAULT_INPUT_IMAGE_DIR, DEFAULT_PROMPT, ROBOTWIN_ACTION_GUIDANCE_SCALE, ROBOTWIN_ACTION_INFERENCE_STEPS, @@ -71,7 +70,7 @@ class LingbotVAEngineConfig: checkpoint_root: str | Path = DEFAULT_CHECKPOINT_ROOT checkpoint_revision: str | None = None - input_image_dir: Path = DEFAULT_INPUT_IMAGE_DIR + input_image_dir: Path prompt: str | Path = DEFAULT_PROMPT num_chunks: int = 10 seed: int = 42 diff --git a/integrations/lingbot_va/tests/test_engine.py b/integrations/lingbot_va/tests/test_engine.py index f17925873..11c5fc9dd 100644 --- a/integrations/lingbot_va/tests/test_engine.py +++ b/integrations/lingbot_va/tests/test_engine.py @@ -17,6 +17,7 @@ from collections.abc import Callable from pathlib import Path +from typing import Any import pytest import torch @@ -36,6 +37,11 @@ pytestmark = pytest.mark.ci_cpu +def _config(**changes: Any) -> LingbotVAEngineConfig: + """Build a config with an explicit inert input directory.""" + return LingbotVAEngineConfig(input_image_dir=Path("."), **changes) + + class _StandInEngine(LingbotVAEngine): """Return fixed CPU tensors without loading external model packages.""" @@ -58,7 +64,7 @@ def _release_denoising_state(self) -> None: def test_pipeline_config_applies_every_model_override(tmp_path: Path) -> None: - config = LingbotVAEngineConfig( + config = _config( seed=17, compile_network=False, guidance_scale=2.5, @@ -92,7 +98,7 @@ def test_pipeline_config_applies_every_model_override(tmp_path: Path) -> None: def test_engine_is_one_run_and_close_is_idempotent() -> None: - engine = _StandInEngine(LingbotVAEngineConfig()) + engine = _StandInEngine(_config()) output = engine.run() @@ -108,7 +114,7 @@ def test_engine_is_one_run_and_close_is_idempotent() -> None: def test_engine_failure_closes_partial_state() -> None: - engine = _FailingEngine(LingbotVAEngineConfig()) + engine = _FailingEngine(_config()) with pytest.raises(RuntimeError, match="inference failed"): engine.run() @@ -134,7 +140,7 @@ def test_validate_input_images_returns_camera_mapping(tmp_path: Path) -> None: def test_expected_shape_scales_only_with_chunk_count() -> None: - config = LingbotVAEngineConfig(num_chunks=3) + config = _config(num_chunks=3) assert expected_output_shape(config) == (21, 3, 256, 320) @@ -142,13 +148,13 @@ def test_expected_shape_scales_only_with_chunk_count() -> None: @pytest.mark.parametrize( ("config_factory", "message"), [ - (lambda: LingbotVAEngineConfig(num_chunks=0), "num_chunks"), + (lambda: _config(num_chunks=0), "num_chunks"), ( - lambda: LingbotVAEngineConfig(video_inference_steps=0), + lambda: _config(video_inference_steps=0), "step counts", ), - (lambda: LingbotVAEngineConfig(video_snr_shift=0.0), "SNR shifts"), - (lambda: LingbotVAEngineConfig(guidance_scale=-1.0), "guidance scales"), + (lambda: _config(video_snr_shift=0.0), "SNR shifts"), + (lambda: _config(guidance_scale=-1.0), "guidance scales"), ], ) def test_engine_config_rejects_invalid_values( diff --git a/integrations_v2/README.md b/integrations_v2/README.md index 0bed0f0fb..e0a74a3bc 100644 --- a/integrations_v2/README.md +++ b/integrations_v2/README.md @@ -23,6 +23,7 @@ follows is already done for you. - `t2v_self_forcing`, `t2v_causal_forcing`, `t2v_fastvideo_causal_wan22`, `t2v_wan21`, `t2v_cosmos_predict2` — real models, each a thin wrapper over `flashdreams.t2v_v2`. +- `lingbot_va` — the LingBot-VA Robotwin image-to-action/video model. - `null_model` — not an application. A v1 pipeline the framework tests use as a fixture. diff --git a/integrations_v2/lingbot_va/README.md b/integrations_v2/lingbot_va/README.md index 3b28aa20f..50933ca1e 100644 --- a/integrations_v2/lingbot_va/README.md +++ b/integrations_v2/lingbot_va/README.md @@ -19,16 +19,20 @@ uv run --project integrations_v2/lingbot_va flashdreams-run-v2 \ -- \ --checkpoint-root robbyant/lingbot-va-posttrain-robotwin \ --checkpoint-revision 8c9dea8abbc5c91cc9e18bc3264b8915083bbe70 \ - --input-image-dir assets/example_data/lingbot-va/robotwin \ + --input-image-dir /path/to/robotwin-images \ --num-chunks 10 ``` The application describes its natural session before initialization: TCHW, 256x320, 10 FPS, blocking backpressure, present-only-new behavior, and an -`actions[step, channel]` tensor artifact. Model loading is lazy on the model -thread. The finite loop emits one complete rollout and then reports finished. +`actions[step, channel]` tensor artifact. Runtime backpressure and presentation +overrides are preserved; fixed model output properties are validated. Model +loading is lazy on the model thread, and the finite loop emits one complete +rollout before reporting finished. -Use `-- --help` after the application slug for checkpoint, input, compilation, +`--input-image-dir` is required because no camera images are bundled. Use +`-- --help` after the application slug for checkpoint, input, compilation, offload, seed, guidance, inference-step, and scheduler-shift overrides. The -model package README documents checkpoint modes, camera/action contracts, -provenance, opt-in GPU tests, measured parity evidence, and limitations. +[model package README](../../integrations/lingbot_va/README.md) documents input +provenance, checkpoint modes, camera/action contracts, lifecycle, opt-in GPU +tests, measured parity evidence, and limitations. diff --git a/integrations_v2/lingbot_va/lingbot_va_v2/app.py b/integrations_v2/lingbot_va/lingbot_va_v2/app.py index 258fca343..b40a0e3e5 100644 --- a/integrations_v2/lingbot_va/lingbot_va_v2/app.py +++ b/integrations_v2/lingbot_va/lingbot_va_v2/app.py @@ -19,14 +19,13 @@ import argparse from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace from pathlib import Path from typing import Protocol from lingbot_va._loaders import validate_checkpoint_root from lingbot_va.constants import ( DEFAULT_CHECKPOINT_ROOT, - DEFAULT_INPUT_IMAGE_DIR, DEFAULT_PROMPT, ROBOTWIN_ACTION_DIM, ROBOTWIN_ACTION_GUIDANCE_SCALE, @@ -244,9 +243,15 @@ def create_session(self, session_desc: SessionDesc) -> ISession: ) canonical = _session_desc() _validate_requested_session(session_desc, canonical) + resolved = replace( + canonical, + backpressure_mode=session_desc.backpressure_mode, + presentation_mode=session_desc.presentation_mode, + metadata={**session_desc.metadata, **canonical.metadata}, + ) return LingbotVASession( self._config, - canonical, + resolved, self._engine_factory, ) @@ -262,7 +267,8 @@ def _parse_args(commandline_args: Sequence[str]) -> argparse.Namespace: parser.add_argument( "--input-image-dir", type=Path, - default=DEFAULT_INPUT_IMAGE_DIR, + required=True, + help="Directory containing the three Robotwin camera PNGs.", ) prompt_group = parser.add_mutually_exclusive_group() prompt_group.add_argument("--prompt", default=DEFAULT_PROMPT) @@ -332,11 +338,9 @@ def _validate_requested_session( requested: SessionDesc, canonical: SessionDesc, ) -> None: - """Reject runtime requests that would misdescribe fixed Robotwin output.""" + """Reject fixed-output changes while accepting runtime presentation policies.""" fields = ( "output_layout", - "backpressure_mode", - "presentation_mode", "frames_per_second_for_ui", "frames_per_second_for_step", "video_width", diff --git a/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py b/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py index c4ed7ab87..85a247010 100644 --- a/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py +++ b/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py @@ -230,6 +230,25 @@ def test_create_session_rejects_misdescribed_robotwin_output(tmp_path: Path) -> app.create_session(replace(app.session_desc(), video_width=640)) +def test_create_session_preserves_runtime_policies(tmp_path: Path) -> None: + app = _init_app(tmp_path, _FakeEngineFactory()) + requested = replace( + app.session_desc(), + backpressure_mode=BackpressureMode.DROP_OLDEST, + presentation_mode=PresentationMode.ONLY_PRESENT_NEWEST, + metadata={"caller": "preserved"}, + ) + + session = app.create_session(requested) + + assert session.session_desc.backpressure_mode is BackpressureMode.DROP_OLDEST + assert ( + session.session_desc.presentation_mode is PresentationMode.ONLY_PRESENT_NEWEST + ) + assert session.session_desc.metadata["caller"] == "preserved" + assert session.session_desc.metadata["action_dim"] == 30 + + def test_create_session_before_init_fails() -> None: app = LingbotVAApplication(_FakeEngineFactory()) @@ -237,6 +256,13 @@ def test_create_session_before_init_fails() -> None: app.create_session(app.session_desc()) +def test_init_requires_an_explicit_input_image_directory() -> None: + app = LingbotVAApplication(_FakeEngineFactory()) + + with pytest.raises(SystemExit): + app.init(["--device", "cpu"]) + + def test_init_rejects_missing_camera_inputs(tmp_path: Path) -> None: app = LingbotVAApplication(_FakeEngineFactory()) @@ -338,7 +364,7 @@ def test_runtime_routes_actions_through_generic_sink(tmp_path: Path) -> None: run_session( session, window, - tensor_artifact_output_sink=TensorArtifactOutputSink(artifact_dir), + model_output_sinks=[TensorArtifactOutputSink(artifact_dir)], ) assert window.closed diff --git a/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_real_model.py b/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_real_model.py index 990581cb9..1a91d0f1b 100644 --- a/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_real_model.py +++ b/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_real_model.py @@ -18,14 +18,16 @@ The production-path test downloads about 23 GiB unless a resolved snapshot is provided and writes a short MP4, metrics JSON, and actions array:: - LINGBOT_VA_REAL_MODEL_RUN=1 uv run --no-sync pytest \ - integrations_v2/lingbot_va -m ci_gpu -s + LINGBOT_VA_REAL_MODEL_RUN=1 \ + LINGBOT_VA_INPUT_DIR=/path/to/robotwin-images \ + uv run --no-sync pytest integrations_v2/lingbot_va -m ci_gpu -s The separate compile gate is intentionally explicit because cold Inductor autotuning can take minutes:: - LINGBOT_VA_REAL_MODEL_COMPILE_RUN=1 uv run --no-sync pytest \ - integrations_v2/lingbot_va -m ci_gpu -s -k compile + LINGBOT_VA_REAL_MODEL_COMPILE_RUN=1 \ + LINGBOT_VA_INPUT_DIR=/path/to/robotwin-images \ + uv run --no-sync pytest integrations_v2/lingbot_va -m ci_gpu -s -k compile """ from __future__ import annotations @@ -56,8 +58,6 @@ pytestmark = pytest.mark.ci_gpu _CHECKPOINT_REVISION = "8c9dea8abbc5c91cc9e18bc3264b8915083bbe70" -_REPOSITORY_ROOT = Path(__file__).resolve().parents[4] -_DEFAULT_INPUT_DIR = _REPOSITORY_ROOT / "assets/example_data/lingbot-va/robotwin" _RUN_SKIP = real_model_run_skip_reason("LINGBOT_VA_REAL_MODEL_RUN") @@ -79,8 +79,14 @@ def _checkpoint_root() -> str: def _input_dir() -> Path: - """Return an optional three-camera input override or checked-in example.""" - return Path(os.environ.get("LINGBOT_VA_INPUT_DIR", _DEFAULT_INPUT_DIR)) + """Return the explicitly supplied three-camera input directory.""" + value = os.environ.get("LINGBOT_VA_INPUT_DIR") + if value is None: + raise RuntimeError( + "Set LINGBOT_VA_INPUT_DIR to a directory containing the three " + "Robotwin camera PNGs." + ) + return Path(value) @pytest.mark.skipif(_RUN_SKIP is not None, reason=_RUN_SKIP or "") @@ -94,7 +100,7 @@ def test_real_model_v2_offload_writes_video_actions_and_metrics( application, Mp4ClientWindow(video_path), metrics_output_sink=MetricsOutputSink(metrics_path), - tensor_artifact_output_sink=TensorArtifactOutputSink(tmp_path), + model_output_sinks=[TensorArtifactOutputSink(tmp_path)], ).run( application.session_desc(), [ From 5b59b64a2602c9df721aaae5b88e64c800bc6bc6 Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Wed, 26 Aug 2026 12:27:59 -0700 Subject: [PATCH 12/18] Fix LingBot cache rollover and teardown Signed-off-by: Jonathan McCaffrey --- integrations/lingbot_va/README.md | 9 +- integrations/lingbot_va/lingbot_va/config.py | 6 +- .../lingbot_va/lingbot_va/constants.py | 1 - integrations/lingbot_va/lingbot_va/engine.py | 53 +++------ .../lingbot_va/lingbot_va/pipeline.py | 2 +- .../lingbot_va/transformer/__init__.py | 10 +- .../lingbot_va/transformer/impl/kvcache.py | 36 ++---- .../lingbot_va/transformer/impl/network.py | 25 +---- integrations/lingbot_va/lingbot_va/utils.py | 6 +- .../lingbot_va/tests/test_cfg_cache.py | 104 +++++++++++++++++- integrations/lingbot_va/tests/test_engine.py | 64 ++++++++++- 11 files changed, 215 insertions(+), 101 deletions(-) diff --git a/integrations/lingbot_va/README.md b/integrations/lingbot_va/README.md index 087fe8145..ce68b5665 100644 --- a/integrations/lingbot_va/README.md +++ b/integrations/lingbot_va/README.md @@ -206,12 +206,13 @@ Set `LINGBOT_VA_CHECKPOINT_ROOT` to reuse a local snapshot. ### Final stacked-PR revalidation -After the PR split, the same pinned checkpoint and input hashes were run through -the final stacked code with two chunks, default CFG, offload, and no compilation: +After the review fixes, the same pinned checkpoint and input hashes were run +through the final stacked code with two chunks, default CFG, offload, and no +compilation: -- GPU test: 1 passed in 38.56 s; +- GPU test: 1 passed in 34.02 s; - MP4: H.264, 320x256, 10 FPS, 13 frames, SHA-256 - `d462cd1f0ef5afe01733afb9dd72aff1ac9365780dc078429223188c15ece4b6`; + `15bcdc4307e080218255e83946c2c2e5dbc30f3b7acd26c8925167017234e586`; - actions: float32 `[64, 16]`, finite, distinct chunks, SHA-256 `463b307b667c1ca13a47bbbc5a17f68604621dfe3c3a10fc5860077216928d95`; - peak allocation: 39,804,415,488 bytes (37.07 GiB). diff --git a/integrations/lingbot_va/lingbot_va/config.py b/integrations/lingbot_va/lingbot_va/config.py index e55576d04..df49fc129 100644 --- a/integrations/lingbot_va/lingbot_va/config.py +++ b/integrations/lingbot_va/lingbot_va/config.py @@ -20,12 +20,14 @@ from flashdreams.infra.diffusion.model import DiffusionModelConfig from lingbot_va.constants import ( DEFAULT_CHECKPOINT_ROOT, + ROBOTWIN_ACTION_GUIDANCE_SCALE, ROBOTWIN_ACTION_INFERENCE_STEPS, ROBOTWIN_ACTION_PER_FRAME, ROBOTWIN_ACTION_SNR_SHIFT, ROBOTWIN_ACTION_TOKEN_PER_CHUNK, ROBOTWIN_ATTENTION_WINDOW, ROBOTWIN_FRAME_CHUNK_SIZE, + ROBOTWIN_GUIDANCE_SCALE, ROBOTWIN_HEIGHT, ROBOTWIN_LATENT_CHANNELS, ROBOTWIN_LATENT_HEIGHT, @@ -60,8 +62,8 @@ seed=42, transformer=LingbotVATransformerConfig( checkpoint_root=DEFAULT_CHECKPOINT_ROOT, - guidance_scale=5.0, - action_guidance_scale=1.0, + guidance_scale=ROBOTWIN_GUIDANCE_SCALE, + action_guidance_scale=ROBOTWIN_ACTION_GUIDANCE_SCALE, latent_height=ROBOTWIN_LATENT_HEIGHT, latent_width=ROBOTWIN_LATENT_WIDTH, frame_chunk_size=ROBOTWIN_FRAME_CHUNK_SIZE, diff --git a/integrations/lingbot_va/lingbot_va/constants.py b/integrations/lingbot_va/lingbot_va/constants.py index cfd077f98..f1092d360 100644 --- a/integrations/lingbot_va/lingbot_va/constants.py +++ b/integrations/lingbot_va/lingbot_va/constants.py @@ -39,7 +39,6 @@ ROBOTWIN_SNR_SHIFT = 5.0 ROBOTWIN_ACTION_SNR_SHIFT = 1.0 ROBOTWIN_PATCH_SIZE = (1, 2, 2) -ROBOTWIN_ENV_TYPE = "robotwin_tshape" ROBOTWIN_OBS_CAM_KEYS = ( "observation.images.cam_high", "observation.images.cam_left_wrist", diff --git a/integrations/lingbot_va/lingbot_va/engine.py b/integrations/lingbot_va/lingbot_va/engine.py index 9f5d72fed..a5f712e8b 100644 --- a/integrations/lingbot_va/lingbot_va/engine.py +++ b/integrations/lingbot_va/lingbot_va/engine.py @@ -20,6 +20,7 @@ import gc import html +import logging import re import time from collections.abc import Mapping @@ -28,6 +29,7 @@ from pathlib import Path from typing import Any +import ftfy import numpy as np import torch import torch.nn.functional as F @@ -63,6 +65,8 @@ from lingbot_va.pipeline import LingbotVAInferencePipelineConfig from lingbot_va.utils import resolve_prompt +logger = logging.getLogger(__name__) + @dataclass(frozen=True, slots=True, kw_only=True) class LingbotVAEngineConfig: @@ -186,12 +190,7 @@ def build_pipeline_config( def _prompt_clean(text: str) -> str: """Apply the upstream double HTML decode and whitespace cleanup.""" - try: - import ftfy - - text = ftfy.fix_text(text) - except ImportError: - pass + text = ftfy.fix_text(text) return re.sub(r"\s+", " ", html.unescape(html.unescape(text))).strip() @@ -237,7 +236,7 @@ def run(self) -> LingbotVAEngineOutput: try: self.close() except Exception: - pass + logger.exception("LingBot-VA cleanup failed after inference failure") raise self._state = LingbotVAEngineState.FINISHED return output @@ -501,20 +500,11 @@ def _generate_chunks( def _release_denoising_state(self) -> None: """Release cache, DiT, text, and streaming encoder state before decode.""" - cache = self._pipeline_cache - if cache is not None: - transformer_cache = getattr(cache, "transformer_cache", None) - if transformer_cache is not None: - for network_cache in ( - getattr(transformer_cache, "network_cache", None), - getattr(transformer_cache, "network_cache_uncond", None), - ): - if network_cache is None: - continue - for block_cache in network_cache.block_caches: - block_cache.self_attn.reset() - block_cache.cross_attn.text.k = torch.empty(0) - block_cache.cross_attn.text.v = torch.empty(0) + # Drop owning references before collecting and trimming CUDA's allocator. + # Moving components to CPU would create a needless host copy, while + # resetting BlockKVCache only resets bookkeeping and keeps its storage + # allocated. + self._pipeline_cache = None for wrapper in (self._streaming_vae, self._streaming_vae_half): @@ -523,15 +513,7 @@ def _release_denoising_state(self) -> None: self._streaming_vae = None self._streaming_vae_half = None - if self._pipeline is not None: - transformer = self._pipeline.transformer - network = getattr(transformer, "_network", None) - if network is not None: - network.to("cpu") - object.__setattr__(transformer, "_network", None) self._pipeline = None - if self._text_encoder is not None: - self._text_encoder.to("cpu") self._text_encoder = None self._tokenizer = None gc.collect() @@ -548,15 +530,12 @@ def _decode_video(self, latents: Tensor, device: torch.device) -> Tensor: device=device, dtype=self.config.dtype, ).view(1, vae.config.z_dim, 1, 1, 1) - latent_inverse_std = ( - 1.0 - / torch.as_tensor( - vae.config.latents_std, - device=device, - dtype=self.config.dtype, - ) + latent_std = torch.as_tensor( + vae.config.latents_std, + device=device, + dtype=self.config.dtype, ).view(1, vae.config.z_dim, 1, 1, 1) - latent = latent / latent_inverse_std + latent_mean + latent = latent * latent_std + latent_mean vae.clear_cache() decoded_input = vae.post_quant_conv(latent) diff --git a/integrations/lingbot_va/lingbot_va/pipeline.py b/integrations/lingbot_va/lingbot_va/pipeline.py index 30624bd35..df6ec7735 100644 --- a/integrations/lingbot_va/lingbot_va/pipeline.py +++ b/integrations/lingbot_va/lingbot_va/pipeline.py @@ -63,7 +63,7 @@ class LingbotVAInferencePipelineConfig(StreamInferencePipelineConfig): frame_chunk_size: int = 2 action_dim: int = 30 action_per_frame: int = 16 - attn_window: int = 64 + attn_window: int = 72 latent_height: int = 24 latent_width: int = 20 latent_channels: int = 48 diff --git a/integrations/lingbot_va/lingbot_va/transformer/__init__.py b/integrations/lingbot_va/lingbot_va/transformer/__init__.py index 53ad2348c..b792171e3 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/__init__.py +++ b/integrations/lingbot_va/lingbot_va/transformer/__init__.py @@ -106,7 +106,7 @@ class LingbotVATransformerConfig(TransformerConfig): latent_width: int = 0 frame_chunk_size: int = 4 action_per_frame: int = 16 - attn_window: int = 64 + attn_window: int = 72 # --------------------------------------------------------------------------- @@ -252,7 +252,9 @@ def predict_flow( assert video_kv_cond is not None cache.video_kv_cond = video_kv_cond - if cache.network_cache_uncond is not None: + if cache.network_cache_uncond is not None and ( + persist or self.config.guidance_scale > 1.0 + ): flow_uncond, video_kv_uncond = self.network.forward_video( noisy_latent, timestep, @@ -297,7 +299,9 @@ def predict_action_flow( if persist: cache.video_kv_cond = None - if cache.network_cache_uncond is not None: + if cache.network_cache_uncond is not None and ( + persist or self.config.action_guidance_scale > 1.0 + ): flow_uncond = self.network.forward_action( noisy_action, timestep, diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/kvcache.py b/integrations/lingbot_va/lingbot_va/transformer/impl/kvcache.py index d41abc714..abe357d40 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/impl/kvcache.py +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/kvcache.py @@ -16,8 +16,8 @@ """VAKVCache — rolling-window KV cache for video-action transformers. Wraps a ``BlockKVCache`` with a compile-friendly read path: intermediate -denoising steps read committed cache + concat fresh tokens (no writes), -while the final step writes the full [video|action] chunk via BlockKVCache.update(). +denoising steps read committed cache tensors (no writes), while the final step +writes the full [video|action] chunk via ``BlockKVCache.update()``. """ from __future__ import annotations @@ -36,7 +36,7 @@ class VAKVCache: Lifecycle per AR step: cache.before_update(chunk_idx) - ... intermediate denoising: read via committed_kv_plus_fresh (no cache mutation) + ... intermediate denoising: read via committed_kv (no cache mutation) ... final video step: write_video(k, v) ... final action step: write_action(k, v) → commits full chunk cache.after_update(chunk_idx) @@ -84,8 +84,8 @@ def create( @property def n_committed_tokens(self) -> int: - """Number of committed tokens from prior AR steps.""" - return self.kv_cache._n_cached + """Number of valid prior-step tokens after opening the update window.""" + return self.kv_cache.write_end - self.kv_cache.chunk_size def before_update(self, chunk_idx: int) -> None: """Open the update window for a new AR step.""" @@ -95,28 +95,16 @@ def after_update(self, chunk_idx: int) -> None: """Close the update window and commit.""" self.kv_cache.after_update(chunk_idx) - def committed_kv_plus_fresh( - self, k_fresh: Tensor, v_fresh: Tensor - ) -> tuple[Tensor, Tensor]: - """Read-only: committed prior tokens + fresh current tokens. + def committed_kv(self) -> tuple[Tensor, Tensor]: + """Return the valid prior-step KV prefix after any window roll. - Used during intermediate denoising steps. Does NOT mutate the cache. - This path is compile-friendly (pure tensor ops, no side effects). - - Args: - k_fresh: Shape ``[batch, L_fresh, heads, head_dim]``. - v_fresh: Same shape as ``k_fresh``. - - Returns: - ``(full_k, full_v)`` for attention context. + ``BlockKVCache.before_update`` rolls a full cache before the current + chunk is written. Its trailing chunk-sized region is therefore stale + until ``write_video``/``write_action`` replace it and must not be + exposed as committed context. """ n = self.n_committed_tokens - committed_k = self.kv_cache._k[:, :n] - committed_v = self.kv_cache._v[:, :n] - return ( - torch.cat([committed_k, k_fresh], dim=1), - torch.cat([committed_v, v_fresh], dim=1), - ) + return self.kv_cache.cached_k()[:, :n], self.kv_cache.cached_v()[:, :n] def write_video(self, k: Tensor, v: Tensor) -> None: """Write video KV to the current chunk (pads action with zeros). diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/network.py b/integrations/lingbot_va/lingbot_va/transformer/impl/network.py index 2825bdf20..156a060aa 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/impl/network.py +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/network.py @@ -295,29 +295,14 @@ def _extract_cache_tensors( (committed_k_stack, committed_v_stack, cross_k_stack, cross_v_stack) All shapes: [num_layers, batch, seq_len, heads, head_dim] """ - committed_k = torch.stack( - [ - bc.self_attn.kv_cache._k[:, : bc.self_attn.n_committed_tokens] - for bc in cache.block_caches - ] - ) - committed_v = torch.stack( - [ - bc.self_attn.kv_cache._v[:, : bc.self_attn.n_committed_tokens] - for bc in cache.block_caches - ] - ) + committed = [bc.self_attn.committed_kv() for bc in cache.block_caches] + committed_k = torch.stack([key for key, _ in committed]) + committed_v = torch.stack([value for _, value in committed]) cross_k = torch.stack( - [ - bc.cross_attn.text._k[:, : bc.cross_attn.text._n_cached] - for bc in cache.block_caches - ] + [bc.cross_attn.text.cached_k() for bc in cache.block_caches] ) cross_v = torch.stack( - [ - bc.cross_attn.text._v[:, : bc.cross_attn.text._n_cached] - for bc in cache.block_caches - ] + [bc.cross_attn.text.cached_v() for bc in cache.block_caches] ) return committed_k, committed_v, cross_k, cross_v diff --git a/integrations/lingbot_va/lingbot_va/utils.py b/integrations/lingbot_va/lingbot_va/utils.py index 3d0385311..ccd4796f7 100644 --- a/integrations/lingbot_va/lingbot_va/utils.py +++ b/integrations/lingbot_va/lingbot_va/utils.py @@ -101,7 +101,9 @@ def resolve_prompt(value: str | Path) -> str: lines = [ line.strip() for line in value.read_text().splitlines() if line.strip() ] - assert lines, f"prompt file {value} has no non-empty lines" + if not lines: + raise ValueError(f"prompt file {value} has no non-empty lines") return lines[0] - assert value, "prompt must be a non-empty string or a path" + if not value: + raise ValueError("prompt must be a non-empty string or a path") return value diff --git a/integrations/lingbot_va/tests/test_cfg_cache.py b/integrations/lingbot_va/tests/test_cfg_cache.py index c6f19dfd6..70386781c 100644 --- a/integrations/lingbot_va/tests/test_cfg_cache.py +++ b/integrations/lingbot_va/tests/test_cfg_cache.py @@ -52,6 +52,8 @@ def __init__( self._cond_cache = cond_cache self._uncond_cache = uncond_cache self.action_video_kv: dict[str, VideoKV | None] = {} + self.video_branches: list[str] = [] + self.action_branches: list[str] = [] def _branch(self, cache: WanVADiTNetworkCache) -> tuple[str, float]: if cache is self._cond_cache: @@ -68,7 +70,8 @@ def forward_video( persist: bool = False, ) -> tuple[Tensor, VideoKV | None]: del timesteps, rope_freqs - _, value = self._branch(cache) + branch, value = self._branch(cache) + self.video_branches.append(branch) video_kv = ((torch.tensor([value]), torch.tensor([value * 10])),) return torch.full_like(x, value), video_kv if persist else None @@ -83,6 +86,7 @@ def forward_action( ) -> Tensor: del timesteps, rope_freqs branch, value = self._branch(cache) + self.action_branches.append(branch) if persist: self.action_video_kv[branch] = video_kv return torch.full_like(x, value * 3) @@ -133,6 +137,45 @@ def test_cfg_action_branches_consume_their_matching_video_kv() -> None: assert cache.video_kv_uncond is None +def test_inactive_action_cfg_branch_only_runs_when_committing_cache() -> None: + cond_cache = WanVADiTNetworkCache(block_caches=[]) + uncond_cache = WanVADiTNetworkCache(block_caches=[]) + config = LingbotVATransformerConfig( + network=WanVADiTNetworkConfig(dim=12, num_heads=1, num_layers=1), + guidance_scale=5.0, + action_guidance_scale=1.0, + compile_network=False, + ) + transformer = LingbotVATransformer(config) + network = _BranchRecordingNetwork(cond_cache, uncond_cache) + object.__setattr__(transformer, "_network", network) + cache = LingbotVATransformerCache( + network_cache=cond_cache, + network_cache_uncond=uncond_cache, + ) + noisy = torch.zeros(1, 1, 1) + timestep = torch.zeros(1, 1) + model_input: dict[str, Any] = {"grid_id": torch.zeros(3, 1)} + + transformer.predict_flow(noisy, timestep, cache, input=model_input) + transformer.predict_action_flow(noisy, timestep, cache, input=model_input) + + assert network.video_branches == ["cond", "uncond"] + assert network.action_branches == ["cond"] + + transformer.predict_flow(noisy, timestep, cache, input=model_input, persist=True) + transformer.predict_action_flow( + noisy, + timestep, + cache, + input=model_input, + persist=True, + ) + + assert network.video_branches == ["cond", "uncond", "cond", "uncond"] + assert network.action_branches == ["cond", "cond", "uncond"] + + def test_action_block_loop_attends_to_committed_then_current_video_kv() -> None: config = WanVADiTNetworkConfig( dim=12, @@ -150,12 +193,22 @@ def test_action_block_loop_attends_to_committed_then_current_video_kv() -> None: current_video_v = torch.tensor([[[[4.0] * 12]]]) text_k = torch.zeros(1, 1, 1, 12) text_v = torch.zeros_like(text_k) + self_attn = VAKVCache.create( + video_chunk=1, + action_chunk=0, + window_slots=2, + batch_size=1, + num_heads=1, + head_dim=12, + device="cpu", + dtype=torch.float32, + ) + self_attn.before_update(0) + self_attn.write_video(prior_k, prior_v) + self_attn.after_update(0) + self_attn.before_update(1) block_cache = VABlockCache( - self_attn=VAKVCache( - kv_cache=BlockKVCache.from_tensor(prior_k, prior_v, seq_dim=1), - video_chunk=1, - action_chunk=1, - ), + self_attn=self_attn, cross_attn=CrossAttnCache( text=BlockKVCache.from_tensor(text_k, text_v, seq_dim=1) ), @@ -217,3 +270,42 @@ def test_action_block_loop_requires_current_video_kv() -> None: WanVADiTNetworkCache(block_caches=[]), torch.zeros(1, 1, 1, 12), ) + + +def test_rolling_window_excludes_stale_trailing_chunk() -> None: + cache = VAKVCache.create( + video_chunk=2, + action_chunk=1, + window_slots=3, + batch_size=1, + num_heads=1, + head_dim=1, + device="cpu", + dtype=torch.float32, + ) + + for chunk_idx in range(3): + value = float(chunk_idx + 1) + video_k = torch.full((1, 2, 1, 1), value) + video_v = torch.full((1, 2, 1, 1), value + 10) + action_k = torch.full((1, 1, 1, 1), value) + action_v = torch.full((1, 1, 1, 1), value + 10) + cache.before_update(chunk_idx) + cache.write_video(video_k, video_v) + cache.write_action( + action_k, + action_v, + video_k, + video_v, + ) + cache.after_update(chunk_idx) + + cache.before_update(3) + committed_k, committed_v = cache.committed_kv() + + expected_k = torch.tensor([2.0, 2.0, 2.0, 3.0, 3.0, 3.0]).view(1, 6, 1, 1) + expected_v = expected_k + 10 + torch.testing.assert_close(committed_k, expected_k) + torch.testing.assert_close(committed_v, expected_v) + assert cache.n_committed_tokens == 6 + assert committed_k.shape == (1, 6, 1, 1) diff --git a/integrations/lingbot_va/tests/test_engine.py b/integrations/lingbot_va/tests/test_engine.py index 11c5fc9dd..9a96c48ac 100644 --- a/integrations/lingbot_va/tests/test_engine.py +++ b/integrations/lingbot_va/tests/test_engine.py @@ -15,6 +15,7 @@ """CPU tests for LingBot engine configuration and one-run lifecycle.""" +import weakref from collections.abc import Callable from pathlib import Path from typing import Any @@ -113,13 +114,74 @@ def test_engine_is_one_run_and_close_is_idempotent() -> None: assert engine.state is LingbotVAEngineState.CLOSED -def test_engine_failure_closes_partial_state() -> None: +def test_engine_failure_closes_partial_state(caplog: pytest.LogCaptureFixture) -> None: engine = _FailingEngine(_config()) with pytest.raises(RuntimeError, match="inference failed"): engine.run() assert engine.state is LingbotVAEngineState.CLOSED + assert "cleanup failed after inference failure" in caplog.text + + +def test_denoising_owners_are_released_before_cuda_allocator_trim() -> None: + class _Tracked: + _network: Any | None = None + transformer: Any | None = None + + def to(self, *_args: Any, **_kwargs: Any) -> None: + raise AssertionError("teardown must not copy model components to CPU") + + class _ObservingEngine(LingbotVAEngine): + def __init__(self) -> None: + super().__init__(_config()) + self.released_refs: list[weakref.ReferenceType[Any]] = [] + self.trimmed = False + + def _empty_cuda_cache(self) -> None: + assert all(reference() is None for reference in self.released_refs) + self.trimmed = True + + class _Wrapper: + def __init__(self) -> None: + self.cleared = False + + def clear_cache(self) -> None: + self.cleared = True + + engine = _ObservingEngine() + pipeline_cache = _Tracked() + network = _Tracked() + transformer = _Tracked() + transformer._network = network + pipeline = _Tracked() + pipeline.transformer = transformer + text_encoder = _Tracked() + tokenizer = _Tracked() + wrapper = _Wrapper() + wrapper_half = _Wrapper() + + engine._pipeline_cache = pipeline_cache + engine._pipeline = pipeline + engine._text_encoder = text_encoder + engine._tokenizer = tokenizer + engine._streaming_vae = wrapper # type: ignore[assignment] # ty: ignore[invalid-assignment] + engine._streaming_vae_half = wrapper_half # type: ignore[assignment] # ty: ignore[invalid-assignment] + engine.released_refs = [ + weakref.ref(pipeline_cache), + weakref.ref(network), + weakref.ref(transformer), + weakref.ref(pipeline), + weakref.ref(text_encoder), + weakref.ref(tokenizer), + ] + del pipeline_cache, network, transformer, pipeline, text_encoder, tokenizer + + engine._release_denoising_state() + + assert engine.trimmed + assert wrapper.cleared + assert wrapper_half.cleared def test_validate_input_images_names_all_missing_cameras(tmp_path: Path) -> None: From d255c11ed67de11702fcf4a8514e1ec020cc635d Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Wed, 26 Aug 2026 12:29:42 -0700 Subject: [PATCH 13/18] Refresh LingBot review validation evidence Signed-off-by: Jonathan McCaffrey --- integrations/lingbot_va/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integrations/lingbot_va/README.md b/integrations/lingbot_va/README.md index ce68b5665..84b115975 100644 --- a/integrations/lingbot_va/README.md +++ b/integrations/lingbot_va/README.md @@ -215,7 +215,7 @@ compilation: `15bcdc4307e080218255e83946c2c2e5dbc30f3b7acd26c8925167017234e586`; - actions: float32 `[64, 16]`, finite, distinct chunks, SHA-256 `463b307b667c1ca13a47bbbc5a17f68604621dfe3c3a10fc5860077216928d95`; -- peak allocation: 39,804,415,488 bytes (37.07 GiB). +- peak allocation: 39,804,413,440 bytes (37.07 GiB). The action hash exactly matches the earlier resident/offload parity run. From cb3e1bfa962cc4f403ed3b6a724a88631d298b16 Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Thu, 27 Aug 2026 00:02:11 -0700 Subject: [PATCH 14/18] Document LingBot architecture and model card Signed-off-by: Jonathan McCaffrey --- integrations/lingbot_va/README.md | 346 +++++++++++++++++++++++++-- integrations_v2/README.md | 4 +- integrations_v2/lingbot_va/README.md | 8 +- 3 files changed, 338 insertions(+), 20 deletions(-) diff --git a/integrations/lingbot_va/README.md b/integrations/lingbot_va/README.md index 84b115975..5de615408 100644 --- a/integrations/lingbot_va/README.md +++ b/integrations/lingbot_va/README.md @@ -16,6 +16,53 @@ propagation, lifecycle, and output contracts were corrected for the V2 API. The draft performance claims are intentionally omitted; only matched measurements from the checked-in implementation are recorded below. +## Model card + +### Identity and supported task + +| Field | Integrated behavior | +| --- | --- | +| Model family | [LingBot-VA](https://arxiv.org/abs/2601.21998), an autoregressive diffusion video-action world-model policy with a shared video/action backbone | +| Checkpoint | [`robbyant/lingbot-va-posttrain-robotwin`](https://huggingface.co/robbyant/lingbot-va-posttrain-robotwin), pinned to `8c9dea8abbc5c91cc9e18bc3264b8915083bbe70` | +| Implemented task | Offline, batch-one Robotwin image-and-instruction to predicted video-and-action rollout (I2AV) | +| Input modalities | One natural-language prompt and three RGB camera PNGs: high, left wrist, and right wrist | +| Output modalities | Predicted high-camera video, 16 denormalized Robotwin action channels, and inference metrics | +| Core models | UMT5-XXL text encoder (4,096-wide states), Wan VAE (48 latent channels, 16x spatial and 4x temporal scaling), and the shared video/action DiT | +| DiT architecture | 5,088,872,670 parameters; 30 blocks; width 3,072; FFN width 14,336; 24 heads of width 128; `[1, 2, 2]` video patches | +| Chunk and cache geometry | Two latent frames per chunk; 240 video plus 32 action tokens; 36 rolling chunk slots per conditional/unconditional branch | +| Checkpoint footprint | About 22.7 GiB of resolved weight files (9.48 GiB transformer, 10.58 GiB text encoder, 2.63 GiB VAE), measured from the pinned snapshot | +| Runtime profile | BF16 on one CUDA device; optional `torch.compile` and component offload; one complete rollout per engine | +| License | This package is Apache-2.0. The [upstream repository](https://github.com/Robbyant/lingbot-va) and published checkpoint model card also identify Apache-2.0; users remain responsible for checkpoint and dataset terms. | + +### Provenance, data, and evaluation boundary + +- The architecture and inference behavior are ported from + [`Robbyant/lingbot-va@7c6ffa9`](https://github.com/Robbyant/lingbot-va/tree/7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb). +- The upstream project identifies + [`robotwin-clean-and-aug-lerobot`](https://huggingface.co/datasets/robbyant/robotwin-clean-and-aug-lerobot) + as its cleaned and augmented RoboTwin post-training dataset. FlashDreams does + not train, alter, or independently audit the checkpoint or dataset. +- The upstream [paper](https://arxiv.org/abs/2601.21998) reports simulated and + real-robot results. This integration has not reproduced task success rates; + the evidence below establishes native-flow parity, output contracts, and + system execution on the pinned checkpoint. + +### Intended use and safety boundary + +| Intended and validated | Outside this integration's validated scope | +| --- | --- | +| Research and development of offline Robotwin video/action rollout inference | Direct, unattended, or safety-critical robot actuation | +| Numerical parity, lifecycle, packaging, and V2 runtime regression testing | Claims of physical accuracy, task success, or safe action execution | +| Generating inspectable MP4, NumPy action, and JSON metric artifacts | Training, fine-tuning, LIBERO checkpoints, multi-GPU/FSDP, or online serving | + +Predicted actions can be wrong, temporally inconsistent, or unsafe. This port +starts from one fixed camera observation and does not implement the upstream +closed-loop observation feedback or asynchronous motor-execution system. +Distribution shift, camera calibration, prompt ambiguity, and accumulated +autoregressive error can affect both modalities. Evaluate in simulation or a +hardware-interlocked environment with task-specific limits and human oversight +before considering any physical use. + ## Install and run From the repository root: @@ -94,7 +141,260 @@ single model-result channel. Backpressure and presentation policies selected by the runtime are preserved; the fixed video layout, dimensions, rate, and artifact schema are validated. -## Design and lifecycle +## Architecture design review + +The architecture keeps model-specific numerical code in +[`integrations/lingbot_va`](.) and the FlashDreams V2 protocol adapter in +[`integrations_v2/lingbot_va`](../../integrations_v2/lingbot_va). Generic +runtime code has no LingBot-specific branches. + +### Goals and key decisions + +| Decision | Rationale and consequence | +| --- | --- | +| Separate model and V2 adapter packages | The model package owns checkpoint/model/tensor behavior; `app.py` owns only V2 contracts and lifecycle adaptation. | +| Declare the natural session before model initialization | CLI/runtime compatibility and output schemas fail fast without loading approximately 23 GiB of checkpoint data. | +| Session-owned, lazily created engine | Mutable CUDA state is isolated to the model thread and reset creates a fresh one-run engine. | +| One complete rollout per `StepResult` | Decode requires destructive DiT/KV teardown, so per-chunk presentation would claim a streaming capability the engine does not provide. | +| Separate conditional/unconditional caches | CFG branches never contaminate one another; inactive stream CFG is skipped except on the terminal cache-commit pass. | +| Plain tensors across the compiled block boundary | Cache extraction and writes remain eager while the 30-block video/action loops can be compiled without cache-object graph breaks. | +| Generic runtime sinks | The adapter returns TCHW video, metrics, and a typed `actions` artifact; MP4/JSON/NumPy serialization stays reusable. | + +### Static view: packages and components + +```mermaid +flowchart LR + subgraph Runtime["FlashDreams V2 runtime"] + CLI["flashdreams-run-v2"] + Runner["ApplicationRunner"] + Sinks["Generic sinks
MP4, metrics JSON, actions NPY"] + end + + subgraph Adapter["integrations_v2/lingbot_va/lingbot_va_v2/app.py"] + App["LingbotVAApplication
IApplication"] + Session["LingbotVASession
ISession"] + Loop["LingbotVAModelLoop
IModelLoop"] + end + + subgraph Model["integrations/lingbot_va/lingbot_va"] + Engine["engine.py
LingbotVAEngine"] + Pipeline["pipeline.py
LingbotVAInferencePipeline"] + Transformer["transformer/__init__.py
LingbotVATransformer"] + Network["transformer/impl/network.py
WanVADiTNetwork"] + Cache["transformer/impl/kvcache.py
VAKVCache"] + Support["_loaders.py, action.py, scheduler.py"] + end + + Snapshot["Pinned local or Hugging Face snapshot"] + Inputs["Prompt and three camera PNGs"] + Core["FlashDreams core/infra
BlockKVCache, pipeline and transformer bases"] + + CLI --> Runner + Runner -->|IApplication| App + App -->|creates| Session + Session -->|registers| Loop + Loop -->|LingbotVAEngineLike| Engine + Inputs --> Engine + Snapshot --> Support + Support --> Engine + Engine -->|setup and generate| Pipeline + Pipeline -->|predict video and action flow| Transformer + Transformer --> Network + Network --> Cache + Core --> Pipeline + Core --> Transformer + Core --> Cache + Engine -->|LingbotVAEngineOutput| Loop + Loop -->|StepResult| Runner + Runner --> Sinks +``` + +The only adapter-to-model interface needed by CPU stand-ins is +`LingbotVAEngineLike.run() -> LingbotVAEngineOutput` plus `close()`. The +production engine is therefore replaceable in lifecycle and contract tests +without importing or constructing the checkpoint. + +### Static view: class and interface ownership + +```mermaid +classDiagram + direction LR + + class IApplication { + <> + +session_desc() + +init(args) + +create_session(desc) + } + class ISession { + <> + +init() + +session_desc + +close() + } + class IModelLoop { + <> + +step(index, events) + +is_finished() + +reset() + +close() + } + class LingbotVAEngineLike { + <> + +run() + +close() + } + class LingbotVAApplication + class LingbotVASession + class LingbotVAModelLoop + class LingbotVAModelState + class LingbotVAEngine + class LingbotVAInferencePipeline + class LingbotVATransformer + class LingbotVATransformerCache + class WanVADiTNetwork + class VABlock + class VAKVCache + class BlockKVCache + + IApplication <|.. LingbotVAApplication + ISession <|.. LingbotVASession + IModelLoop <|.. LingbotVAModelLoop + LingbotVAEngineLike <|.. LingbotVAEngine + LingbotVAApplication --> LingbotVASession : creates + LingbotVASession *-- LingbotVAModelLoop : registers + LingbotVAModelLoop *-- LingbotVAModelState : owns + LingbotVAModelState o-- LingbotVAEngineLike : lazy engine + LingbotVAEngine *-- LingbotVAInferencePipeline + LingbotVAInferencePipeline *-- LingbotVATransformer + LingbotVATransformer *-- LingbotVATransformerCache + LingbotVATransformer *-- WanVADiTNetwork + WanVADiTNetwork *-- VABlock : 30 blocks + VABlock *-- VAKVCache : per-block self-attention + VAKVCache *-- BlockKVCache : rolling storage +``` + +### Use cases, modalities, and functionality + +```mermaid +flowchart LR + User(["Researcher or integration evaluator"]) + + subgraph Inputs["Input modalities"] + Prompt["Natural-language prompt"] + Cameras["RGB high and two wrist views"] + Weights["Pinned Robotwin checkpoint"] + end + + subgraph Capabilities["Integrated functionality"] + Validate["Validate device, files, session and checkpoint"] + Text["UMT5 text conditioning"] + Vision["Wan VAE T-layout observation encoding"] + Joint["Autoregressive video/action denoising
shared DiT and rolling KV"] + Decode["Deferred video decode and high-camera crop"] + Actions["Quantile action denormalization and channel selection"] + Export["V2 result validation and generic artifact export"] + end + + subgraph Outputs["Output modalities"] + Video["Predicted 320x256 video at 10 FPS"] + ActionTensor["Float32 actions: 32N x 16"] + Metrics["Phase timing and peak CUDA allocation"] + end + + Live["Live feedback, policy serving, and robot actuation
not implemented"]:::outside + + User --> Validate + Prompt --> Text + Cameras --> Vision + Weights --> Validate + Validate --> Text + Validate --> Vision + Text --> Joint + Vision --> Joint + Joint --> Decode + Joint --> Actions + Decode --> Export + Actions --> Export + Export --> Video + Export --> ActionTensor + Export --> Metrics + User -. outside validated scope .-> Live + + classDef outside fill:#f5f5f5,stroke:#888,stroke-dasharray:5 5,color:#555 +``` + +### Dynamic view: one V2 rollout + +```mermaid +sequenceDiagram + autonumber + actor User + participant R as ApplicationRunner + participant A as LingbotVAApplication + participant S as LingbotVASession / ModelLoop + participant E as LingbotVAEngine + participant P as LingbotVAInferencePipeline + participant T as LingbotVATransformer / DiT + participant V as UMT5 / Wan VAE + participant O as Generic output sinks + + User->>R: run slug, runtime flags, and model flags + R->>A: session_desc() then init(model args) + A->>A: validate fixed contract, device, inputs, prompt, checkpoint reference + R->>A: create_session(requested SessionDesc) + A-->>R: session preserving runtime backpressure/presentation + R->>S: init() and step(0) + S->>E: lazily construct engine and run() + E->>V: load tokenizer, UMT5, and Wan VAE; encode prompt/cameras + E->>P: initialize conditional and optional unconditional caches + + loop chunk 0 through N-1 + E->>P: generate(chunk, observation latent, action mask) + P->>T: cache.start(chunk) + loop video schedule: 25 updates plus terminal persist + P->>T: predict_flow(noisy video, persist=last) + T->>T: conditional DiT forward + opt video guidance > 1 or terminal persist + T->>T: unconditional DiT forward + end + end + loop action schedule: 50 updates plus terminal persist + P->>T: predict_action_flow(noisy action, current video KV, persist=last) + T->>T: conditional DiT forward + opt action guidance > 1 or terminal persist + T->>T: unconditional DiT forward + end + end + P->>T: commit combined video/action KV and finalize chunk + P-->>E: two latent frames and 32 action steps + E->>E: move completed chunk outputs to CPU + end + + E->>E: drop caches, DiT, text owners; collect; empty CUDA cache + E->>V: decode all latent frames, crop high camera, release VAE + E-->>S: video, denormalized actions, metrics + S-->>R: one validated StepResult + R->>O: route video, metrics, and typed actions artifact + R->>S: close() + S->>E: idempotent close() +``` + +### Data and tensor contracts + +Let `N` be `--num-chunks` (positive, default 10). Batch size is fixed at one. + +| Boundary | Shape and semantics | +| --- | --- | +| Prompt encoding | token ids `[1, 512]` to padded UMT5 states `[1, 512, 4096]`; an empty negative prompt is encoded when either CFG scale is active | +| Observation encoding | high view 256x320 plus two 128x160 wrist views arranged as a T; normalized latent `[1, 48, 1, 24, 20]` | +| Video working chunk | BF16 latent `[1, 48, 2, 24, 20]`; `[1, 2, 2]` patching produces 240 video tokens | +| Action working chunk | BF16 `[1, 30, 2, 16, 1]`; 32 action tokens; unused model channels are masked | +| Self-attention cache | per block and CFG branch, K/V up to `[1, 9792, 24, 128]` = 36 slots x (240 video + 32 action tokens) | +| Engine aggregate | latent `[1, 48, 2N, 24, 20]`; selected denormalized actions `[32N, 16]` on CPU | +| V2 result | floating TCHW video `[8N - 3, 3, 256, 320]` in `[-1, 1]`, float32 `actions[32N, 16]`, numeric phase metrics | + +### Lifecycle and memory phases Each session owns one `LingbotVAEngine`; the application owns only immutable configuration and an engine factory. The successful engine state machine is @@ -118,11 +418,19 @@ earlier inference failure. | decode | VAE plus one decoded frame | output frames/actions | VAE cache and GPU frames | | finished | none | returned video/actions/metrics | all model components | -The default CFG scales are video 5 and action 1. Conditional and unconditional -branches own distinct video KV and both advance whenever CFG is active. Each -action denoise pass attends to committed prior chunks plus its matching branch's -current video KV before adding fresh action KV. The cache attention window is 72, -matching the pinned upstream Robotwin configuration. +### Cache and ownership invariants + +- The default scales are video 5 and action 1. Video evaluates both CFG + branches; action skips its unconditional branch on intermediate denoise steps + but runs both branches on the terminal persist pass so their caches advance. +- Conditional and unconditional branches own distinct self-attention, text, and + current-video KV. Action attention receives committed history followed by the + matching branch's current-video KV and then its fresh action tokens. +- `cache.start()` rolls before denoising and excludes the stale trailing write + region. Terminal video writes provisional KV, terminal action overwrites the + full video/action slot, and `finalize()` advances bookkeeping exactly once. +- The upstream attention setting of 72 frames becomes 36 two-frame chunk slots, + or 9,792 tokens per block and branch. Deliberate exclusions are the V1 runner, application-owned output files, application-owned model components, multi-GPU/FSDP claims, live RoboTwin @@ -130,10 +438,12 @@ control, unmeasured speedup claims, and root CUDA/Torch policy changes. ## Validation evidence -The evidence below was produced on 2026-08-25 using an NVIDIA RTX PRO 6000 -Blackwell Workstation Edition (97,887 MiB), driver 595.84, PyTorch 2.12.1+cu130, -CUDA 13.0, BF16, one CUDA device, and seed 42. It is validation evidence, not a -general performance claim. +Baseline parity and matched resident/offload evidence were produced on +2026-08-25; final stacked-PR revalidation was produced on 2026-08-26. All runs +used an NVIDIA RTX PRO 6000 Blackwell Workstation Edition (97,887 MiB), driver +595.84, PyTorch 2.12.1+cu130, CUDA 13.0, BF16, one CUDA device, and seed 42. +This is implementation evidence, not a general model-performance or robot-task +success claim. The official checkpoint revision `8c9dea8abbc5c91cc9e18bc3264b8915083bbe70` contained 841 transformer entries. @@ -225,9 +535,13 @@ The current implementation supports one GPU, one complete deferred-decode rollout, and Diffusers 0.38 private VAE streaming state. It makes no FSDP, context-parallel, or per-chunk presentation claim. -- Source architecture/inference: - `robbyant/lingbot-va@7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb`. -- Official checkpoint: - `robbyant/lingbot-va-posttrain-robotwin@8c9dea8abbc5c91cc9e18bc3264b8915083bbe70`. -- Initial draft source: - FlashDreams PR #312 at `f98cae4a18ddf6c189a6cfa2099265d6d570e337`. +- Architecture and inference source: + [`Robbyant/lingbot-va@7c6ffa9`](https://github.com/Robbyant/lingbot-va/tree/7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb). +- Model description and evaluation source: + [LingBot-VA paper](https://arxiv.org/abs/2601.21998). +- Checkpoint: + [`robbyant/lingbot-va-posttrain-robotwin@8c9dea8`](https://huggingface.co/robbyant/lingbot-va-posttrain-robotwin/tree/8c9dea8abbc5c91cc9e18bc3264b8915083bbe70). +- Post-training data named by upstream: + [`robotwin-clean-and-aug-lerobot`](https://huggingface.co/datasets/robbyant/robotwin-clean-and-aug-lerobot). +- Initial FlashDreams draft source: PR #312 at + `f98cae4a18ddf6c189a6cfa2099265d6d570e337`. diff --git a/integrations_v2/README.md b/integrations_v2/README.md index e0a74a3bc..20116707f 100644 --- a/integrations_v2/README.md +++ b/integrations_v2/README.md @@ -23,7 +23,9 @@ follows is already done for you. - `t2v_self_forcing`, `t2v_causal_forcing`, `t2v_fastvideo_causal_wan22`, `t2v_wan21`, `t2v_cosmos_predict2` — real models, each a thin wrapper over `flashdreams.t2v_v2`. -- `lingbot_va` — the LingBot-VA Robotwin image-to-action/video model. +- [`lingbot_va`](lingbot_va/README.md) — the LingBot-VA Robotwin + image-to-action/video model; its linked model card and architecture review + document the atypical finite, deferred-decode V2 boundary. - `null_model` — not an application. A v1 pipeline the framework tests use as a fixture. diff --git a/integrations_v2/lingbot_va/README.md b/integrations_v2/lingbot_va/README.md index 50933ca1e..9a5fd22a5 100644 --- a/integrations_v2/lingbot_va/README.md +++ b/integrations_v2/lingbot_va/README.md @@ -33,6 +33,8 @@ rollout before reporting finished. `--input-image-dir` is required because no camera images are bundled. Use `-- --help` after the application slug for checkpoint, input, compilation, offload, seed, guidance, inference-step, and scheduler-shift overrides. The -[model package README](../../integrations/lingbot_va/README.md) documents input -provenance, checkpoint modes, camera/action contracts, lifecycle, opt-in GPU -tests, measured parity evidence, and limitations. +[model card](../../integrations/lingbot_va/README.md#model-card) records +provenance, intended use, safety boundaries, tensor geometry, and measured +evidence. The accompanying +[architecture design review](../../integrations/lingbot_va/README.md#architecture-design-review) +documents package and class ownership, modalities, runtime sequencing, cache From cf669506c21f44fe0b822304b0319fc46713860d Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Thu, 27 Aug 2026 00:14:33 -0700 Subject: [PATCH 15/18] Clarify LingBot cache and text invariants Signed-off-by: Jonathan McCaffrey --- integrations/lingbot_va/README.md | 6 ++- .../lingbot_va/transformer/__init__.py | 6 ++- .../lingbot_va/transformer/impl/network.py | 4 ++ .../lingbot_va/tests/test_cfg_cache.py | 37 +++++++++++++++++++ 4 files changed, 50 insertions(+), 3 deletions(-) diff --git a/integrations/lingbot_va/README.md b/integrations/lingbot_va/README.md index 5de615408..c0668265b 100644 --- a/integrations/lingbot_va/README.md +++ b/integrations/lingbot_va/README.md @@ -429,8 +429,10 @@ earlier inference failure. - `cache.start()` rolls before denoising and excludes the stale trailing write region. Terminal video writes provisional KV, terminal action overwrites the full video/action slot, and `finalize()` advances bookkeeping exactly once. -- The upstream attention setting of 72 frames becomes 36 two-frame chunk slots, - or 9,792 tokens per block and branch. +- The upstream Robotwin setting `attn_window=72` allocates 36 autoregressive + slots, or 9,792 tokens per block and branch. Each slot stores one video chunk + and one action chunk; the divisor reflects those paired modalities, not the + two latent frames in a chunk. Deliberate exclusions are the V1 runner, application-owned output files, application-owned model components, multi-GPU/FSDP claims, live RoboTwin diff --git a/integrations/lingbot_va/lingbot_va/transformer/__init__.py b/integrations/lingbot_va/lingbot_va/transformer/__init__.py index b792171e3..738bc5a52 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/__init__.py +++ b/integrations/lingbot_va/lingbot_va/transformer/__init__.py @@ -104,7 +104,7 @@ class LingbotVATransformerConfig(TransformerConfig): # Spatial layout latent_height: int = 0 latent_width: int = 0 - frame_chunk_size: int = 4 + frame_chunk_size: int = 2 action_per_frame: int = 16 attn_window: int = 72 @@ -195,6 +195,10 @@ def initialize_autoregressive_cache( * (cfg.latent_width // ps[2]) ) action_chunk = cfg.frame_chunk_size * cfg.action_per_frame + # Preserve upstream ``create_empty_cache`` semantics: ``attn_window`` + # counts alternating video/action regions, so two regions make one AR + # cache slot. The divisor is intentionally independent of chunk size; + # upstream applies it to both two- and four-frame configurations. window_slots = cfg.attn_window // 2 network_cache = self.network.initialize_cache( diff --git a/integrations/lingbot_va/lingbot_va/transformer/impl/network.py b/integrations/lingbot_va/lingbot_va/transformer/impl/network.py index 156a060aa..e27420d42 100644 --- a/integrations/lingbot_va/lingbot_va/transformer/impl/network.py +++ b/integrations/lingbot_va/lingbot_va/transformer/impl/network.py @@ -188,6 +188,10 @@ def __init__(self, config: WanVADiTNetworkConfig) -> None: nn.SiLU(), nn.Linear(self.dim, self.dim * 6), ) + # Retained for strict upstream checkpoint-schema parity. The pinned + # upstream model embeds text through ``condition_embedder.text_embedder`` + # for both video and action inference; its copied action text embedder + # is never called, so using this module would change model behavior. self.action_text_embedding = nn.Sequential( nn.Linear(config.text_dim, self.dim), nn.GELU(approximate="tanh"), diff --git a/integrations/lingbot_va/tests/test_cfg_cache.py b/integrations/lingbot_va/tests/test_cfg_cache.py index 70386781c..c03f43e97 100644 --- a/integrations/lingbot_va/tests/test_cfg_cache.py +++ b/integrations/lingbot_va/tests/test_cfg_cache.py @@ -92,6 +92,43 @@ def forward_action( return torch.full_like(x, value * 3) +def test_transformer_default_chunk_size_matches_robotwin() -> None: + assert LingbotVATransformerConfig().frame_chunk_size == 2 + + +def test_cache_window_counts_paired_video_action_regions() -> None: + config = LingbotVATransformerConfig( + network=WanVADiTNetworkConfig( + patch_size=(1, 2, 2), + dim=12, + num_heads=1, + num_layers=1, + text_dim=8, + ), + guidance_scale=1.0, + action_guidance_scale=1.0, + latent_height=8, + latent_width=8, + frame_chunk_size=4, + action_per_frame=3, + attn_window=30, + compile_network=False, + ) + transformer = LingbotVATransformer(config) + network = WanVADiTNetwork(config.network) + object.__setattr__(transformer, "_network", network) + + cache = transformer.initialize_autoregressive_cache( + text_embeddings=torch.zeros(1, 1, 8), + batch_size=1, + ) + + self_attn = cache.network_cache.block_caches[0].self_attn + assert self_attn.video_chunk == 64 + assert self_attn.action_chunk == 12 + assert self_attn.kv_cache.window_size == 15 * (64 + 12) + + def test_cfg_action_branches_consume_their_matching_video_kv() -> None: cond_cache = WanVADiTNetworkCache(block_caches=[]) uncond_cache = WanVADiTNetworkCache(block_caches=[]) From 6b3c6fbb647a55d54c8b331e0d8839b6f3f3a481 Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Thu, 27 Aug 2026 00:23:10 -0700 Subject: [PATCH 16/18] Add LingBot-VA model card to docs Signed-off-by: Jonathan McCaffrey --- docs/source/community/faq.rst | 5 +- docs/source/models/index.rst | 24 +- docs/source/models/lingbot_va.rst | 319 +++++++++++++++++++++++++++ integrations/lingbot_va/README.md | 50 +---- integrations_v2/README.md | 5 +- integrations_v2/lingbot_va/README.md | 2 +- 6 files changed, 347 insertions(+), 58 deletions(-) create mode 100644 docs/source/models/lingbot_va.rst diff --git a/docs/source/community/faq.rst b/docs/source/community/faq.rst index 7724b872b..c0a1fecff 100644 --- a/docs/source/community/faq.rst +++ b/docs/source/community/faq.rst @@ -51,13 +51,16 @@ integrations are: - :doc:`/models/causal_forcing` — Streaming Wan 2.1 T2V / I2V (1.3B). - :doc:`/models/causal_wan22` — FastVideo Causal Wan 2.2 14B MoE T2V. - :doc:`/models/lingbot_world` — Camera-controllable I2V world model. +- :doc:`/models/lingbot_va` — Offline autoregressive RoboTwin + video-and-action rollout. - :doc:`/models/flashvsr` — Streaming video super-resolution. - :doc:`/models/wan21` — Bidirectional Wan 2.1 T2V / I2V reference. - :doc:`/models/cosmos_predict2` — Bidirectional Cosmos-Predict2.5 T2V / I2V reference. Each model page has the canonical CLI invocation, checkpoint source, -multi-GPU command, and per-method knobs. +requirements, and per-method knobs. Pages include multi-GPU commands where the +integration supports them. Installation and packaging -------------------------- diff --git a/docs/source/models/index.rst b/docs/source/models/index.rst index 5fdca56a0..f558ce5d4 100644 --- a/docs/source/models/index.rst +++ b/docs/source/models/index.rst @@ -28,6 +28,7 @@ Models flashvsr hy_worldplay lingbot_world + lingbot_va sana_wm_streaming sana_wm_bidirectional wan21 @@ -41,14 +42,13 @@ method. Available models ---------------- -The models come in three flavors. Streaming and autoregressive generation -methods build a video step by step and stay fast once warmed up, aiming for -sub-second latency per step; bidirectional methods produce a clip in a single -pass and serve as the quality reference for their streaming counterparts; and -super-resolution methods upscale existing frames in chunks, so their latency -scales with output resolution rather than step count. Each card links to that -method's page, where you'll find the exact command to run it, the checkpoint it -uses, and the settings you can tune. +The models cover streaming and autoregressive generation, bidirectional +generation, and super-resolution. Streaming methods can present output +incrementally; other autoregressive methods may generate step by step but +return one complete rollout. Bidirectional methods produce a clip in a single +pass, while super-resolution methods upscale existing frames in chunks. Each +card links to that method's page, where you'll find the exact command to run, +the checkpoint it uses, and the settings you can tune. .. container:: fd-eyebrow @@ -127,6 +127,14 @@ uses, and the settings you can tune. Camera-controllable image-to-video world model. + .. grid-item-card:: LingBot-VA + :class-card: fd-feature + :link: /models/lingbot_va + :link-type: doc + + Offline autoregressive RoboTwin image-and-instruction to joint + video/action rollout. + .. grid-item-card:: HY-WorldPlay :class-card: fd-feature :link: /models/hy_worldplay diff --git a/docs/source/models/lingbot_va.rst b/docs/source/models/lingbot_va.rst new file mode 100644 index 000000000..38df33bc5 --- /dev/null +++ b/docs/source/models/lingbot_va.rst @@ -0,0 +1,319 @@ +.. SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +.. SPDX-License-Identifier: Apache-2.0 +.. +.. Licensed under the Apache License, Version 2.0 (the "License"); +.. you may not use this file except in compliance with the License. +.. You may obtain a copy of the License at +.. +.. http://www.apache.org/licenses/LICENSE-2.0 +.. +.. Unless required by applicable law or agreed to in writing, software +.. distributed under the License is distributed on an "AS IS" BASIS, +.. WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +.. See the License for the specific language governing permissions and +.. limitations under the License. + +LingBot-VA +========== + +.. container:: fd-cta-row + + .. button-link:: https://technology.robbyant.com/lingbot-va + :color: primary + + Project page + + .. button-link:: https://arxiv.org/abs/2601.21998 + :color: primary + + arXiv paper + + .. button-link:: https://github.com/Robbyant/lingbot-va + :color: primary + + Official code + + .. button-link:: https://huggingface.co/robbyant/lingbot-va-posttrain-robotwin + :color: primary + + Checkpoint + +LingBot-VA is an autoregressive diffusion video-action world-model policy. It +uses a shared video/action backbone to predict visual dynamics and robot actions. +The FlashDreams integration implements the pinned RoboTwin image-to-video-action +(I2AV) path as an offline, batch-one rollout. It does not implement the upstream +closed-loop observation-feedback or asynchronous motor-execution system. + +Supported FlashDreams method +---------------------------- + +.. list-table:: + :header-rows: 1 + :widths: 30 70 + + * - Field + - Integrated behavior + * - Application slug + - ``lingbot-va-robotwin-i2av`` + * - Task + - One instruction and three RGB camera observations to a predicted + high-camera video and denormalized RoboTwin actions + * - Inputs + - Natural-language prompt plus high, left-wrist, and right-wrist PNGs + * - Outputs + - TCHW video at 10 FPS, float32 ``actions[step, channel]``, and numeric + inference metrics + * - Execution + - BF16 on one CUDA device, with optional ``torch.compile`` and component + offload; one complete rollout per engine + * - Checkpoint + - ``robbyant/lingbot-va-posttrain-robotwin`` pinned to + ``8c9dea8abbc5c91cc9e18bc3264b8915083bbe70`` + +Requirements +------------ + +- **Python**: 3.10 or newer. +- **Runtime dependencies**: Diffusers 0.38.x and Transformers 5.x. The + integration uses private Wan VAE streaming fields, so widening the Diffusers + range requires a real-model retest. +- **GPU validation**: one NVIDIA RTX PRO 6000 Blackwell Workstation Edition + (97,887 MiB), BF16. The measured two-chunk peak was 37.07 GiB with component + offload and 40.35 GiB with components resident. These are measurements, not + guaranteed minimum-VRAM requirements. +- **Checkpoint storage**: approximately 22.7 GiB of resolved weight files. +- **Input data**: three camera images are required and are not bundled. + +Installation +------------ + +From the repository root: + +.. code-block:: bash + + uv sync --project integrations_v2/lingbot_va + +Running the method +------------------ + +.. code-block:: bash + + uv run --project integrations_v2/lingbot_va flashdreams-run-v2 \ + lingbot-va-robotwin-i2av \ + --mode mp4 \ + --output-path outputs/lingbot_va/demo.mp4 \ + --stats-path outputs/lingbot_va/metrics.json \ + --tensor-artifact-dir outputs/lingbot_va \ + -- \ + --checkpoint-root robbyant/lingbot-va-posttrain-robotwin \ + --checkpoint-revision 8c9dea8abbc5c91cc9e18bc3264b8915083bbe70 \ + --input-image-dir /path/to/robotwin-images \ + --num-chunks 10 + +Use ``--no-compile`` for correctness debugging and ``--enable-offload`` when +GPU memory is constrained. Run the following to list every model-specific +option: + +.. code-block:: bash + + uv run --project integrations_v2/lingbot_va flashdreams-run-v2 \ + lingbot-va-robotwin-i2av -- --help + +Inputs and outputs +------------------ + +The input directory must contain these exact filenames: + +- ``observation.images.cam_high.png`` +- ``observation.images.cam_left_wrist.png`` +- ``observation.images.cam_right_wrist.png`` + +The high camera is encoded at 256x320. Each wrist camera is encoded at 128x160; +their latents form the upper bar of the upstream RoboTwin T layout. + +Let ``N`` be the positive ``--num-chunks`` value. One model step returns: + +- floating video ``[8N - 3, 3, 256, 320]`` in ``[-1, 1]`` at 10 FPS; +- float32 actions ``[32N, 16]``, selected in channel order + ``0..6, 28, 7..13, 29``; +- prompt, observation, denoise, decode, total, and peak-allocation metrics. + +MP4, NumPy action, and JSON metric serialization are provided by generic V2 +runtime sinks rather than model-specific file handling. + +Model details +------------- + +.. list-table:: + :header-rows: 1 + :widths: 27 73 + + * - Component + - Integrated configuration + * - Text encoder + - UMT5-XXL; 4,096-wide states; at most 512 tokens + * - VAE + - Wan VAE; 48 latent channels; 16x spatial and 4x temporal scaling + * - Video-action DiT + - 5,088,872,670 parameters; 30 shared blocks; width 3,072; FFN width + 14,336; 24 attention heads of width 128 + * - Video patches + - ``[1, 2, 2]`` + * - Chunk geometry + - Two latent frames; 240 video tokens and 32 action tokens per chunk + * - Attention cache + - 36 autoregressive slots per conditional or unconditional branch; + 9,792 tokens per block and branch + * - Guidance defaults + - Video CFG 5; action CFG 1 + * - Checkpoint footprint + - 9.48 GiB transformer, 10.58 GiB text encoder, and 2.63 GiB VAE, + measured from the pinned snapshot + +The upstream RoboTwin setting ``attn_window=72`` counts alternating video and +action regions: two regions form one autoregressive cache slot. The divisor is +therefore independent of the two latent frames in a chunk. + +Intended use and safety +----------------------- + +.. list-table:: + :header-rows: 1 + :widths: 50 50 + + * - Intended and validated + - Outside this integration's validated scope + * - Research and development of offline RoboTwin video/action rollout + inference + - Direct, unattended, or safety-critical robot actuation + * - Numerical parity, lifecycle, packaging, and V2 runtime regression tests + - Claims of physical accuracy, task success, or safe action execution + * - Generating inspectable video, action, and metric artifacts + - Training, fine-tuning, LIBERO checkpoints, multi-GPU/FSDP, or online + policy serving + +Predicted actions can be wrong, temporally inconsistent, or unsafe. Distribution +shift, camera calibration, prompt ambiguity, and accumulated autoregressive +error can affect both modalities. Evaluate in simulation or a +hardware-interlocked environment with task-specific limits and human oversight +before considering physical use. + +Data, evaluation, and provenance +-------------------------------- + +The architecture and inference behavior are ported from +`Robbyant/lingbot-va at 7c6ffa9 +`_. +Upstream identifies +`robotwin-clean-and-aug-lerobot +`_ +as its cleaned and augmented RoboTwin post-training dataset. FlashDreams does +not train, alter, or independently audit the checkpoint or dataset. + +The `LingBot-VA paper `_ reports simulated +and real-robot results. This integration has not reproduced its task-success +rates. The evidence below establishes native-flow parity, output contracts, and +system execution for the pinned checkpoint. + +Validation evidence +------------------- + +The pinned checkpoint contains 841 transformer entries. Two obsolete +``patch_embedding.*`` entries are dropped; the remaining 839 entries map +bijectively to the native model's 839 entries and load strictly. + +Matched first-chunk upstream/native comparisons use maximum absolute error +``<= 0.07`` and mean absolute error ``<= 0.012`` as acceptance gates. + +.. list-table:: + :header-rows: 1 + :widths: 18 14 22 22 22 + + * - Native mode + - Stream + - Maximum absolute error + - Mean absolute error + - RMS error + * - eager + - video + - 0.04296875 + - 0.00751040 + - 0.00951632 + * - eager + - action + - 0.06250000 + - 0.00875314 + - 0.01267146 + * - compiled + - video + - 0.05468750 + - 0.00970979 + - 0.01229867 + * - compiled + - action + - 0.06250000 + - 0.01116651 + - 0.01560142 + +Matched resident and offloaded two-chunk runs used default CFG, 25 video steps, +50 action steps, no compilation, one CUDA device, and seed 42. + +.. list-table:: + :header-rows: 1 + :widths: 18 18 18 18 18 + + * - Mode + - Prompt + observation + - Denoise + - Total + - Peak allocation + * - resident + - 0.461 s + - 4.735 s + - 33.220 s + - 40.35 GiB + * - offload + - 5.831 s + - 4.744 s + - 34.009 s + - 37.07 GiB + +Both runs returned finite video ``[13, 3, 256, 320]`` and actions +``[64, 16]`` with byte-identical outputs. Final stacked-PR revalidation passed +in 34.02 seconds with peak allocation 39,804,413,440 bytes. The action SHA-256 +remained +``463b307b667c1ca13a47bbbc5a17f68604621dfe3c3a10fc5860077216928d95``. + +These are implementation measurements from 2026-08-25 and 2026-08-26, not +general model-performance or robot-success claims. Full input hashes, +reproduction commands, phase timings, MP4 metadata, and architectural diagrams +are maintained in the +`LingBot-VA integration README +`_. + +Limitations and license +----------------------- + +The integration supports one GPU, one complete deferred-decode rollout, and the +RoboTwin checkpoint above. It makes no FSDP, context-parallel, live-control, +online-serving, or per-chunk presentation claim. + +The FlashDreams packages are Apache-2.0. The upstream repository and checkpoint +model card also identify Apache-2.0. Users remain responsible for checkpoint and +dataset terms. + +Citation +-------- + +If you use LingBot-VA, cite the original work: + +.. code-block:: bibtex + + @article{lingbot-va2026, + title={Causal World Modeling for Robot Control}, + author={Li, Lin and Zhang, Qihang and Luo, Yiming and Yang, Shuai and + Wang, Ruilin and Han, Fei and Yu, Mingrui and Gao, Zelin and Xue, Nan + and Zhu, Xing and Shen, Yujun and Xu, Yinghao}, + journal={arXiv preprint arXiv:2601.21998}, + year={2026} + } diff --git a/integrations/lingbot_va/README.md b/integrations/lingbot_va/README.md index c0668265b..6f5d9dda9 100644 --- a/integrations/lingbot_va/README.md +++ b/integrations/lingbot_va/README.md @@ -16,52 +16,10 @@ propagation, lifecycle, and output contracts were corrected for the V2 API. The draft performance claims are intentionally omitted; only matched measurements from the checked-in implementation are recorded below. -## Model card - -### Identity and supported task - -| Field | Integrated behavior | -| --- | --- | -| Model family | [LingBot-VA](https://arxiv.org/abs/2601.21998), an autoregressive diffusion video-action world-model policy with a shared video/action backbone | -| Checkpoint | [`robbyant/lingbot-va-posttrain-robotwin`](https://huggingface.co/robbyant/lingbot-va-posttrain-robotwin), pinned to `8c9dea8abbc5c91cc9e18bc3264b8915083bbe70` | -| Implemented task | Offline, batch-one Robotwin image-and-instruction to predicted video-and-action rollout (I2AV) | -| Input modalities | One natural-language prompt and three RGB camera PNGs: high, left wrist, and right wrist | -| Output modalities | Predicted high-camera video, 16 denormalized Robotwin action channels, and inference metrics | -| Core models | UMT5-XXL text encoder (4,096-wide states), Wan VAE (48 latent channels, 16x spatial and 4x temporal scaling), and the shared video/action DiT | -| DiT architecture | 5,088,872,670 parameters; 30 blocks; width 3,072; FFN width 14,336; 24 heads of width 128; `[1, 2, 2]` video patches | -| Chunk and cache geometry | Two latent frames per chunk; 240 video plus 32 action tokens; 36 rolling chunk slots per conditional/unconditional branch | -| Checkpoint footprint | About 22.7 GiB of resolved weight files (9.48 GiB transformer, 10.58 GiB text encoder, 2.63 GiB VAE), measured from the pinned snapshot | -| Runtime profile | BF16 on one CUDA device; optional `torch.compile` and component offload; one complete rollout per engine | -| License | This package is Apache-2.0. The [upstream repository](https://github.com/Robbyant/lingbot-va) and published checkpoint model card also identify Apache-2.0; users remain responsible for checkpoint and dataset terms. | - -### Provenance, data, and evaluation boundary - -- The architecture and inference behavior are ported from - [`Robbyant/lingbot-va@7c6ffa9`](https://github.com/Robbyant/lingbot-va/tree/7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb). -- The upstream project identifies - [`robotwin-clean-and-aug-lerobot`](https://huggingface.co/datasets/robbyant/robotwin-clean-and-aug-lerobot) - as its cleaned and augmented RoboTwin post-training dataset. FlashDreams does - not train, alter, or independently audit the checkpoint or dataset. -- The upstream [paper](https://arxiv.org/abs/2601.21998) reports simulated and - real-robot results. This integration has not reproduced task success rates; - the evidence below establishes native-flow parity, output contracts, and - system execution on the pinned checkpoint. - -### Intended use and safety boundary - -| Intended and validated | Outside this integration's validated scope | -| --- | --- | -| Research and development of offline Robotwin video/action rollout inference | Direct, unattended, or safety-critical robot actuation | -| Numerical parity, lifecycle, packaging, and V2 runtime regression testing | Claims of physical accuracy, task success, or safe action execution | -| Generating inspectable MP4, NumPy action, and JSON metric artifacts | Training, fine-tuning, LIBERO checkpoints, multi-GPU/FSDP, or online serving | - -Predicted actions can be wrong, temporally inconsistent, or unsafe. This port -starts from one fixed camera observation and does not implement the upstream -closed-loop observation feedback or asynchronous motor-execution system. -Distribution shift, camera calibration, prompt ambiguity, and accumulated -autoregressive error can affect both modalities. Evaluate in simulation or a -hardware-interlocked environment with task-specific limits and human oversight -before considering any physical use. +The public [LingBot-VA model card](../../docs/source/models/lingbot_va.rst) +records model identity, checkpoint and data provenance, requirements, intended +use, safety boundaries, and measured validation evidence. This README focuses +on package contracts, architecture, and reproduction details. ## Install and run diff --git a/integrations_v2/README.md b/integrations_v2/README.md index 20116707f..c750b4748 100644 --- a/integrations_v2/README.md +++ b/integrations_v2/README.md @@ -24,8 +24,9 @@ follows is already done for you. `t2v_wan21`, `t2v_cosmos_predict2` — real models, each a thin wrapper over `flashdreams.t2v_v2`. - [`lingbot_va`](lingbot_va/README.md) — the LingBot-VA Robotwin - image-to-action/video model; its linked model card and architecture review - document the atypical finite, deferred-decode V2 boundary. + image-to-action/video model. See its + [model card](../docs/source/models/lingbot_va.rst) and + [architecture review](../integrations/lingbot_va/README.md#architecture-design-review). - `null_model` — not an application. A v1 pipeline the framework tests use as a fixture. diff --git a/integrations_v2/lingbot_va/README.md b/integrations_v2/lingbot_va/README.md index 9a5fd22a5..7cd850d6a 100644 --- a/integrations_v2/lingbot_va/README.md +++ b/integrations_v2/lingbot_va/README.md @@ -33,7 +33,7 @@ rollout before reporting finished. `--input-image-dir` is required because no camera images are bundled. Use `-- --help` after the application slug for checkpoint, input, compilation, offload, seed, guidance, inference-step, and scheduler-shift overrides. The -[model card](../../integrations/lingbot_va/README.md#model-card) records +[model card](../../docs/source/models/lingbot_va.rst) records provenance, intended use, safety boundaries, tensor geometry, and measured evidence. The accompanying [architecture design review](../../integrations/lingbot_va/README.md#architecture-design-review) From f64394bb66fe5b420945f9272b67bd83be5e8b7f Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Thu, 27 Aug 2026 10:24:49 -0700 Subject: [PATCH 17/18] Fix LingBot Mermaid sequence syntax Signed-off-by: Jonathan McCaffrey --- integrations/lingbot_va/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/integrations/lingbot_va/README.md b/integrations/lingbot_va/README.md index 6f5d9dda9..ee80156fe 100644 --- a/integrations/lingbot_va/README.md +++ b/integrations/lingbot_va/README.md @@ -304,7 +304,7 @@ sequenceDiagram A-->>R: session preserving runtime backpressure/presentation R->>S: init() and step(0) S->>E: lazily construct engine and run() - E->>V: load tokenizer, UMT5, and Wan VAE; encode prompt/cameras + E->>V: load tokenizer, UMT5, and Wan VAE, then encode prompt and cameras E->>P: initialize conditional and optional unconditional caches loop chunk 0 through N-1 @@ -329,7 +329,7 @@ sequenceDiagram E->>E: move completed chunk outputs to CPU end - E->>E: drop caches, DiT, text owners; collect; empty CUDA cache + E->>E: drop caches, DiT, and text owners, collect garbage, empty CUDA cache E->>V: decode all latent frames, crop high camera, release VAE E-->>S: video, denormalized actions, metrics S-->>R: one validated StepResult From 7a15f4399082b88ba98ce951ba25ec22df319edd Mon Sep 17 00:00:00 2001 From: Jonathan McCaffrey Date: Thu, 27 Aug 2026 18:21:13 -0700 Subject: [PATCH 18/18] Clarify LingBot offline action contract --- docs/source/models/lingbot_va.rst | 61 +++- integrations/lingbot_va/README.md | 89 +++++- .../lingbot_va/lingbot_va/action_artifact.py | 289 ++++++++++++++++++ integrations/lingbot_va/pyproject.toml | 8 + .../lingbot_va/tests/test_action_artifact.py | 100 ++++++ integrations_v2/lingbot_va/README.md | 12 + .../lingbot_va/lingbot_va_v2/app.py | 177 +---------- .../lingbot_va/lingbot_va_v2/session.py | 220 +++++++++++++ .../lingbot_va_v2/tests/test_app.py | 7 +- uv.lock | 13 +- 10 files changed, 781 insertions(+), 195 deletions(-) create mode 100644 integrations/lingbot_va/lingbot_va/action_artifact.py create mode 100644 integrations/lingbot_va/tests/test_action_artifact.py create mode 100644 integrations_v2/lingbot_va/lingbot_va_v2/session.py diff --git a/docs/source/models/lingbot_va.rst b/docs/source/models/lingbot_va.rst index 38df33bc5..afe5a8fc2 100644 --- a/docs/source/models/lingbot_va.rst +++ b/docs/source/models/lingbot_va.rst @@ -44,6 +44,16 @@ The FlashDreams integration implements the pinned RoboTwin image-to-video-action (I2AV) path as an offline, batch-one rollout. It does not implement the upstream closed-loop observation-feedback or asynchronous motor-execution system. +This limitation is specific to the FlashDreams adapter. The pinned upstream +repository also contains a +`RoboTwin evaluator +`_ +and `model server +`_ +that execute action chunks, capture actual simulator observations, and feed the +observations and executed state back into the model cache. That environment and +execution bridge is not included here. + Supported FlashDreams method ---------------------------- @@ -141,6 +151,50 @@ Let ``N`` be the positive ``--num-chunks`` value. One model step returns: MP4, NumPy action, and JSON metric serialization are provided by generic V2 runtime sinks rather than model-specific file handling. +The 16 action columns are relative two-arm RoboTwin commands: + +.. list-table:: + :header-rows: 1 + :widths: 18 82 + + * - Columns + - Meaning + * - ``0..2`` + - Left end-effector x/y/z translation delta + * - ``3..6`` + - Left relative quaternion x/y/z/w + * - ``7`` + - Left gripper command + * - ``8..10`` + - Right end-effector x/y/z translation delta + * - ``11..14`` + - Right relative quaternion x/y/z/w + * - ``15`` + - Right gripper command + +The upstream evaluator composes the relative poses with the episode's initial +end-effector poses and normalizes the resulting quaternions before simulator +execution. FlashDreams emits the denormalized relative values and does not +perform pose composition or actuation. + +Inspecting action outputs +------------------------- + +The integration supplies a model-specific inspector that validates a committed +tensor-artifact directory, plots both arms' translation/quaternion/gripper +channels, and can export named CSV columns: + +.. code-block:: bash + + uv run --project integrations/lingbot_va --extra visualization \ + lingbot-va-visualize-actions outputs/lingbot_va \ + --output outputs/lingbot_va/actions.png \ + --csv-output outputs/lingbot_va/actions.csv + +A direct ``actions.npy`` path from an older validation run is also accepted. +The plot is a diagnostic for inspection and batch comparison; it is not a task +success, physical-validity, or robot-safety evaluation. + Model details ------------- @@ -279,12 +333,13 @@ Matched resident and offloaded two-chunk runs used default CFG, 25 video steps, - 37.07 GiB Both runs returned finite video ``[13, 3, 256, 320]`` and actions -``[64, 16]`` with byte-identical outputs. Final stacked-PR revalidation passed -in 34.02 seconds with peak allocation 39,804,413,440 bytes. The action SHA-256 +``[64, 16]`` with byte-identical outputs. Final post-rebase revalidation passed +in 35.82 seconds, with model-reported total 31.516 seconds and peak allocation +39,804,413,440 bytes. The action SHA-256 remained ``463b307b667c1ca13a47bbbc5a17f68604621dfe3c3a10fc5860077216928d95``. -These are implementation measurements from 2026-08-25 and 2026-08-26, not +These are implementation measurements from 2026-08-25 and 2026-08-27, not general model-performance or robot-success claims. Full input hashes, reproduction commands, phase timings, MP4 metadata, and architectural diagrams are maintained in the diff --git a/integrations/lingbot_va/README.md b/integrations/lingbot_va/README.md index ee80156fe..df0fa3a3b 100644 --- a/integrations/lingbot_va/README.md +++ b/integrations/lingbot_va/README.md @@ -21,6 +21,24 @@ records model identity, checkpoint and data provenance, requirements, intended use, safety boundaries, and measured validation evidence. This README focuses on package contracts, architecture, and reproduction details. +## Integrated scope versus the upstream policy + +This integration deliberately wraps the pinned upstream `generate()` I2AV path: +one instruction and three initial RGB observations are consumed, then all video +and action chunks are generated internally. Its V2 `step()` does not consume +live `UserInputEvents`, simulator observations, or controller input. It is an +offline rollout generator, not a closed-loop robot-control session. + +The distinction is about this adapter, not LingBot-VA as a whole. Pinned +upstream code also includes a RoboTwin client/server evaluation loop. That loop +executes an action chunk in the simulator, captures actual keyframe +observations, and sends both observations and executed state back to the model +to update its KV cache. See the upstream +[`eval_polict_client_openpi.py`](https://github.com/Robbyant/lingbot-va/blob/7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb/evaluation/robotwin/eval_polict_client_openpi.py#L542-L609) +and [`wan_va_server.py`](https://github.com/Robbyant/lingbot-va/blob/7c6ffa9bfc4b83582cafc860fab4c82cc7deeeeb/wan_va/wan_va_server.py#L572-L627). +FlashDreams does not port that environment bridge, motor execution, asynchronous +policy serving, or feedback-cache update path in this PR. + ## Install and run From the repository root: @@ -93,12 +111,51 @@ turns `2N` accumulated latent frames into `8N - 3` pixel frames. The decoded T layout is cropped to its 256x320 high-camera view. Actions contain 16 selected Robotwin channels in the order `0..6, 28, 7..13, 29`. +The emitted columns have these upstream meanings: + +| Columns | Meaning | +| --- | --- | +| `0..2` | left end-effector translation delta, x/y/z | +| `3..6` | left relative quaternion, x/y/z/w | +| `7` | left gripper command | +| `8..10` | right end-effector translation delta, x/y/z | +| `11..14` | right relative quaternion, x/y/z/w | +| `15` | right gripper command | + +The upstream RoboTwin evaluator composes each relative pose with the episode's +initial end-effector pose, normalizes the resulting quaternions, and then calls +the simulator's end-effector action API. The FlashDreams integration returns the +denormalized relative values but intentionally performs none of those execution +steps. + MP4, JSON, and NumPy serialization belong to generic V2 runtime sinks. The adapter declares `actions[step, channel]` once and attaches the tensor to its single model-result channel. Backpressure and presentation policies selected by the runtime are preserved; the fixed video layout, dimensions, rate, and artifact schema are validated. +The generic V2 result already represents artifacts as a tuple, so one result +can carry multiple independently named tensors without changing its API. +LingBot-VA is currently the first model integration exercising that facility; +validation with a second model remains an explicit generalization follow-up. + +### Inspect the action artifact + +Install the optional plotting dependency, then render the persisted trajectory +and optionally export named columns to CSV: + +```bash +uv run --project integrations/lingbot_va --extra visualization \ + lingbot-va-visualize-actions outputs/lingbot_va \ + --output outputs/lingbot_va/actions.png \ + --csv-output outputs/lingbot_va/actions.csv +``` + +The command requires a complete `tensor_artifacts.json` when given an output +directory. It also accepts a direct `actions.npy` from older validation runs. +The plot is for human inspection and batch comparison; it does not establish +task success, physical correctness, or safe executability. + ## Architecture design review The architecture keeps model-specific numerical code in @@ -110,7 +167,7 @@ runtime code has no LingBot-specific branches. | Decision | Rationale and consequence | | --- | --- | -| Separate model and V2 adapter packages | The model package owns checkpoint/model/tensor behavior; `app.py` owns only V2 contracts and lifecycle adaptation. | +| Separate model and V2 adapter packages | The model package owns checkpoint/model/tensor behavior; V2 `app.py` owns configuration and the session contract, while `session.py` owns the finite loop and lifecycle adaptation. | | Declare the natural session before model initialization | CLI/runtime compatibility and output schemas fail fast without loading approximately 23 GiB of checkpoint data. | | Session-owned, lazily created engine | Mutable CUDA state is isolated to the model thread and reset creates a fresh one-run engine. | | One complete rollout per `StepResult` | Decode requires destructive DiT/KV teardown, so per-chunk presentation would claim a streaming capability the engine does not provide. | @@ -118,6 +175,14 @@ runtime code has no LingBot-specific branches. | Plain tensors across the compiled block boundary | Cache extraction and writes remain eager while the 30-block video/action loops can be compiled without cache-object graph breaks. | | Generic runtime sinks | The adapter returns TCHW video, metrics, and a typed `actions` artifact; MP4/JSON/NumPy serialization stays reusable. | +The adapter does not use the shared text-to-video session because its contract +is materially different: it requires three image observations in addition to +text, produces robot actions alongside video, and completes all autoregressive +chunks before a destructive teardown and deferred decode. It does not expose +the per-block `generate`/`finalize` lifecycle expected by the shared T2V +adapter. Reusing V2 interfaces and generic sinks preserves the useful common +surface without misclassifying the model as text-only video generation. + ### Static view: packages and components ```mermaid @@ -128,10 +193,10 @@ flowchart LR Sinks["Generic sinks
MP4, metrics JSON, actions NPY"] end - subgraph Adapter["integrations_v2/lingbot_va/lingbot_va_v2/app.py"] - App["LingbotVAApplication
IApplication"] - Session["LingbotVASession
ISession"] - Loop["LingbotVAModelLoop
IModelLoop"] + subgraph Adapter["integrations_v2/lingbot_va/lingbot_va_v2"] + App["app.py
LingbotVAApplication / IApplication"] + Session["session.py
LingbotVASession / ISession"] + Loop["session.py
LingbotVAModelLoop / IModelLoop"] end subgraph Model["integrations/lingbot_va/lingbot_va"] @@ -260,7 +325,7 @@ flowchart LR Metrics["Phase timing and peak CUDA allocation"] end - Live["Live feedback, policy serving, and robot actuation
not implemented"]:::outside + Live["Upstream closed-loop feedback, policy serving,
simulator or robot actuation: not integrated"]:::outside User --> Validate Prompt --> Text @@ -399,7 +464,7 @@ control, unmeasured speedup claims, and root CUDA/Torch policy changes. ## Validation evidence Baseline parity and matched resident/offload evidence were produced on -2026-08-25; final stacked-PR revalidation was produced on 2026-08-26. All runs +2026-08-25; post-rebase PR revalidation was produced on 2026-08-27. All runs used an NVIDIA RTX PRO 6000 Blackwell Workstation Edition (97,887 MiB), driver 595.84, PyTorch 2.12.1+cu130, CUDA 13.0, BF16, one CUDA device, and seed 42. This is implementation evidence, not a general model-performance or robot-task @@ -474,13 +539,13 @@ uv run --no-sync pytest integrations_v2/lingbot_va -m ci_gpu -s Set `LINGBOT_VA_CHECKPOINT_ROOT` to reuse a local snapshot. `LINGBOT_VA_REAL_MODEL_COMPILE_RUN=1` separately enables the cold compile test. -### Final stacked-PR revalidation +### Final post-rebase PR revalidation -After the review fixes, the same pinned checkpoint and input hashes were run -through the final stacked code with two chunks, default CFG, offload, and no -compilation: +After the design-review refactor and rebase onto the updated tensor-artifact +base, the same pinned checkpoint and input hashes were run through the final +code with two chunks, default CFG, offload, and no compilation: -- GPU test: 1 passed in 34.02 s; +- GPU test: 1 passed in 35.82 s; model-reported total 31.516 s; - MP4: H.264, 320x256, 10 FPS, 13 frames, SHA-256 `15bcdc4307e080218255e83946c2c2e5dbc30f3b7acd26c8925167017234e586`; - actions: float32 `[64, 16]`, finite, distinct chunks, SHA-256 diff --git a/integrations/lingbot_va/lingbot_va/action_artifact.py b/integrations/lingbot_va/lingbot_va/action_artifact.py new file mode 100644 index 000000000..c6e4930c5 --- /dev/null +++ b/integrations/lingbot_va/lingbot_va/action_artifact.py @@ -0,0 +1,289 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Inspect persisted LingBot-VA Robotwin action artifacts.""" + +from __future__ import annotations + +import argparse +import csv +import json +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any, cast + +import numpy as np +import numpy.typing as npt + +FloatArray = npt.NDArray[np.float32] + +ROBOTWIN_ACTION_CHANNEL_NAMES = ( + "left_delta_x_m", + "left_delta_y_m", + "left_delta_z_m", + "left_relative_qx", + "left_relative_qy", + "left_relative_qz", + "left_relative_qw", + "left_gripper", + "right_delta_x_m", + "right_delta_y_m", + "right_delta_z_m", + "right_relative_qx", + "right_relative_qy", + "right_relative_qz", + "right_relative_qw", + "right_gripper", +) +"""Column names for the 16 action channels selected by the Robotwin config.""" + +_ARTIFACT_TYPE = "flashdreams.runtime_v2.tensor_artifacts" +_MANIFEST_FILENAME = "tensor_artifacts.json" + + +def load_action_artifact(source: str | Path) -> FloatArray: + """Load and validate one LingBot-VA action array. + + A directory must contain a committed FlashDreams tensor-artifact manifest. + A direct ``.npy`` path is also accepted for older validation outputs created + before the manifest sink was available. + + Args: + source: Artifact directory or direct ``actions.npy`` path. + + Returns: + A finite, non-empty float32 array with shape ``[step, 16]``. + + Raises: + FileNotFoundError: A required manifest or array is missing. + ValueError: The manifest or action tensor violates the expected schema. + """ + source_path = Path(source) + expected_shape: tuple[int, ...] | None = None + expected_dtype: str | None = None + if source_path.is_dir(): + array_path, expected_shape, expected_dtype = _artifact_path(source_path) + else: + array_path = source_path + + loaded = np.load(array_path, allow_pickle=False) + if not isinstance(loaded, np.ndarray): + raise ValueError(f"Action artifact is not a NumPy array: {array_path}") + if expected_shape is not None and tuple(loaded.shape) != expected_shape: + raise ValueError( + f"Action array shape {tuple(loaded.shape)} does not match manifest " + f"shape {expected_shape}." + ) + if expected_dtype is not None and str(loaded.dtype) != expected_dtype: + raise ValueError( + f"Action array dtype {loaded.dtype} does not match manifest dtype " + f"{expected_dtype}." + ) + if loaded.dtype != np.float32: + raise ValueError(f"Actions must have dtype float32; received {loaded.dtype}.") + if loaded.ndim != 2 or loaded.shape[0] == 0 or loaded.shape[1] != 16: + raise ValueError( + "Actions must have non-empty shape [step, 16]; received " + f"{tuple(loaded.shape)}." + ) + if not np.isfinite(loaded).all(): + raise ValueError("Actions contain non-finite values.") + return cast(FloatArray, loaded) + + +def write_action_csv(actions: FloatArray, output_path: str | Path) -> Path: + """Write named Robotwin action channels as a CSV table. + + Args: + actions: Validated action values with shape ``[step, 16]``. + output_path: Destination CSV path. + + Returns: + The resolved output path. + """ + _validate_actions(actions) + destination = Path(output_path) + destination.parent.mkdir(parents=True, exist_ok=True) + with destination.open("w", encoding="utf-8", newline="") as output_file: + writer = csv.writer(output_file) + writer.writerow(("step", *ROBOTWIN_ACTION_CHANNEL_NAMES)) + for step, row in enumerate(actions): + writer.writerow((step, *(float(value) for value in row))) + return destination.resolve() + + +def write_action_plot(actions: FloatArray, output_path: str | Path) -> Path: + """Plot the two-arm Robotwin trajectory channels for human inspection. + + The plot is diagnostic only: it does not apply a robot's initial pose, + execute the actions, or establish task success or physical safety. + + Args: + actions: Validated action values with shape ``[step, 16]``. + output_path: Destination image path. + + Returns: + The resolved output path. + + Raises: + RuntimeError: Matplotlib is not installed. + """ + _validate_actions(actions) + try: + import matplotlib.pyplot as plt + except ImportError as error: + raise RuntimeError( + "Action plotting requires the 'visualization' package extra." + ) from error + + destination = Path(output_path) + destination.parent.mkdir(parents=True, exist_ok=True) + steps = np.arange(actions.shape[0]) + figure, axes = plt.subplots(2, 2, figsize=(14, 8), sharex=True) + panels = ( + (axes[0, 0], range(0, 3), "Left end-effector translation delta"), + (axes[0, 1], range(3, 7), "Left relative quaternion"), + (axes[1, 0], range(8, 11), "Right end-effector translation delta"), + (axes[1, 1], range(11, 15), "Right relative quaternion"), + ) + for axis, channel_ids, title in panels: + for channel_id in channel_ids: + axis.plot( + steps, + actions[:, channel_id], + label=ROBOTWIN_ACTION_CHANNEL_NAMES[channel_id], + ) + axis.set_title(title) + axis.set_ylabel("predicted value") + axis.grid(alpha=0.25) + axis.legend(fontsize="small") + + axes[0, 0].plot( + steps, + actions[:, 7], + linestyle="--", + label=ROBOTWIN_ACTION_CHANNEL_NAMES[7], + ) + axes[1, 0].plot( + steps, + actions[:, 15], + linestyle="--", + label=ROBOTWIN_ACTION_CHANNEL_NAMES[15], + ) + axes[0, 0].legend(fontsize="small") + axes[1, 0].legend(fontsize="small") + axes[1, 0].set_xlabel("action step") + axes[1, 1].set_xlabel("action step") + figure.suptitle("LingBot-VA Robotwin predicted actions") + figure.tight_layout() + figure.savefig(destination, dpi=150) + plt.close(figure) + return destination.resolve() + + +def _artifact_path( + output_dir: Path, +) -> tuple[Path, tuple[int, ...], str]: + """Resolve ``actions.npy`` from a complete tensor-artifact manifest.""" + manifest_path = output_dir / _MANIFEST_FILENAME + with manifest_path.open(encoding="utf-8") as manifest_file: + payload = json.load(manifest_file) + if not isinstance(payload, Mapping): + raise ValueError("Tensor artifact manifest must be a JSON object.") + manifest = cast(Mapping[str, Any], payload) + if manifest.get("artifact_type") != _ARTIFACT_TYPE: + raise ValueError("Directory is not a FlashDreams tensor-artifact output.") + if manifest.get("complete") is not True: + raise ValueError("Tensor artifact output is incomplete and cannot be consumed.") + artifacts = manifest.get("artifacts") + if not isinstance(artifacts, list): + raise ValueError("Tensor artifact manifest has no artifact list.") + action_entries = [ + cast(Mapping[str, Any], entry) + for entry in artifacts + if isinstance(entry, Mapping) and entry.get("name") == "actions" + ] + if len(action_entries) != 1: + raise ValueError("Manifest must declare exactly one 'actions' artifact.") + entry = action_entries[0] + if entry.get("emitted") is not True: + raise ValueError("Manifest declares 'actions' but no array was emitted.") + if entry.get("dimension_names") != ["step", "channel"]: + raise ValueError("Actions must use dimensions ['step', 'channel'].") + relative_path = entry.get("path") + shape = entry.get("shape") + dtype = entry.get("dtype") + if not isinstance(relative_path, str) or Path(relative_path).name != relative_path: + raise ValueError("Actions manifest path must be a local filename.") + if not ( + isinstance(shape, list) + and len(shape) == 2 + and all(isinstance(value, int) for value in shape) + ): + raise ValueError("Actions manifest shape must contain two integer dimensions.") + if not isinstance(dtype, str): + raise ValueError("Actions manifest dtype is missing.") + return output_dir / relative_path, tuple(shape), dtype + + +def _validate_actions(actions: FloatArray) -> None: + """Validate programmatically supplied action data before export.""" + if actions.dtype != np.float32: + raise ValueError(f"Actions must have dtype float32; received {actions.dtype}.") + if actions.ndim != 2 or actions.shape[0] == 0 or actions.shape[1] != 16: + raise ValueError( + "Actions must have non-empty shape [step, 16]; received " + f"{tuple(actions.shape)}." + ) + if not np.isfinite(actions).all(): + raise ValueError("Actions contain non-finite values.") + + +def _parse_args(commandline_args: Sequence[str] | None) -> argparse.Namespace: + """Parse action inspector arguments.""" + parser = argparse.ArgumentParser( + description="Plot or export LingBot-VA Robotwin action artifacts.", + ) + parser.add_argument( + "source", + type=Path, + help="Artifact output directory or direct actions.npy path.", + ) + parser.add_argument( + "--output", + type=Path, + help="Plot destination (default: /actions.png).", + ) + parser.add_argument("--csv-output", type=Path, help="Optional CSV destination.") + return parser.parse_args(commandline_args) + + +def main(commandline_args: Sequence[str] | None = None) -> None: + """Run the LingBot-VA action artifact inspector.""" + args = _parse_args(commandline_args) + source = cast(Path, args.source) + default_parent = source if source.is_dir() else source.parent + output = cast(Path | None, args.output) or default_parent / "actions.png" + actions = load_action_artifact(source) + plot_path = write_action_plot(actions, output) + print(f"Wrote action plot: {plot_path}") + csv_output = cast(Path | None, args.csv_output) + if csv_output is not None: + csv_path = write_action_csv(actions, csv_output) + print(f"Wrote action CSV: {csv_path}") + + +if __name__ == "__main__": + main() diff --git a/integrations/lingbot_va/pyproject.toml b/integrations/lingbot_va/pyproject.toml index c2e64413c..ad03d3096 100644 --- a/integrations/lingbot_va/pyproject.toml +++ b/integrations/lingbot_va/pyproject.toml @@ -28,6 +28,7 @@ dependencies = [ # The engine uses Wan VAE private streaming fields; retest before widening. "diffusers>=0.38,<0.39", "einops", + "numpy>=1.24,<2.5", "Pillow>=10", "transformers>=5.0,<6", ] @@ -37,8 +38,15 @@ flashdreams = { workspace = true } [project.optional-dependencies] dev = [ + "matplotlib>=3.8", "pytest>=8.0", ] +visualization = [ + "matplotlib>=3.8", +] + +[project.scripts] +lingbot-va-visualize-actions = "lingbot_va.action_artifact:main" [tool.setuptools.packages.find] include = ["lingbot_va*"] diff --git a/integrations/lingbot_va/tests/test_action_artifact.py b/integrations/lingbot_va/tests/test_action_artifact.py new file mode 100644 index 000000000..771546b6e --- /dev/null +++ b/integrations/lingbot_va/tests/test_action_artifact.py @@ -0,0 +1,100 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""CPU tests for LingBot-VA action artifact inspection.""" + +from __future__ import annotations + +import csv +import json +from pathlib import Path + +import matplotlib +import numpy as np +import pytest +from lingbot_va.action_artifact import ( + ROBOTWIN_ACTION_CHANNEL_NAMES, + load_action_artifact, + write_action_csv, + write_action_plot, +) + +matplotlib.use("Agg") +pytestmark = pytest.mark.ci_cpu + + +def _write_artifact(output_dir: Path, *, complete: bool = True) -> np.ndarray: + """Write a representative committed or incomplete action artifact.""" + actions = np.arange(64, dtype=np.float32).reshape(4, 16) + output_dir.mkdir() + np.save(output_dir / "actions.npy", actions) + manifest = { + "schema_version": 1, + "artifact_type": "flashdreams.runtime_v2.tensor_artifacts", + "complete": complete, + "generation": 0, + "artifacts": [ + { + "name": "actions", + "path": "actions.npy", + "emitted": True, + "dimension_names": ["step", "channel"], + "concatenate_axis": 0, + "dtype": "float32", + "shape": [4, 16], + } + ], + } + (output_dir / "tensor_artifacts.json").write_text( + json.dumps(manifest), + encoding="utf-8", + ) + return actions + + +def test_loads_complete_manifest_artifact(tmp_path: Path) -> None: + expected = _write_artifact(tmp_path / "output") + + actual = load_action_artifact(tmp_path / "output") + + np.testing.assert_array_equal(actual, expected) + + +def test_rejects_incomplete_manifest_artifact(tmp_path: Path) -> None: + _write_artifact(tmp_path / "output", complete=False) + + with pytest.raises(ValueError, match="incomplete"): + load_action_artifact(tmp_path / "output") + + +def test_accepts_direct_legacy_numpy_output(tmp_path: Path) -> None: + actions = np.zeros((32, 16), dtype=np.float32) + output_path = tmp_path / "actions.npy" + np.save(output_path, actions) + + np.testing.assert_array_equal(load_action_artifact(output_path), actions) + + +def test_writes_named_csv_and_plot(tmp_path: Path) -> None: + actions = np.linspace(-1.0, 1.0, 64, dtype=np.float32).reshape(4, 16) + + csv_path = write_action_csv(actions, tmp_path / "actions.csv") + plot_path = write_action_plot(actions, tmp_path / "actions.png") + + with csv_path.open(newline="", encoding="utf-8") as csv_file: + rows = list(csv.reader(csv_file)) + assert rows[0] == ["step", *ROBOTWIN_ACTION_CHANNEL_NAMES] + assert len(rows) == 5 + assert plot_path.read_bytes().startswith(b"\x89PNG\r\n\x1a\n") diff --git a/integrations_v2/lingbot_va/README.md b/integrations_v2/lingbot_va/README.md index 7cd850d6a..0272c19cb 100644 --- a/integrations_v2/lingbot_va/README.md +++ b/integrations_v2/lingbot_va/README.md @@ -30,6 +30,18 @@ overrides are preserved; fixed model output properties are validated. Model loading is lazy on the model thread, and the finite loop emits one complete rollout before reporting finished. +`app.py` owns argument parsing, validation, and the application/session +contract. `session.py` separately owns the session, finite model loop, lazy +engine lifecycle, and output validation. The loop intentionally ignores live +user events: this adapter wraps the fixed prompt-plus-three-images I2AV +generator, not upstream LingBot-VA's separate closed-loop RoboTwin evaluator. + +This application does not map to the shared text-to-video adapter. Its inputs +include three camera observations, its result includes robot actions, and its +engine completes all chunks before destructive denoising-state teardown and +deferred video decode. It still reuses the common V2 interfaces and generic +MP4, metrics, and typed tensor-artifact sinks. + `--input-image-dir` is required because no camera images are bundled. Use `-- --help` after the application slug for checkpoint, input, compilation, offload, seed, guidance, inference-step, and scheduler-shift overrides. The diff --git a/integrations_v2/lingbot_va/lingbot_va_v2/app.py b/integrations_v2/lingbot_va/lingbot_va_v2/app.py index b40a0e3e5..b85888a26 100644 --- a/integrations_v2/lingbot_va/lingbot_va_v2/app.py +++ b/integrations_v2/lingbot_va/lingbot_va_v2/app.py @@ -18,10 +18,9 @@ from __future__ import annotations import argparse -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass, replace +from collections.abc import Sequence +from dataclasses import replace from pathlib import Path -from typing import Protocol from lingbot_va._loaders import validate_checkpoint_root from lingbot_va.constants import ( @@ -30,9 +29,7 @@ ROBOTWIN_ACTION_DIM, ROBOTWIN_ACTION_GUIDANCE_SCALE, ROBOTWIN_ACTION_INFERENCE_STEPS, - ROBOTWIN_ACTION_PER_FRAME, ROBOTWIN_ACTION_SNR_SHIFT, - ROBOTWIN_FRAME_CHUNK_SIZE, ROBOTWIN_GUIDANCE_SCALE, ROBOTWIN_HEIGHT, ROBOTWIN_SNR_SHIFT, @@ -43,155 +40,24 @@ from lingbot_va.engine import ( LingbotVAEngine, LingbotVAEngineConfig, - LingbotVAEngineOutput, - expected_output_shape, validate_device, validate_input_images, ) from lingbot_va.utils import resolve_prompt from flashdreams.api_v2.application import IApplication -from flashdreams.api_v2.loop import IModelLoop from flashdreams.api_v2.session import ISession from flashdreams.runtime_v2.session_desc import ( BackpressureMode, PresentationMode, SessionDesc, ) -from flashdreams.runtime_v2.step_result import StepResult -from flashdreams.runtime_v2.tensor_artifact import ( - TensorArtifactOutput, - TensorArtifactSchema, -) -from flashdreams.runtime_v2.user_input_events import UserInputEvents from flashdreams.runtime_v2.video_tensor import VideoTensorLayout +from lingbot_va_v2.session import ACTIONS_SCHEMA, EngineFactory, LingbotVASession _FRAMES_PER_SECOND = 10 """Native Robotwin video playback rate.""" -ACTIONS_SCHEMA = TensorArtifactSchema( - name="actions", - dimension_names=("step", "channel"), - concatenate_axis=0, -) -"""Generic tensor artifact schema for denormalized Robotwin actions.""" - - -class LingbotVAEngineLike(Protocol): - """Minimal engine boundary used by the V2 adapter and CPU stand-ins.""" - - def run(self) -> LingbotVAEngineOutput: - """Generate one complete rollout.""" - ... - - def close(self) -> None: - """Release partially or fully initialized model state.""" - ... - - -EngineFactory = Callable[[LingbotVAEngineConfig], LingbotVAEngineLike] -"""Create a session-owned engine from immutable application config.""" - - -@dataclass(slots=True) -class LingbotVAModelState: - """Mutable state owned exclusively by the model loop.""" - - config: LingbotVAEngineConfig - session_desc: SessionDesc - engine_factory: EngineFactory - engine: LingbotVAEngineLike | None = None - generated: bool = False - - -class LingbotVAModelLoop(IModelLoop[LingbotVAModelState]): - """Generate one complete video/action rollout in one honest model step.""" - - def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: - del events - if self.state.generated: - raise RuntimeError("LingBot-VA has already generated this rollout.") - if self.state.engine is None: - self.state.engine = self.state.engine_factory(self.state.config) - output = self.state.engine.run() - _validate_engine_output(self.state.config, output) - self.state.generated = True - return [ - StepResult( - step_index=step_index, - output=output.video, - frame_count=output.video.shape[0], - output_layout=self.state.session_desc.output_layout, - metrics=dict(output.metrics), - tensor_artifacts=( - TensorArtifactOutput( - schema=ACTIONS_SCHEMA, - tensor=output.actions, - ), - ), - ) - ] - - def is_finished(self) -> bool: - return self.state.generated - - def reset(self) -> None: - """Discard the destructive engine; the next step creates a new one.""" - self.close() - self.state.generated = False - - def close(self) -> None: - """Idempotently close the session-owned engine.""" - engine = self.state.engine - self.state.engine = None - if engine is not None: - engine.close() - - -class LingbotVASession(ISession): - """Own one isolated, resettable LingBot-VA rollout.""" - - def __init__( - self, - config: LingbotVAEngineConfig, - session_desc: SessionDesc, - engine_factory: EngineFactory, - ) -> None: - """ - Args: - config: Immutable model and input settings. - session_desc: Canonical Robotwin output description. - engine_factory: Factory used lazily on the model thread. - """ - self._config = config - self._session_desc = session_desc - self._engine_factory = engine_factory - self._model_loop: LingbotVAModelLoop | None = None - - def init(self) -> None: - """Register one finite model loop; model loading remains lazy.""" - if self._model_loop is not None: - raise RuntimeError("LingbotVASession.init() may only run once.") - model_loop = self.register_model_loop( - LingbotVAModelLoop, - state=LingbotVAModelState( - config=self._config, - session_desc=self._session_desc, - engine_factory=self._engine_factory, - ), - ) - assert isinstance(model_loop, LingbotVAModelLoop) - self._model_loop = model_loop - - @property - def session_desc(self) -> SessionDesc: - return self._session_desc - - def close(self) -> None: - """Close a loop even when the runtime never started it.""" - if self._model_loop is not None: - self._model_loop.close() - class LingbotVAApplication(IApplication): """Parse LingBot settings and create session-owned one-run engines.""" @@ -376,49 +242,12 @@ def _validate_checkpoint_reference(checkpoint_root: str | Path) -> None: ) -def _validate_engine_output( - config: LingbotVAEngineConfig, - output: LingbotVAEngineOutput, -) -> None: - """Keep incorrect model shapes from reaching generic runtime sinks.""" - expected_video = expected_output_shape(config) - if tuple(output.video.shape) != expected_video: - raise ValueError( - f"LingBot-VA engine returned video shape {tuple(output.video.shape)}; " - f"expected {expected_video}." - ) - expected_action_shape = ( - config.num_chunks * ROBOTWIN_FRAME_CHUNK_SIZE * ROBOTWIN_ACTION_PER_FRAME, - len(ROBOTWIN_USED_ACTION_CHANNEL_IDS), - ) - if tuple(output.actions.shape) != expected_action_shape: - raise ValueError( - f"LingBot-VA engine returned action shape {tuple(output.actions.shape)}; " - f"expected {expected_action_shape}." - ) - if not output.video.is_floating_point() or not output.actions.is_floating_point(): - raise TypeError("LingBot-VA video and action outputs must be floating point.") - _validate_metrics(output.metrics) - - -def _validate_metrics(metrics: Mapping[str, float]) -> None: - """Require numeric model metrics before constructing a StepResult.""" - if any( - isinstance(value, bool) or not isinstance(value, (int, float)) - for value in metrics.values() - ): - raise TypeError("LingBot-VA engine metrics must be numeric.") - - def create_app() -> IApplication: """Return a new uninitialized LingBot-VA V2 application.""" return LingbotVAApplication() __all__ = [ - "ACTIONS_SCHEMA", "LingbotVAApplication", - "LingbotVAModelLoop", - "LingbotVASession", "create_app", ] diff --git a/integrations_v2/lingbot_va/lingbot_va_v2/session.py b/integrations_v2/lingbot_va/lingbot_va_v2/session.py new file mode 100644 index 000000000..e2af56350 --- /dev/null +++ b/integrations_v2/lingbot_va/lingbot_va_v2/session.py @@ -0,0 +1,220 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Finite V2 session adapter for LingBot-VA Robotwin I2AV rollouts.""" + +from __future__ import annotations + +from collections.abc import Callable, Mapping +from dataclasses import dataclass +from typing import Protocol + +from lingbot_va.constants import ( + ROBOTWIN_ACTION_PER_FRAME, + ROBOTWIN_FRAME_CHUNK_SIZE, + ROBOTWIN_USED_ACTION_CHANNEL_IDS, +) +from lingbot_va.engine import ( + LingbotVAEngineConfig, + LingbotVAEngineOutput, + expected_output_shape, +) + +from flashdreams.api_v2.loop import IModelLoop +from flashdreams.api_v2.session import ISession +from flashdreams.runtime_v2.session_desc import SessionDesc +from flashdreams.runtime_v2.step_result import StepResult +from flashdreams.runtime_v2.tensor_artifact import ( + TensorArtifactOutput, + TensorArtifactSchema, +) +from flashdreams.runtime_v2.user_input_events import UserInputEvents + +ACTIONS_SCHEMA = TensorArtifactSchema( + name="actions", + dimension_names=("step", "channel"), + concatenate_axis=0, +) +"""Tensor artifact schema for denormalized Robotwin actions.""" + + +class LingbotVAEngineLike(Protocol): + """Minimal engine boundary used by the V2 adapter and CPU stand-ins.""" + + def run(self) -> LingbotVAEngineOutput: + """Generate one complete fixed-input rollout.""" + ... + + def close(self) -> None: + """Release partially or fully initialized model state.""" + ... + + +EngineFactory = Callable[[LingbotVAEngineConfig], LingbotVAEngineLike] +"""Create a session-owned engine from immutable application config.""" + + +@dataclass(slots=True) +class LingbotVAModelState: + """Mutable state owned exclusively by the model loop.""" + + config: LingbotVAEngineConfig + """Model, checkpoint, and fixed input settings.""" + + session_desc: SessionDesc + """Canonical output contract for the session.""" + + engine_factory: EngineFactory + """Factory used to construct the destructive one-run engine.""" + + engine: LingbotVAEngineLike | None = None + """Session-owned engine, created lazily on the model thread.""" + + generated: bool = False + """Whether this finite session has produced its only result.""" + + +class LingbotVAModelLoop(IModelLoop[LingbotVAModelState]): + """Generate one complete fixed-input video/action rollout in one step.""" + + def step(self, step_index: int, events: UserInputEvents) -> list[StepResult]: + """Run the offline rollout; live user events are outside this adapter's scope.""" + del events + if self.state.generated: + raise RuntimeError("LingBot-VA has already generated this rollout.") + if self.state.engine is None: + self.state.engine = self.state.engine_factory(self.state.config) + output = self.state.engine.run() + _validate_engine_output(self.state.config, output) + self.state.generated = True + return [ + StepResult( + step_index=step_index, + output=output.video, + frame_count=output.video.shape[0], + output_layout=self.state.session_desc.output_layout, + metrics=dict(output.metrics), + tensor_artifacts=( + TensorArtifactOutput( + schema=ACTIONS_SCHEMA, + tensor=output.actions, + ), + ), + ) + ] + + def is_finished(self) -> bool: + """Return whether the only rollout has completed.""" + return self.state.generated + + def reset(self) -> None: + """Discard the destructive engine; the next step creates a new one.""" + self.close() + self.state.generated = False + + def close(self) -> None: + """Idempotently close the session-owned engine.""" + engine = self.state.engine + self.state.engine = None + if engine is not None: + engine.close() + + +class LingbotVASession(ISession): + """Own one isolated, resettable LingBot-VA rollout.""" + + def __init__( + self, + config: LingbotVAEngineConfig, + session_desc: SessionDesc, + engine_factory: EngineFactory, + ) -> None: + """ + Args: + config: Immutable model and input settings. + session_desc: Canonical Robotwin output description. + engine_factory: Factory used lazily on the model thread. + """ + self._config = config + self._session_desc = session_desc + self._engine_factory = engine_factory + self._model_loop: LingbotVAModelLoop | None = None + + def init(self) -> None: + """Register one finite model loop; model loading remains lazy.""" + if self._model_loop is not None: + raise RuntimeError("LingbotVASession.init() may only run once.") + model_loop = self.register_model_loop( + LingbotVAModelLoop, + state=LingbotVAModelState( + config=self._config, + session_desc=self._session_desc, + engine_factory=self._engine_factory, + ), + ) + assert isinstance(model_loop, LingbotVAModelLoop) + self._model_loop = model_loop + + @property + def session_desc(self) -> SessionDesc: + """Return the immutable output contract for this session.""" + return self._session_desc + + def close(self) -> None: + """Close a loop even when the runtime never started it.""" + if self._model_loop is not None: + self._model_loop.close() + + +def _validate_engine_output( + config: LingbotVAEngineConfig, + output: LingbotVAEngineOutput, +) -> None: + """Keep incorrect model shapes from reaching generic runtime sinks.""" + expected_video = expected_output_shape(config) + if tuple(output.video.shape) != expected_video: + raise ValueError( + f"LingBot-VA engine returned video shape {tuple(output.video.shape)}; " + f"expected {expected_video}." + ) + expected_action_shape = ( + config.num_chunks * ROBOTWIN_FRAME_CHUNK_SIZE * ROBOTWIN_ACTION_PER_FRAME, + len(ROBOTWIN_USED_ACTION_CHANNEL_IDS), + ) + if tuple(output.actions.shape) != expected_action_shape: + raise ValueError( + f"LingBot-VA engine returned action shape {tuple(output.actions.shape)}; " + f"expected {expected_action_shape}." + ) + if not output.video.is_floating_point() or not output.actions.is_floating_point(): + raise TypeError("LingBot-VA video and action outputs must be floating point.") + _validate_metrics(output.metrics) + + +def _validate_metrics(metrics: Mapping[str, float]) -> None: + """Require numeric model metrics before constructing a StepResult.""" + if any( + isinstance(value, bool) or not isinstance(value, (int, float)) + for value in metrics.values() + ): + raise TypeError("LingBot-VA engine metrics must be numeric.") + + +__all__ = [ + "ACTIONS_SCHEMA", + "EngineFactory", + "LingbotVAModelLoop", + "LingbotVASession", +] diff --git a/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py b/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py index 85a247010..f61a6061b 100644 --- a/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py +++ b/integrations_v2/lingbot_va/lingbot_va_v2/tests/test_app.py @@ -26,11 +26,8 @@ import torch from lingbot_va.constants import ROBOTWIN_OBS_CAM_KEYS from lingbot_va.engine import LingbotVAEngineConfig, LingbotVAEngineOutput -from lingbot_va_v2.app import ( - ACTIONS_SCHEMA, - LingbotVAApplication, - create_app, -) +from lingbot_va_v2.app import LingbotVAApplication, create_app +from lingbot_va_v2.session import ACTIONS_SCHEMA from flashdreams.api_v2.application import IApplication from flashdreams.api_v2.client_window import IClientWindow diff --git a/uv.lock b/uv.lock index da13ab0f1..baaf92f96 100644 --- a/uv.lock +++ b/uv.lock @@ -1355,25 +1355,36 @@ dependencies = [ { name = "diffusers" }, { name = "einops" }, { name = "flashdreams" }, + { name = "numpy", version = "2.2.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "numpy", version = "2.4.6", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, { name = "pillow" }, { name = "transformers" }, ] [package.optional-dependencies] dev = [ + { name = "matplotlib", version = "3.10.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "matplotlib", version = "3.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, { name = "pytest" }, ] +visualization = [ + { name = "matplotlib", version = "3.10.9", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version < '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, + { name = "matplotlib", version = "3.11.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.11' or (extra == 'extra-11-flashdreams-dev' and extra == 'group-11-flashdreams-cuda12') or (extra == 'group-11-flashdreams-cuda12' and extra == 'group-11-flashdreams-cuda13')" }, +] [package.metadata] requires-dist = [ { name = "diffusers", specifier = ">=0.38,<0.39" }, { name = "einops" }, { name = "flashdreams", editable = "flashdreams" }, + { name = "matplotlib", marker = "extra == 'dev'", specifier = ">=3.8" }, + { name = "matplotlib", marker = "extra == 'visualization'", specifier = ">=3.8" }, + { name = "numpy", specifier = ">=1.24,<2.5" }, { name = "pillow", specifier = ">=10" }, { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.0" }, { name = "transformers", specifier = ">=5.0,<6" }, ] -provides-extras = ["dev"] +provides-extras = ["dev", "visualization"] [[package]] name = "flashdreams-lingbot-va-v2"