Repository navigation
Description
Describe the bug
Cosmos3OmniPipeline can enter the audio or action path just because the caller supplies sound_latents or action_latents. The arguments are documented as being consulted only when the corresponding modality is enabled. In text-only generation, passing a tensor for either argument should produce the same video latents as omitting it and leave sound and action unset.
The latent preparation step returns both tensors even when their streams are disabled. The denoising loop then infers which streams to run from whether those returned tensors are present. I can submit a small fix and regression test if this scope looks right.
Reproduction
The following exercises real pipeline, transformer, VAE, and scheduler classes in a small CPU configuration.
import torch
from PIL import Image
from tokenizers import Tokenizer, models, pre_tokenizers
from transformers import PreTrainedTokenizerFast
from diffusers import AutoencoderKLWan, Cosmos3OmniPipeline, Cosmos3OmniTransformer, UniPCMultistepScheduler
from diffusers.utils.torch_utils import randn_tensor
def build_pipeline():
tokenizer = Tokenizer(models.WordLevel({"<unk>": 0, "<eos>": 1, "<|vision_start|>": 2}, unk_token="<unk>"))
tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
tokenizer = PreTrainedTokenizerFast(tokenizer_object=tokenizer, unk_token="<unk>", eos_token="<eos>")
tokenizer.chat_template = "{% for message in messages %}{{ message['content'] }}{% endfor %}"
torch.manual_seed(0)
transformer = Cosmos3OmniTransformer(
head_dim=6, hidden_act="relu2", hidden_size=6, intermediate_size=12, latent_channel=16, latent_patch_size=1,
num_attention_heads=1, num_hidden_layers=1, num_key_value_heads=1, patch_latent_dim=16,
qk_norm_for_text=False, rms_norm_eps=1e-5, rope_axes_dim=[1, 1, 1], vocab_size=3,
)
vae = AutoencoderKLWan(
base_dim=3, z_dim=16, dim_mult=[1, 1, 1, 1], num_res_blocks=1, temperal_downsample=[False, True, True],
)
pipe = Cosmos3OmniPipeline(
transformer=transformer, text_tokenizer=tokenizer, vae=vae, scheduler=UniPCMultistepScheduler(),
enable_safety_checker=False,
)
pipe.set_progress_bar_config(disable=True)
return pipe
torch.set_num_threads(1)
pipe = build_pipeline()
common = dict(
prompt="a dog", negative_prompt="bad quality", height=16, width=16, num_inference_steps=2,
guidance_scale=1.0, output_type="latent", use_system_prompt=False, add_resolution_template=False,
add_duration_template=False,
)
def gen():
return torch.Generator().manual_seed(0)
plain = pipe(**common, num_frames=1, generator=gen())
assert torch.isfinite(plain.video).all()
print(f"Text-only control: shape={tuple(plain.video.shape)}; sound={plain.sound}; action={plain.action}")
for key, value in {"sound_latents": torch.zeros(2, 3), "action_latents": torch.zeros(3, 7)}.items():
try:
actual = pipe(**common, num_frames=1, generator=gen(), **{key: value})
torch.testing.assert_close(actual.video, plain.video)
assert actual.sound is None and actual.action is None
except Exception as exc:
print(f"Inactive {key}: {type(exc).__name__}: {exc}")
else:
print(f"Inactive {key}: matches the text-only control")Logs
Text-only control: shape=(1, 16, 1, 2, 2); sound=None; action=None
Inactive sound_latents: AttributeError: 'Cosmos3OmniTransformer' object has no attribute 'audio_proj_in'
Inactive action_latents: TypeError: 'NoneType' object is not iterable
System Info
Diffusers 0.44.0.dev0 at 8edb6d8, Windows 11 x64, Python 3.12.14, PyTorch 2.11.0+cpu, Transformers 5.20.0. CPU with tiny randomly initialized components.
Activity