Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion src/diffusers/models/modeling_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@
is_peft_available,
logging,
)
from ..utils.distributed_utils import is_torch_dist_rank_zero
from ..utils.distributed_utils import is_torch_dist_rank_zero, sync_default_rng_seed
from ..utils.hub_utils import PushToHubMixin, load_or_create_model_card, populate_model_card
from ..utils.torch_utils import empty_device_cache
from ._modeling_parallel import (
Expand Down Expand Up @@ -1719,6 +1719,11 @@ def _resolve_parallel_config(
# `config.setup()` records the mesh resolved above onto the config; see `ParallelConfig.setup`.
config.setup(rank, world_size, device, mesh=mesh)
self._parallel_config = config

if mesh is not None:
# Only the ranks of this mesh must draw the same numbers; replicas outside it keep their own seeds.
group = (mesh._flatten() if mesh.ndim > 1 else mesh).get_group()
sync_default_rng_seed(group=group, device=config._device)
return config

def enable_parallelism(
Expand Down
13 changes: 13 additions & 0 deletions src/diffusers/utils/distributed_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,3 +34,16 @@ def is_torch_dist_rank_zero() -> bool:
return dist_module.get_rank() == 0
except (RuntimeError, ValueError):
return True


def sync_default_rng_seed(group=None, device=None) -> None:
"""Reset the default RNGs of every rank in `group` to the seed of the group's first rank.

Under tensor or context parallelism, every rank must draw the same random numbers (initial latents, scheduler
noise), or the sharded activations stop matching and the output is noise. Ranks are reset even when their seeds
already match, because the same seed does not mean the same position in the stream.
Comment thread
sayakpaul marked this conversation as resolved.
Resolved
"""
seeds = [torch.initial_seed()]
src = torch.distributed.get_global_rank(group, 0) if group is not None else 0
torch.distributed.broadcast_object_list(seeds, src=src, group=group, device=device)
torch.manual_seed(seeds[0])
211 changes: 211 additions & 0 deletions tests/models/testing_utils/parallelism.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
)
from diffusers.models.attention_dispatch import AttentionBackendName, _AttentionBackendRegistry
from diffusers.utils import SAFE_WEIGHTS_INDEX_NAME
from diffusers.utils.torch_utils import randn_tensor

from ...testing_utils import (
is_attention,
Expand Down Expand Up @@ -447,6 +448,158 @@ def _tensor_parallel_from_pretrained_worker(
dist.destroy_process_group()


def _parallel_rng_sync_worker(
rank,
world_size,
master_port,
model_class,
checkpoint_dir,
inputs_dict,
main_input_name,
parallelism,
loading,
return_dict,
device_type=torch_device,
):
"""Worker for the default RNG sync that loading with a parallel config does across the ranks of its mesh.

Each rank seeds itself differently, loads the model with `parallelism`, then draws the main input from the default
RNG and runs a forward pass. Rank 0 reports whether every rank drew the same input, along with its own input and
output so the caller can compare against a single-device run on that input.
"""
try:
os.environ["MASTER_ADDR"] = "localhost"
os.environ["MASTER_PORT"] = str(master_port)
os.environ["RANK"] = str(rank)
os.environ["LOCAL_RANK"] = str(rank)
os.environ["WORLD_SIZE"] = str(world_size)

device_config = DEVICE_CONFIG.get(device_type, DEVICE_CONFIG["cuda"])
dist.init_process_group(backend=device_config["backend"], rank=rank, world_size=world_size)

if device_type == "tpu":
# Each spawned process is bound to one chip. Avoid bf16 matmuls to keep the tolerance tight.
device = torch.device("tpu")
torch.set_float32_matmul_precision("highest")
else:
device_config["module"].set_device(rank)
device = torch.device(f"{device_type}:{rank}")

from torch.distributed.tensor import DTensor

if parallelism == "tensor_parallel":
config = TensorParallelConfig(tp_degree=world_size)
elif parallelism == "context_parallel":
config = ContextParallelConfig(ulysses_degree=world_size)
else:
# Every rank is its own one-rank TP group, i.e. an independent replica.
mesh = torch.distributed.device_mesh.init_device_mesh(
device_type, mesh_shape=(world_size, 1), mesh_dim_names=("dp", "tp")
)
config = TensorParallelConfig(mesh=mesh["tp"])

torch.manual_seed(rank)
if loading == "from_pretrained":
model = model_class.from_pretrained(checkpoint_dir, parallel_config=config).to(device)
else:
model = model_class.from_pretrained(checkpoint_dir).to(device)
model.enable_parallelism(config=config)
model.eval()

inputs_on_device = {k: v.to(device) if isinstance(v, torch.Tensor) else v for k, v in inputs_dict.items()}
noise = randn_tensor(
inputs_on_device[main_input_name].shape, device=device, dtype=inputs_on_device[main_input_name].dtype
)
inputs_on_device[main_input_name] = noise
with torch.no_grad():
output = model(**inputs_on_device, return_dict=False)[0]
if isinstance(output, DTensor):
output = output.full_tensor()

gathered_noise = [torch.empty_like(noise) for _ in range(world_size)]
dist.all_gather(gathered_noise, noise)
if rank == 0:
return_dict["status"] = "success"
return_dict["same_noise"] = all(torch.equal(noise, other) for other in gathered_noise)
# Redraw from rank 0's seed, which is what a non-parallel run with that seed would draw.
torch.manual_seed(rank)
seeded_noise = randn_tensor(noise.shape, device=device, dtype=noise.dtype)
return_dict["rank_0_seed_kept"] = torch.equal(noise, seeded_noise)
# Serialise via nested list so the manager dict can transport it across processes.
return_dict["noise"] = noise.float().cpu().tolist()
return_dict["output"] = output.float().cpu().tolist()

except Exception as e:
if rank == 0:
return_dict["status"] = "error"
return_dict["error"] = f"{type(e).__name__}: {e}"
finally:
if dist.is_initialized():
dist.destroy_process_group()


def _check_parallel_rng_sync(
tester, inputs_dict, tmp_path, parallelism, loading, world_size=2, device_type=torch_device, atol=1e-3, rtol=1e-3
):
"""Check that loading `tester.model_class` with `parallelism` syncs the default RNG across the ranks of its mesh.

The ranks start from different seeds and draw the main input after loading. They must all draw the same one,
unless each rank is an independent replica (`"tensor_parallel_submesh"`), and rank 0's output must match a
single-device run on its input. That run uses the device `inputs_dict` is on.
"""
if not torch.distributed.is_available():
pytest.skip("torch.distributed is not available.")

model_class = tester.model_class
plan_name = "_cp_plan" if parallelism == "context_parallel" else "_tp_plan"
if getattr(model_class, plan_name, None) is None:
pytest.skip(f"Model does not define a `{plan_name}`.")

init_dict = tester.get_init_dict()
num_heads = init_dict.get("num_attention_heads")
if parallelism == "tensor_parallel" and num_heads is not None and num_heads % world_size != 0:
pytest.skip(f"`num_attention_heads` ({num_heads}) is not divisible by tp_degree ({world_size}).")

main_input_name = tester.main_input_name
reference_device = inputs_dict[main_input_name].device
model = model_class(**init_dict).eval()
checkpoint_dir = str(tmp_path / "checkpoint")
model.save_pretrained(checkpoint_dir)

return_dict = mp.Manager().dict()
mp.spawn(
_parallel_rng_sync_worker,
args=(
world_size,
_find_free_port(),
model_class,
checkpoint_dir,
{k: v.cpu() if isinstance(v, torch.Tensor) else v for k, v in inputs_dict.items()},
main_input_name,
parallelism,
loading,
return_dict,
device_type,
),
nprocs=world_size,
join=True,
)
assert return_dict.get("status") == "success", (
f"Parallel RNG sync failed: {return_dict.get('error', 'Unknown error')}"
)
# Independent replicas are outside each other's mesh, so they must keep their own seeds.
expect_same_noise = parallelism != "tensor_parallel_submesh"
assert return_dict["same_noise"] == expect_same_noise, (
f"Ranks drew {'different' if expect_same_noise else 'the same'} inputs from the default RNG."
)
assert return_dict["rank_0_seed_kept"], "Rank 0 did not draw the input a non-parallel run with its seed draws."

noise = torch.tensor(return_dict["noise"], dtype=inputs_dict[main_input_name].dtype, device=reference_device)
with torch.no_grad():
reference = model.to(reference_device)(**{**inputs_dict, main_input_name: noise}, return_dict=False)[0]
torch.testing.assert_close(reference.float().cpu(), torch.tensor(return_dict["output"]), atol=atol, rtol=rtol)


@is_tensor_parallel
@require_torch_multi_accelerator
class TensorParallelTesterMixin:
Expand Down Expand Up @@ -559,6 +712,12 @@ def test_tensor_parallel_from_pretrained(self, tmp_path, sharded):
# Sharded matmuls + all-reduce reorder the summation, so allow a small tolerance over the reference.
torch.testing.assert_close(reference, torch.tensor(return_dict["output"]), atol=1e-3, rtol=1e-3)

@pytest.mark.parametrize("loading", ["enable_parallelism", "from_pretrained"])
@pytest.mark.parametrize("parallelism", ["tensor_parallel", "tensor_parallel_submesh"], ids=["mesh", "submesh"])
def test_tensor_parallel_rng_sync(self, tmp_path, parallelism, loading):
"""Loading with a TP config gives every rank of the TP mesh the same default RNG, and only those ranks."""
_check_parallel_rng_sync(self, self.get_dummy_inputs(), tmp_path, parallelism, loading)


@is_tensor_parallel
@require_torch_tpu
Expand Down Expand Up @@ -624,6 +783,36 @@ def test_tensor_parallel_tpu_inference(self, atol=1e-3, rtol=1e-3):
tp_output = torch.tensor(return_dict["output"])
torch.testing.assert_close(ref_output, tp_output, atol=atol, rtol=rtol)

def test_tensor_parallel_tpu_rng_sync(self, tmp_path, atol=2e-2, rtol=2e-2):
"""Enabling TP gives every rank of the TP mesh the same default RNG."""
from torch_tpu._internal.distributed.launchers.singlehost_wrapper import prepare_tpu_environment
from torch_tpu._internal.utils import hardware

world_size = self.tp_world_size
if hardware.get_tpu_device_count() < world_size:
pytest.skip(f"Needs at least {world_size} TPU chips.")

tpu_env = ("TORCH_TPU_TOPOLOGY", "TORCH_TPU_SLICEBUILDER_ADDRESSES")
for key in tpu_env:
os.environ.pop(key, None)
prepare_tpu_environment(world_size)
try:
# Inputs, and so the reference, stay on CPU: touching the TPU here would bind every chip to this process.
_check_parallel_rng_sync(
self,
self.get_dummy_inputs(device="cpu"),
tmp_path,
"tensor_parallel",
"enable_parallelism",
world_size=world_size,
device_type="tpu",
atol=atol,
rtol=rtol,
)
finally:
for key in tpu_env:
os.environ.pop(key, None)


@is_tensor_parallel
@require_torch_neuron
Expand Down Expand Up @@ -677,6 +866,21 @@ def test_tensor_parallel_neuron_inference(self, atol=2e-2, rtol=2e-2):
# Neuron runs matmuls in bf16 internally, so compare with a bf16-level tolerance.
torch.testing.assert_close(ref_output, tp_output, atol=atol, rtol=rtol)

def test_tensor_parallel_neuron_rng_sync(self, tmp_path, atol=2e-2, rtol=2e-2):
"""Enabling TP gives every rank of the TP mesh the same default RNG."""
# Inputs, and so the reference, stay on CPU: touching the Neuron device here would break the spawned workers.
_check_parallel_rng_sync(
self,
self.get_dummy_inputs(device="cpu"),
tmp_path,
"tensor_parallel",
"enable_parallelism",
world_size=self.tp_world_size,
device_type="neuron",
atol=atol,
rtol=rtol,
)


@is_context_parallel
@require_torch_multi_accelerator
Expand Down Expand Up @@ -839,6 +1043,13 @@ def test_context_parallel_custom_mesh(self, cp_type, mesh_shape, mesh_dim_names)
f"Custom mesh context parallel inference failed: {return_dict.get('error', 'Unknown error')}"
)

@pytest.mark.parametrize("loading", ["enable_parallelism", "from_pretrained"])
def test_context_parallel_rng_sync(self, tmp_path, loading):
"""Loading with a CP config gives every rank of the CP mesh the same default RNG."""
_check_parallel_rng_sync(
self, self.get_dummy_inputs(), tmp_path, "context_parallel", loading, atol=1e-4, rtol=1e-4
)


@is_attention
@is_context_parallel
Expand Down
Loading