{
  "id": "pytorch/fsdp-param-mismatch",
  "signature": "RuntimeError: FSDP parameter mismatch across ranks. Expected parameter 'model.fc.weight' to have shape [100, 256] but got [100, 128] on rank 2.",
  "signature_zh": "RuntimeError: FSDP 参数在多个 rank 间不匹配。期望参数 'model.fc.weight' 的形状为 [100, 256]，但在 rank 2 上得到 [100, 128]。",
  "regex": "RuntimeError: FSDP parameter mismatch across ranks\\. Expected parameter '[^']+' to have shape \\[\\d+[,\\s]*\\d+\\] but got \\[\\d+[,\\s]*\\d+\\] on rank \\d+",
  "domain": "pytorch",
  "category": "config_error",
  "subcategory": null,
  "root_cause": "Fully Sharded Data Parallel (FSDP) expects identical model architectures across all ranks; a mismatch in layer dimensions indicates inconsistent model initialization or different model files loaded on different processes.",
  "root_cause_type": "generic",
  "root_cause_zh": "完全分片数据并行（FSDP）期望所有 rank 上的模型架构相同；层维度不匹配表示模型初始化不一致或不同进程加载了不同的模型文件。",
  "versions": [
    {
      "version": "torch>=1.12",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "torch>=2.0",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "CUDA 11.x+",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "NCCL 2.10+",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    }
  ],
  "os_specific": {},
  "dead_ends": [
    {
      "action": "Setting FSDP sync_module_states=True without fixing architecture",
      "why_fails": "sync_module_states broadcasts parameters from rank 0 but does not resolve shape mismatches; it will fail with the same error.",
      "fail_rate": 0.95,
      "condition": "",
      "sources": []
    },
    {
      "action": "Using different random seeds on each rank",
      "why_fails": "Random seeds affect weight initialization but not model architecture; shape mismatch persists regardless of seed.",
      "fail_rate": 0.8,
      "condition": "",
      "sources": []
    }
  ],
  "workarounds": [
    {
      "action": "Ensure identical model definition across all ranks by loading the same checkpoint: if rank == 0: torch.save(model.state_dict(), 'init.pt'); torch.distributed.barrier(); model.load_state_dict(torch.load('init.pt', map_location='cpu'))",
      "success_rate": 0.9,
      "how": "Ensure identical model definition across all ranks by loading the same checkpoint: if rank == 0: torch.save(model.state_dict(), 'init.pt'); torch.distributed.barrier(); model.load_state_dict(torch.load('init.pt', map_location='cpu'))",
      "condition": "",
      "sources": []
    },
    {
      "action": "Use torch.distributed.barrier() after model creation to synchronize before FSDP wrapping: model = MyModel(); torch.distributed.barrier(); fsdp_model = FullyShardedDataParallel(model)",
      "success_rate": 0.85,
      "how": "Use torch.distributed.barrier() after model creation to synchronize before FSDP wrapping: model = MyModel(); torch.distributed.barrier(); fsdp_model = FullyShardedDataParallel(model)",
      "condition": "",
      "sources": []
    }
  ],
  "workarounds_zh": [
    "Ensure identical model definition across all ranks by loading the same checkpoint: if rank == 0: torch.save(model.state_dict(), 'init.pt'); torch.distributed.barrier(); model.load_state_dict(torch.load('init.pt', map_location='cpu'))",
    "Use torch.distributed.barrier() after model creation to synchronize before FSDP wrapping: model = MyModel(); torch.distributed.barrier(); fsdp_model = FullyShardedDataParallel(model)"
  ],
  "transition_graph": {
    "leads_to": [],
    "preceded_by": [],
    "frequently_confused_with": []
  },
  "official_doc_url": "https://pytorch.org/docs/stable/fsdp.html",
  "official_doc_section": null,
  "error_code": null,
  "verification_tier": "ai_generated",
  "confidence": 0.87,
  "fix_success_rate": 0.82,
  "resolvable": "true",
  "first_seen": "2023-08-20",
  "last_confirmed": "2024-06-01",
  "last_updated": "2024-06-01",
  "evidence_count": 1,
  "tags": [],
  "locale": "en",
  "aliases": []
}