{
  "id": "pytorch/fsdp-parameter-mismatch",
  "signature": "RuntimeError: FSDP parameter mismatch across ranks: expected parameter 'layer.0.weight' but got parameter 'module.layer.0.weight'",
  "signature_zh": "运行时错误：FSDP参数在各rank间不匹配：期望参数'layer.0.weight'但得到参数'module.layer.0.weight'",
  "regex": "FSDP parameter mismatch across ranks",
  "domain": "pytorch",
  "category": "runtime_error",
  "subcategory": null,
  "root_cause": "Fully Sharded Data Parallel (FSDP) detected that model parameters have different names across distributed ranks, often due to wrapping the model in an extra module (e.g., nn.DataParallel or DDP) before FSDP, or inconsistent model initialization.",
  "root_cause_type": "generic",
  "root_cause_zh": "全分片数据并行（FSDP）检测到各分布式rank上的模型参数名称不同，通常由于在FSDP之前将模型包装在额外模块中（例如nn.DataParallel或DDP），或模型初始化不一致。",
  "versions": [
    {
      "version": "pytorch>=2.0",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "torchvision>=0.15",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "cuda>=11.7",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "nccl>=2.14",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    }
  ],
  "os_specific": {},
  "dead_ends": [
    {
      "action": "Adding more FSDP wrapping layers to fix the mismatch",
      "why_fails": "Extra wrapping adds more prefixes to parameter names, worsening the mismatch.",
      "fail_rate": 0.95,
      "condition": "",
      "sources": []
    },
    {
      "action": "Ignoring the error and continuing training with torch.distributed.barrier()",
      "why_fails": "Parameter mismatch leads to incorrect gradient synchronization and model corruption.",
      "fail_rate": 1.0,
      "condition": "",
      "sources": []
    }
  ],
  "workarounds": [
    {
      "action": "Remove any extra wrappers (e.g., DataParallel, DDP) before applying FSDP: model = MyModel(); model = FSDP(model) directly, not model = FSDP(DataParallel(model))",
      "success_rate": 0.95,
      "how": "Remove any extra wrappers (e.g., DataParallel, DDP) before applying FSDP: model = MyModel(); model = FSDP(model) directly, not model = FSDP(DataParallel(model))",
      "condition": "",
      "sources": []
    },
    {
      "action": "Ensure consistent model initialization across ranks by using the same random seed and loading the same checkpoint: torch.manual_seed(42); model.load_state_dict(torch.load('checkpoint.pt'))",
      "success_rate": 0.85,
      "how": "Ensure consistent model initialization across ranks by using the same random seed and loading the same checkpoint: torch.manual_seed(42); model.load_state_dict(torch.load('checkpoint.pt'))",
      "condition": "",
      "sources": []
    },
    {
      "action": "Use a flat parameter naming scheme by setting param_init_fn in FSDP to avoid prefixes: FSDP(model, param_init_fn=lambda m: m.to_empty(device=torch.cuda.current_device()))",
      "success_rate": 0.8,
      "how": "Use a flat parameter naming scheme by setting param_init_fn in FSDP to avoid prefixes: FSDP(model, param_init_fn=lambda m: m.to_empty(device=torch.cuda.current_device()))",
      "condition": "",
      "sources": []
    }
  ],
  "workarounds_zh": [
    "Remove any extra wrappers (e.g., DataParallel, DDP) before applying FSDP: model = MyModel(); model = FSDP(model) directly, not model = FSDP(DataParallel(model))",
    "Ensure consistent model initialization across ranks by using the same random seed and loading the same checkpoint: torch.manual_seed(42); model.load_state_dict(torch.load('checkpoint.pt'))",
    "Use a flat parameter naming scheme by setting param_init_fn in FSDP to avoid prefixes: FSDP(model, param_init_fn=lambda m: m.to_empty(device=torch.cuda.current_device()))"
  ],
  "transition_graph": {
    "leads_to": [],
    "preceded_by": [],
    "frequently_confused_with": []
  },
  "official_doc_url": "https://pytorch.org/docs/stable/fsdp.html",
  "official_doc_section": null,
  "error_code": null,
  "verification_tier": "ai_generated",
  "confidence": 0.86,
  "fix_success_rate": 0.9,
  "resolvable": "true",
  "first_seen": "2023-08-12",
  "last_confirmed": "2024-06-01",
  "last_updated": "2024-06-01",
  "evidence_count": 1,
  "tags": [],
  "locale": "en",
  "aliases": []
}