{
  "id": "pytorch/nccl-communicator-aborted-watchdog-timeout",
  "signature": "RuntimeError: NCCL communicator was aborted on rank 2. Original reason for failure was: watchdog callback timed out.",
  "signature_zh": "RuntimeError: NCCL 通信器在 rank 2 上被中止。原始失败原因：看门狗回调超时。",
  "regex": "NCCL communicator was aborted on rank \\d+.*watchdog callback timed out",
  "domain": "pytorch",
  "category": "network_error",
  "subcategory": null,
  "root_cause": "A NCCL watchdog callback timed out, indicating that a collective operation (e.g., allreduce) hung for too long, often due to network congestion, GPU compute imbalance, or a single slow node.",
  "root_cause_type": "generic",
  "root_cause_zh": "NCCL 看门狗回调超时，表明一个集合操作（例如 allreduce）挂起时间过长，通常是由于网络拥塞、GPU 计算不平衡或单个慢节点导致。",
  "versions": [
    {
      "version": "torch>=1.13",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "NCCL 2.16",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "CUDA 11.8",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "Slurm 23.02",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    }
  ],
  "os_specific": {},
  "dead_ends": [
    {
      "action": "Increasing NCCL_TIMEOUT environment variable to a very large value",
      "why_fails": "This only delays the timeout but does not fix the root cause (e.g., network congestion or slow node); the job will eventually hang or fail later.",
      "fail_rate": 0.8,
      "condition": "",
      "sources": []
    },
    {
      "action": "Restarting the job with the same number of GPUs",
      "why_fails": "If the underlying issue (e.g., network topology or GPU imbalance) is not addressed, the same failure will recur.",
      "fail_rate": 0.9,
      "condition": "",
      "sources": []
    },
    {
      "action": "Disabling NCCL watchdog with NCCL_DEBUG=WARN",
      "why_fails": "Disabling the watchdog hides the error but does not prevent the hang; the job will stall indefinitely without feedback.",
      "fail_rate": 0.95,
      "condition": "",
      "sources": []
    }
  ],
  "workarounds": [
    {
      "action": "Set environment variables to improve NCCL stability:\n\nexport NCCL_IB_TIMEOUT=22\nexport NCCL_SOCKET_IFNAME=eth0\nexport NCCL_DEBUG=INFO\n\nThen rerun the job to identify the slow rank.",
      "success_rate": 0.85,
      "how": "Set environment variables to improve NCCL stability:\n\nexport NCCL_IB_TIMEOUT=22\nexport NCCL_SOCKET_IFNAME=eth0\nexport NCCL_DEBUG=INFO\n\nThen rerun the job to identify the slow rank.",
      "condition": "",
      "sources": []
    },
    {
      "action": "Add gradient accumulation to reduce communication frequency:\n\naccumulation_steps = 4\nfor i, (inputs, labels) in enumerate(dataloader):\n    outputs = model(inputs)\n    loss = criterion(outputs, labels)\n    loss = loss / accumulation_steps\n    loss.backward()\n    if (i + 1) % accumulation_steps == 0:\n        optimizer.step()\n        optimizer.zero_grad()",
      "success_rate": 0.8,
      "how": "Add gradient accumulation to reduce communication frequency:\n\naccumulation_steps = 4\nfor i, (inputs, labels) in enumerate(dataloader):\n    outputs = model(inputs)\n    loss = criterion(outputs, labels)\n    loss = loss / accumulation_steps\n    loss.backward()\n    if (i + 1) % accumulation_steps == 0:\n        optimizer.step()\n        optimizer.zero_grad()",
      "condition": "",
      "sources": []
    },
    {
      "action": "Use torch.nn.parallel.DistributedDataParallel with find_unused_parameters=True and gradient_as_bucket_view=True to reduce communication overhead.",
      "success_rate": 0.75,
      "how": "Use torch.nn.parallel.DistributedDataParallel with find_unused_parameters=True and gradient_as_bucket_view=True to reduce communication overhead.",
      "condition": "",
      "sources": []
    }
  ],
  "workarounds_zh": [
    "Set environment variables to improve NCCL stability:\n\nexport NCCL_IB_TIMEOUT=22\nexport NCCL_SOCKET_IFNAME=eth0\nexport NCCL_DEBUG=INFO\n\nThen rerun the job to identify the slow rank.",
    "Add gradient accumulation to reduce communication frequency:\n\naccumulation_steps = 4\nfor i, (inputs, labels) in enumerate(dataloader):\n    outputs = model(inputs)\n    loss = criterion(outputs, labels)\n    loss = loss / accumulation_steps\n    loss.backward()\n    if (i + 1) % accumulation_steps == 0:\n        optimizer.step()\n        optimizer.zero_grad()",
    "Use torch.nn.parallel.DistributedDataParallel with find_unused_parameters=True and gradient_as_bucket_view=True to reduce communication overhead."
  ],
  "transition_graph": {
    "leads_to": [],
    "preceded_by": [],
    "frequently_confused_with": []
  },
  "official_doc_url": "https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/troubleshooting.html",
  "official_doc_section": null,
  "error_code": null,
  "verification_tier": "ai_generated",
  "confidence": 0.86,
  "fix_success_rate": 0.78,
  "resolvable": "true",
  "first_seen": "2023-07-01",
  "last_confirmed": "2024-06-01",
  "last_updated": "2024-06-01",
  "evidence_count": 1,
  "tags": [],
  "locale": "en",
  "aliases": []
}