{
  "id": "huggingface/model-offloaded-to-cpu",
  "signature": "RuntimeError: expected all tensors to be on the same device, but model parameters have been offloaded to CPU",
  "signature_zh": "RuntimeError：期望所有张量在相同设备上，但模型参数已被卸载到CPU",
  "regex": "RuntimeError: expected all tensors to be on the same device, but model parameters have been offloaded to CPU",
  "domain": "huggingface",
  "category": "runtime_error",
  "subcategory": null,
  "root_cause": "Model was loaded with device_map='auto' or similar offloading, but an operation (e.g., forward pass) is being performed on a different device without proper device placement.",
  "root_cause_type": "generic",
  "root_cause_zh": "模型使用device_map='auto'或类似卸载策略加载，但前向传播等操作在没有正确设备放置的情况下在另一个设备上执行。",
  "versions": [
    {
      "version": "transformers>=4.36.0",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "accelerate>=0.25.0",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "PyTorch>=2.1.0",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    }
  ],
  "os_specific": {},
  "dead_ends": [
    {
      "action": "Manually moving model to GPU with model.to('cuda') after loading with device_map",
      "why_fails": "device_map='auto' already handles placement; model.to() overrides it and may cause offloaded layers to be lost or cause memory issues on large models.",
      "fail_rate": 0.7,
      "condition": "",
      "sources": []
    },
    {
      "action": "Setting device_map=None and manually placing model on GPU",
      "why_fails": "For very large models that require offloading, disabling device_map may cause OOM because the entire model won't fit on GPU.",
      "fail_rate": 0.85,
      "condition": "",
      "sources": []
    }
  ],
  "workarounds": [
    {
      "action": "Ensure all operations use the same device context. Use accelerate's context manager: from accelerate import dispatch_model; dispatch_model(model, device_map='auto') and then run inference within a with torch.no_grad() block on the same device.",
      "success_rate": 0.85,
      "how": "Ensure all operations use the same device context. Use accelerate's context manager: from accelerate import dispatch_model; dispatch_model(model, device_map='auto') and then run inference within a with torch.no_grad() block on the same device.",
      "condition": "",
      "sources": []
    },
    {
      "action": "Set device_map to a specific device (e.g., device_map='cuda:0') instead of 'auto' to force all layers onto GPU if memory permits.",
      "success_rate": 0.75,
      "how": "Set device_map to a specific device (e.g., device_map='cuda:0') instead of 'auto' to force all layers onto GPU if memory permits.",
      "condition": "",
      "sources": []
    }
  ],
  "workarounds_zh": [
    "Ensure all operations use the same device context. Use accelerate's context manager: from accelerate import dispatch_model; dispatch_model(model, device_map='auto') and then run inference within a with torch.no_grad() block on the same device.",
    "Set device_map to a specific device (e.g., device_map='cuda:0') instead of 'auto' to force all layers onto GPU if memory permits."
  ],
  "transition_graph": {
    "leads_to": [],
    "preceded_by": [],
    "frequently_confused_with": []
  },
  "official_doc_url": "https://huggingface.co/docs/accelerate/v0.28.0/en/usage_guides/big_modeling",
  "official_doc_section": null,
  "error_code": null,
  "verification_tier": "ai_generated",
  "confidence": 0.85,
  "fix_success_rate": 0.8,
  "resolvable": "true",
  "first_seen": "2024-03-15",
  "last_confirmed": "2024-06-01",
  "last_updated": "2024-06-01",
  "evidence_count": 1,
  "tags": [],
  "locale": "en",
  "aliases": []
}