{
  "id": "llm/vllm-cuda-oom-during-prefill",
  "signature": "torch.cuda.OutOfMemoryError: CUDA out of memory. Tried to allocate 2.00 GiB. GPU 0 has a total capacity of 79.15 GiB of which 0 bytes is free.",
  "signature_zh": "torch.cuda.OutOfMemoryError：CUDA 内存不足。尝试分配 2.00 GiB。GPU 0 总容量为 79.15 GiB，其中 0 字节空闲。",
  "regex": "CUDA out of memory.*Tried to allocate.*GiB.*GPU.*has a total capacity",
  "domain": "llm",
  "category": "resource_error",
  "subcategory": null,
  "root_cause": "vLLM's prefill phase (processing the prompt) allocates KV cache memory proportional to prompt length; a very long prompt or high batch size exhausts GPU memory despite available capacity being reported as zero due to fragmentation.",
  "root_cause_type": "generic",
  "root_cause_zh": "vLLM 的预填充阶段（处理提示）分配与提示长度成比例的 KV 缓存内存；非常长的提示或高批次大小耗尽 GPU 内存，尽管由于碎片化报告可用容量为零。",
  "versions": [
    {
      "version": "vllm==0.4.3",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "torch==2.3.0",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "cuda==12.1",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    }
  ],
  "os_specific": {},
  "dead_ends": [
    {
      "action": "",
      "why_fails": "This limits throughput severely and may cause request queuing; the OOM may still occur if a single long prompt exceeds the reduced budget.",
      "fail_rate": 0.75,
      "condition": "",
      "sources": []
    },
    {
      "action": "",
      "why_fails": "vLLM already manages memory; increasing utilization can cause it to allocate more KV cache, worsening fragmentation.",
      "fail_rate": 0.7,
      "condition": "",
      "sources": []
    },
    {
      "action": "",
      "why_fails": "This changes the model entirely and may not be feasible for the application; the OOM can still happen with long prompts on smaller models.",
      "fail_rate": 0.8,
      "condition": "",
      "sources": []
    }
  ],
  "workarounds": [
    {
      "action": "Enable automatic prefix caching (`--enable-prefix-caching`) in vLLM to reuse KV cache for common prompt prefixes, reducing memory for repeated prompts.",
      "success_rate": 0.85,
      "how": "Enable automatic prefix caching (`--enable-prefix-caching`) in vLLM to reuse KV cache for common prompt prefixes, reducing memory for repeated prompts.",
      "condition": "",
      "sources": []
    },
    {
      "action": "Set `max_model_len` to a lower value than the model's maximum (e.g., `--max-model-len 4096`) to limit KV cache allocation per request.",
      "success_rate": 0.9,
      "how": "Set `max_model_len` to a lower value than the model's maximum (e.g., `--max-model-len 4096`) to limit KV cache allocation per request.",
      "condition": "",
      "sources": []
    },
    {
      "action": "Use `--enforce-eager` to disable CUDA graphs, which reduces memory fragmentation at the cost of throughput.",
      "success_rate": 0.8,
      "how": "Use `--enforce-eager` to disable CUDA graphs, which reduces memory fragmentation at the cost of throughput.",
      "condition": "",
      "sources": []
    }
  ],
  "workarounds_zh": [
    "Enable automatic prefix caching (`--enable-prefix-caching`) in vLLM to reuse KV cache for common prompt prefixes, reducing memory for repeated prompts.",
    "Set `max_model_len` to a lower value than the model's maximum (e.g., `--max-model-len 4096`) to limit KV cache allocation per request.",
    "Use `--enforce-eager` to disable CUDA graphs, which reduces memory fragmentation at the cost of throughput."
  ],
  "transition_graph": {
    "leads_to": [],
    "preceded_by": [],
    "frequently_confused_with": []
  },
  "official_doc_url": "https://docs.vllm.ai/en/latest/features/automatic_prefix_caching.html",
  "official_doc_section": null,
  "error_code": "CUDA OOM",
  "verification_tier": "ai_generated",
  "confidence": 0.86,
  "fix_success_rate": 0.8,
  "resolvable": "partial",
  "first_seen": "2024-04-05",
  "last_confirmed": "2024-06-01",
  "last_updated": "2024-06-01",
  "evidence_count": 1,
  "tags": [],
  "locale": "en",
  "aliases": []
}