{
  "id": "huggingface/dataset-memory-mapping-failure",
  "signature": "OSError: Unable to mmap dataset file: [Errno 12] Cannot allocate memory",
  "signature_zh": "OSError：无法对数据集文件进行内存映射：[Errno 12] 无法分配内存",
  "regex": "Unable to mmap dataset file: \\[Errno 12\\] Cannot allocate memory",
  "domain": "huggingface",
  "category": "system_error",
  "subcategory": null,
  "root_cause": "The system does not have enough virtual memory address space to memory-map the dataset file, often due to large file size or system ulimit restrictions.",
  "root_cause_type": "generic",
  "root_cause_zh": "系统没有足够的虚拟内存地址空间来对数据集文件进行内存映射，通常是由于文件过大或系统 ulimit 限制所致。",
  "versions": [
    {
      "version": "datasets>=2.10.0",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "Linux kernel >=5.4",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "Python>=3.8",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    }
  ],
  "os_specific": {},
  "dead_ends": [
    {
      "action": "",
      "why_fails": "The error is about virtual memory address space, not physical RAM. Adding RAM does not fix address space exhaustion.",
      "fail_rate": 0.9,
      "condition": "",
      "sources": []
    },
    {
      "action": "",
      "why_fails": "The error occurs during mmap of the original dataset file, not cache. Disabling caching may cause other issues and does not fix the mmap failure.",
      "fail_rate": 0.8,
      "condition": "",
      "sources": []
    },
    {
      "action": "",
      "why_fails": "This error is about mmap, not mlock. The ulimit for locked memory is unrelated to virtual memory mapping limits.",
      "fail_rate": 0.95,
      "condition": "",
      "sources": []
    }
  ],
  "workarounds": [
    {
      "action": "Reduce the dataset file size by splitting it into smaller shards before loading. Use `datasets.load_dataset('path/to/dataset', split='train[:50%]')` to load half the data.",
      "success_rate": 0.7,
      "how": "Reduce the dataset file size by splitting it into smaller shards before loading. Use `datasets.load_dataset('path/to/dataset', split='train[:50%]')` to load half the data.",
      "condition": "",
      "sources": []
    },
    {
      "action": "Increase the system's vm.max_map_count using `sysctl -w vm.max_map_count=2000000` on Linux. This increases the number of memory-mapped regions allowed.",
      "success_rate": 0.85,
      "how": "Increase the system's vm.max_map_count using `sysctl -w vm.max_map_count=2000000` on Linux. This increases the number of memory-mapped regions allowed.",
      "condition": "",
      "sources": []
    },
    {
      "action": "Use `load_dataset(..., streaming=True)` to avoid memory-mapping the entire file by streaming the data in chunks.",
      "success_rate": 0.9,
      "how": "Use `load_dataset(..., streaming=True)` to avoid memory-mapping the entire file by streaming the data in chunks.",
      "condition": "",
      "sources": []
    }
  ],
  "workarounds_zh": [
    "通过将数据集文件分割成更小的分片来减小文件大小，然后再加载。使用 `datasets.load_dataset('path/to/dataset', split='train[:50%]')` 加载一半数据。",
    "在Linux上使用 `sysctl -w vm.max_map_count=2000000` 增加系统的 vm.max_map_count。这增加了允许的内存映射区域数量。",
    "使用 `load_dataset(..., streaming=True)` 通过流式传输数据块来避免对整个文件进行内存映射。"
  ],
  "transition_graph": {
    "leads_to": [],
    "preceded_by": [],
    "frequently_confused_with": []
  },
  "official_doc_url": "https://huggingface.co/docs/datasets/v2.14.0/en/loading#memory-mapping",
  "official_doc_section": null,
  "error_code": "ENOMEM",
  "verification_tier": "ai_generated",
  "confidence": 0.85,
  "fix_success_rate": 0.75,
  "resolvable": "partial",
  "first_seen": "2023-08-15",
  "last_confirmed": "2024-06-01",
  "last_updated": "2024-06-01",
  "evidence_count": 1,
  "tags": [],
  "locale": "en",
  "aliases": []
}