{
  "id": "python/pandas-csv-unicode-decode",
  "signature": "UnicodeDecodeError: 'utf-8' codec can't decode byte 0xff in position 0: invalid start byte",
  "signature_zh": "UnicodeDecodeError: 'utf-8' 编解码器无法解码位置 0 的字节 0xff：无效起始字节",
  "regex": "UnicodeDecodeError:\\ 'utf\\-8'\\ codec\\ can't\\ decode\\ byte\\ 0xff\\ in\\ position\\ 0:\\ invalid\\ start\\ byte",
  "domain": "python",
  "category": "encoding_error",
  "subcategory": null,
  "root_cause": "pd.read_csv defaults to UTF-8, but the file is encoded in UTF-16 (BOM 0xFF 0xFE), Latin-1, or CP1252. 0xff at position 0 is the UTF-16 LE BOM.",
  "root_cause_type": "generic",
  "root_cause_zh": "pd.read_csv 默认使用 UTF-8，但文件编码为 UTF-16（BOM 0xFF 0xFE）、Latin-1 或 CP1252。位置 0 的 0xff 是 UTF-16 LE BOM。",
  "versions": [
    {
      "version": "3.x",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    }
  ],
  "os_specific": {},
  "dead_ends": [
    {
      "action": "",
      "why_fails": "Silently corrupts non-ASCII text",
      "fail_rate": 0.85,
      "condition": "",
      "sources": []
    },
    {
      "action": "",
      "why_fails": "Same decode error on open()",
      "fail_rate": 0.9,
      "condition": "",
      "sources": []
    }
  ],
  "workarounds": [
    {
      "action": "",
      "success_rate": 0.95,
      "how": "pd.read_csv(path, encoding='utf-16')  # or 'latin-1' / 'cp1252'",
      "condition": "",
      "sources": []
    },
    {
      "action": "",
      "success_rate": 0.9,
      "how": "Detect with `import chardet; enc = chardet.detect(open(path,'rb').read(10000))['encoding']`",
      "condition": "",
      "sources": []
    }
  ],
  "workarounds_zh": [],
  "transition_graph": {
    "leads_to": [],
    "preceded_by": [],
    "frequently_confused_with": []
  },
  "official_doc_url": null,
  "official_doc_section": null,
  "error_code": null,
  "verification_tier": "ai_generated",
  "confidence": 0.9,
  "fix_success_rate": 0.8,
  "resolvable": "true",
  "first_seen": "2024-02-19",
  "last_confirmed": "2025-01-01",
  "last_updated": "2025-01-01",
  "evidence_count": 0,
  "tags": [],
  "locale": "en",
  "aliases": []
}