{
  "id": "data/csv-encoding-utf16-vs-utf8-bom",
  "signature": "CSV file with UTF-16 BOM is misinterpreted as UTF-8, producing garbled column headers",
  "signature_zh": "带有 UTF-16 BOM 的 CSV 文件被误解为 UTF-8，产生乱码列标题",
  "regex": "garbled.*column.*name|UTF-16.*BOM.*CSV|unexpected.*characters.*first.*column",
  "domain": "data",
  "category": "encoding_error",
  "subcategory": null,
  "root_cause": "CSV files saved with UTF-16 encoding (often by Excel on Windows) start with a UTF-16 BOM (0xFFFE) that UTF-8 parsers interpret as two characters, corrupting the first column name.",
  "root_cause_type": "generic",
  "root_cause_zh": "以 UTF-16 编码保存的 CSV 文件（通常由 Windows 上的 Excel 生成）以 UTF-16 BOM（0xFFFE）开头，UTF-8 解析器将其解释为两个字符，从而损坏第一个列名。",
  "versions": [
    {
      "version": "Microsoft Excel 365",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "Python csv module 3.10+",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "Pandas 1.5.0+",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "Apache Commons CSV 1.10.0+",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    }
  ],
  "os_specific": {},
  "dead_ends": [
    {
      "action": "",
      "why_fails": "Notepad may silently change line endings or add extra characters, causing further corruption.",
      "fail_rate": 0.5,
      "condition": "",
      "sources": []
    },
    {
      "action": "",
      "why_fails": "utf-8-sig only handles UTF-8 BOM (0xEFBBBF), not UTF-16 BOM (0xFFFE).",
      "fail_rate": 0.8,
      "condition": "",
      "sources": []
    }
  ],
  "workarounds": [
    {
      "action": "Detect and decode UTF-16 BOM explicitly: `with open('file.csv', 'rb') as f: raw = f.read(); if raw[:2] == b'\\xff\\xfe': text = raw.decode('utf-16'); else: text = raw.decode('utf-8')`",
      "success_rate": 0.95,
      "how": "Detect and decode UTF-16 BOM explicitly: `with open('file.csv', 'rb') as f: raw = f.read(); if raw[:2] == b'\\xff\\xfe': text = raw.decode('utf-16'); else: text = raw.decode('utf-8')`",
      "condition": "",
      "sources": []
    },
    {
      "action": "Use pandas with explicit encoding detection: `import chardet; with open('file.csv', 'rb') as f: enc = chardet.detect(f.read(10000)); df = pd.read_csv('file.csv', encoding=enc['encoding'])`",
      "success_rate": 0.9,
      "how": "Use pandas with explicit encoding detection: `import chardet; with open('file.csv', 'rb') as f: enc = chardet.detect(f.read(10000)); df = pd.read_csv('file.csv', encoding=enc['encoding'])`",
      "condition": "",
      "sources": []
    },
    {
      "action": "Convert the file using `iconv -f UTF-16 -t UTF-8 file.csv > fixed.csv`",
      "success_rate": 0.85,
      "how": "Convert the file using `iconv -f UTF-16 -t UTF-8 file.csv > fixed.csv`",
      "condition": "",
      "sources": []
    }
  ],
  "workarounds_zh": [
    "显式检测并解码 UTF-16 BOM：`with open('file.csv', 'rb') as f: raw = f.read(); if raw[:2] == b'\\xff\\xfe': text = raw.decode('utf-16'); else: text = raw.decode('utf-8')`",
    "使用 pandas 和显式编码检测：`import chardet; with open('file.csv', 'rb') as f: enc = chardet.detect(f.read(10000)); df = pd.read_csv('file.csv', encoding=enc['encoding'])`",
    "使用 `iconv -f UTF-16 -t UTF-8 file.csv > fixed.csv` 转换文件"
  ],
  "transition_graph": {
    "leads_to": [],
    "preceded_by": [],
    "frequently_confused_with": []
  },
  "official_doc_url": "https://docs.python.org/3/library/csv.html",
  "official_doc_section": null,
  "error_code": null,
  "verification_tier": "ai_generated",
  "confidence": 0.84,
  "fix_success_rate": 0.9,
  "resolvable": "true",
  "first_seen": "2024-02-14",
  "last_confirmed": "2024-06-01",
  "last_updated": "2024-06-01",
  "evidence_count": 1,
  "tags": [],
  "locale": "en",
  "aliases": []
}