{
  "id": "data/csv-utf8-bom-quote-mismatch",
  "signature": "CSV parser fails to recognize quoted fields when file starts with UTF-8 BOM",
  "signature_zh": "当文件以 UTF-8 BOM 开头时，CSV 解析器无法识别带引号的字段",
  "regex": "(?i)(BOM.*csv|UTF-8 BOM.*quote|first field.*BOM)",
  "domain": "data",
  "category": "encoding_error",
  "subcategory": null,
  "root_cause": "UTF-8 BOM (0xEF BB BF) at file start is not stripped by many CSV parsers, causing the first field to include the BOM and breaking quote detection if the field is quoted.",
  "root_cause_type": "generic",
  "root_cause_zh": "许多 CSV 解析器不会移除文件开头的 UTF-8 BOM（0xEF BB BF），导致第一个字段包含 BOM，如果该字段带引号则会破坏引号检测。",
  "versions": [
    {
      "version": "Python 3.11 csv module",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "Apache Commons CSV 1.10.0",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "Pandas 2.1.0",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    }
  ],
  "os_specific": {},
  "dead_ends": [
    {
      "action": "Opening file in 'utf-8-sig' encoding in Python",
      "why_fails": "While this removes BOM for text reading, the csv.reader still processes the BOM as part of the first field if not handled before parsing.",
      "fail_rate": 0.5,
      "condition": "",
      "sources": []
    },
    {
      "action": "Manually skipping first byte with file.seek(3)",
      "why_fails": "This only works if BOM is exactly 3 bytes; some editors may add extra bytes, and it breaks for files without BOM.",
      "fail_rate": 0.6,
      "condition": "",
      "sources": []
    }
  ],
  "workarounds": [
    {
      "action": "Strip BOM before CSV parsing: with open('file.csv', 'r', encoding='utf-8-sig') as f: content = f.read().lstrip('\\ufeff'); reader = csv.reader(StringIO(content))",
      "success_rate": 0.95,
      "how": "Strip BOM before CSV parsing: with open('file.csv', 'r', encoding='utf-8-sig') as f: content = f.read().lstrip('\\ufeff'); reader = csv.reader(StringIO(content))",
      "condition": "",
      "sources": []
    },
    {
      "action": "Use pandas with encoding='utf-8-sig': pd.read_csv('file.csv', encoding='utf-8-sig')",
      "success_rate": 0.9,
      "how": "Use pandas with encoding='utf-8-sig': pd.read_csv('file.csv', encoding='utf-8-sig')",
      "condition": "",
      "sources": []
    }
  ],
  "workarounds_zh": [
    "Strip BOM before CSV parsing: with open('file.csv', 'r', encoding='utf-8-sig') as f: content = f.read().lstrip('\\ufeff'); reader = csv.reader(StringIO(content))",
    "Use pandas with encoding='utf-8-sig': pd.read_csv('file.csv', encoding='utf-8-sig')"
  ],
  "transition_graph": {
    "leads_to": [],
    "preceded_by": [],
    "frequently_confused_with": []
  },
  "official_doc_url": "https://docs.python.org/3/library/csv.html",
  "official_doc_section": null,
  "error_code": "csv.Error: field larger than field limit (131072)",
  "verification_tier": "ai_generated",
  "confidence": 0.87,
  "fix_success_rate": 0.9,
  "resolvable": "true",
  "first_seen": "2024-02-14",
  "last_confirmed": "2024-06-01",
  "last_updated": "2024-06-01",
  "evidence_count": 1,
  "tags": [],
  "locale": "en",
  "aliases": []
}