{
  "id": "llm/token-count-mismatch-streaming",
  "signature": "Warning: Token count mismatch — prompt tokens (4500) + completion tokens (1200) = 5700, but API reports total_tokens=5800",
  "signature_zh": "警告：Token计数不匹配——提示Token(4500)+补全Token(1200)=5700，但API报告total_tokens=5800",
  "regex": "Warning: Token count mismatch — prompt tokens \\(\\d+\\) \\+ completion tokens \\(\\d+\\) = \\d+, but API reports total_tokens=\\d+",
  "domain": "llm",
  "category": "data_error",
  "subcategory": null,
  "root_cause": "The tiktoken tokenizer counts tokens differently than the OpenAI API's internal tokenizer, especially for multi-byte characters, special tokens, or streaming overhead tokens like role markers and stop sequences.",
  "root_cause_type": "generic",
  "root_cause_zh": "tiktoken分词器对Token的计数方式与OpenAI API内部分词器不同，尤其是在多字节字符、特殊Token或流式传输的额外Token（如角色标记和停止序列）方面存在差异。",
  "versions": [
    {
      "version": "tiktoken==0.7.0",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "openai==1.30.0",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    },
    {
      "version": "python==3.11",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    }
  ],
  "os_specific": {},
  "dead_ends": [
    {
      "action": "",
      "why_fails": "The discrepancy is inherent because the API uses a different tokenization pipeline that includes service-side tokens (e.g., role markers, stop sequences) not counted by tiktoken.",
      "fail_rate": 0.9,
      "condition": "",
      "sources": []
    },
    {
      "action": "",
      "why_fails": "The mismatch varies with prompt structure and completion length; a fixed fudge factor is unreliable and may cause budget overruns or early truncation.",
      "fail_rate": 0.7,
      "condition": "",
      "sources": []
    },
    {
      "action": "",
      "why_fails": "While this avoids the mismatch warning, it doesn't fix the root cause; the token count reported by the API still uses its internal tokenizer, and the mismatch persists in non-streaming mode too.",
      "fail_rate": 0.5,
      "condition": "",
      "sources": []
    }
  ],
  "workarounds": [
    {
      "action": "Use the API's reported `total_tokens` as the source of truth for billing and truncation, and only use tiktoken for approximate pre-flight checks: `response = client.chat.completions.create(model='gpt-4', messages=messages, stream=True); total_tokens = response.usage.total_tokens if hasattr(response, 'usage') else None`",
      "success_rate": 0.95,
      "how": "Use the API's reported `total_tokens` as the source of truth for billing and truncation, and only use tiktoken for approximate pre-flight checks: `response = client.chat.completions.create(model='gpt-4', messages=messages, stream=True); total_tokens = response.usage.total_tokens if hasattr(response, 'usage') else None`",
      "condition": "",
      "sources": []
    },
    {
      "action": "Implement a buffer of 10% when using tiktoken to estimate context window usage: `max_tokens_allowed = int(model_max_tokens * 0.9) - tiktoken_count(prompt)`",
      "success_rate": 0.8,
      "how": "Implement a buffer of 10% when using tiktoken to estimate context window usage: `max_tokens_allowed = int(model_max_tokens * 0.9) - tiktoken_count(prompt)`",
      "condition": "",
      "sources": []
    }
  ],
  "workarounds_zh": [
    "Use the API's reported `total_tokens` as the source of truth for billing and truncation, and only use tiktoken for approximate pre-flight checks: `response = client.chat.completions.create(model='gpt-4', messages=messages, stream=True); total_tokens = response.usage.total_tokens if hasattr(response, 'usage') else None`",
    "Implement a buffer of 10% when using tiktoken to estimate context window usage: `max_tokens_allowed = int(model_max_tokens * 0.9) - tiktoken_count(prompt)`"
  ],
  "transition_graph": {
    "leads_to": [],
    "preceded_by": [],
    "frequently_confused_with": []
  },
  "official_doc_url": "https://cookbook.openai.com/examples/how_to_count_tokens_with_tiktoken",
  "official_doc_section": null,
  "error_code": null,
  "verification_tier": "ai_generated",
  "confidence": 0.82,
  "fix_success_rate": 0.75,
  "resolvable": "true",
  "first_seen": "2024-02-20",
  "last_confirmed": "2024-06-01",
  "last_updated": "2024-06-01",
  "evidence_count": 1,
  "tags": [],
  "locale": "en",
  "aliases": []
}