{
  "id": "huggingface/tokenizer-slow-fast-mismatch",
  "signature": "UserWarning: The tokenizer is a 'PreTrainedTokenizerFast' but the model was loaded with a slow tokenizer. This may cause encoding inconsistencies.",
  "signature_zh": "UserWarning：分词器是'PreTrainedTokenizerFast'，但模型是用慢速分词器加载的。这可能导致编码不一致。",
  "regex": "UserWarning: The tokenizer is a 'PreTrainedTokenizerFast' but the model was loaded with a slow tokenizer",
  "domain": "huggingface",
  "category": "config_error",
  "subcategory": null,
  "root_cause": "When loading a model, the tokenizer type (fast vs slow) is inconsistent between the model configuration and the actual tokenizer used, often due to mismatched AutoTokenizer vs AutoModel loading.",
  "root_cause_type": "generic",
  "root_cause_zh": "加载模型时，模型配置与实际使用的分词器之间的分词器类型（快速 vs 慢速）不一致，通常是由于AutoTokenizer与AutoModel加载不匹配。",
  "versions": [
    {
      "version": "transformers>=4.28.0",
      "introduced": null,
      "deprecated": null,
      "removed": null,
      "behavior_change": null,
      "status": "active"
    }
  ],
  "os_specific": {},
  "dead_ends": [
    {
      "action": "Ignoring the warning and proceeding with training",
      "why_fails": "The inconsistency can cause subtle bugs in tokenization (e.g., different special token handling, whitespace behavior) leading to incorrect model outputs or training divergence.",
      "fail_rate": 0.6,
      "condition": "",
      "sources": []
    },
    {
      "action": "Reinstalling transformers to fix the warning",
      "why_fails": "The warning is not due to installation issues but to inconsistent tokenizer selection during model loading.",
      "fail_rate": 0.95,
      "condition": "",
      "sources": []
    }
  ],
  "workarounds": [
    {
      "action": "Force loading a fast tokenizer by using use_fast=True: tokenizer = AutoTokenizer.from_pretrained('model-name', use_fast=True). This ensures consistency.",
      "success_rate": 0.95,
      "how": "Force loading a fast tokenizer by using use_fast=True: tokenizer = AutoTokenizer.from_pretrained('model-name', use_fast=True). This ensures consistency.",
      "condition": "",
      "sources": []
    },
    {
      "action": "If you must use a slow tokenizer, explicitly load it: tokenizer = AutoTokenizer.from_pretrained('model-name', use_fast=False) and ensure the model was also loaded with a slow tokenizer.",
      "success_rate": 0.85,
      "how": "If you must use a slow tokenizer, explicitly load it: tokenizer = AutoTokenizer.from_pretrained('model-name', use_fast=False) and ensure the model was also loaded with a slow tokenizer.",
      "condition": "",
      "sources": []
    }
  ],
  "workarounds_zh": [
    "Force loading a fast tokenizer by using use_fast=True: tokenizer = AutoTokenizer.from_pretrained('model-name', use_fast=True). This ensures consistency.",
    "If you must use a slow tokenizer, explicitly load it: tokenizer = AutoTokenizer.from_pretrained('model-name', use_fast=False) and ensure the model was also loaded with a slow tokenizer."
  ],
  "transition_graph": {
    "leads_to": [],
    "preceded_by": [],
    "frequently_confused_with": []
  },
  "official_doc_url": "https://huggingface.co/docs/transformers/v4.37.0/en/tokenizer_summary#fast-tokenizers",
  "official_doc_section": null,
  "error_code": null,
  "verification_tier": "ai_generated",
  "confidence": 0.82,
  "fix_success_rate": 0.9,
  "resolvable": "true",
  "first_seen": "2023-11-05",
  "last_confirmed": "2024-06-01",
  "last_updated": "2024-06-01",
  "evidence_count": 1,
  "tags": [],
  "locale": "en",
  "aliases": []
}