{ "schema_version": "transcribe-intake-v1", "family": "whisper", "hf_repo": "openai/whisper-tiny", "hf_revision": "169d4a4341b33bc18d8881c4b69c2e104e1cc0af", "sources": { "config": { "kind": "hf_file", "path": "config.json", "status": "found" }, "preprocessor": { "kind": "hf_file", "path": "preprocessor_config.json", "status": "found" }, "tokenizer_config": { "kind": "hf_file", "path": "tokenizer_config.json", "status": "found" }, "tokenizer_json": { "kind": "hf_file", "path": "tokenizer.json", "status": "found" }, "generation_config": { "kind": "hf_file", "path": "generation_config.json", "status": "found" }, "safetensors_metadata": { "kind": "hf_api", "path": "HfApi.get_safetensors_metadata", "status": "found", "detail": "header-only floating dtype distribution; no tensor payloads downloaded" }, "model_card": { "kind": "hf_api", "path": "https://huggingface.co/openai/whisper-tiny", "status": "found", "detail": "languages, capability flags, and LibriSpeech/Common-Voice WER read from the model card" }, "reference_modeling_code": { "kind": "reference_code", "path": "transformers.models.whisper (WhisperForConditionalGeneration, WhisperFeatureExtractor, WhisperTokenizer); cross-check refs/mlx/mlx-audio/mlx_audio/stt/models/whisper/ and refs/ggml-org/whisper.cpp", "status": "found", "detail": "Transformers is the canonical class the HF card points at. whisper.cpp is a ggml cross-check. mlx-audio is a Python cross-check." } }, "variants": [ { "name": "whisper-tiny", "memory_gb": null, "files": [] } ], "config": { "architecture_candidates": [ "encoder-decoder" ], "key_fields": { "activation_function": "gelu", "architectures": [ "WhisperForConditionalGeneration" ], "d_model": 384, "encoder_layers": 4, "decoder_layers": 4, "encoder_attention_heads": 6, "decoder_attention_heads": 6, "num_mel_bins": 80, "max_source_positions": 1500, "max_target_positions": 448, "model_type": "whisper", "vocab_size": 51865 }, "varying_across_variants": [ "d_model", "encoder_layers", "decoder_layers", "encoder_attention_heads", "decoder_attention_heads" ] }, "dtype": { "expected": "float32", "source": "config", "evidence": "config torch_dtype=float32; safetensors header distribution F32=167 (100%)", "details": { "config_declared": "float32", "header_distribution": { "F32": 167 } } }, "frontend": { "sample_rate": 16000, "n_mels": 80, "hop_length": 160, "fft_size": 400, "window": "hann_periodic", "normalization": "global", "preemphasis": null, "dither": null, "center": true, "padding_mode": "reflect", "mel_filterbank_norm": "slaney" }, "tokenizer": { "type": "bpe", "vocab_size": 51865, "special_tokens": { "bos": 50257, "eos": 50257, "pad": 50257, "decoder_start": 50258, "transcribe": 50359, "translate": 50358, "no_timestamps": 50363, "prev_sot": 50361 }, "has_language_tokens": true, "vocab_sha256": "c4883442728ea9057d2d031c2abdf0e96dbc58cac3cb2a3461f027f4d3c0ec6d" }, "capabilities": { "languages": [ "en", "zh", "de", "es", "ru", "ko", "fr", "ja", "pt", "tr", "pl", "ca", "nl", "ar", "sv", "it", "id", "hi", "fi", "vi", "he", "uk", "el", "ms", "cs", "ro", "da", "hu", "ta", "no", "th", "ur", "hr", "bg", "lt", "la", "mi", "ml", "cy", "sk", "te", "fa", "lv", "bn", "sr", "az", "sl", "kn", "et", "mk", "br", "eu", "is", "hy", "ne", "mn", "bs", "kk", "sq", "sw", "gl", "mr", "pa", "si", "km", "sn", "yo", "so", "af", "oc", "ka", "be", "tg", "sd", "gu", "am", "yi", "lo", "uz", "fo", "ht", "ps", "tk", "nn", "mt", "sa", "lb", "my", "bo", "tl", "mg", "as", "tt", "haw", "ln", "ha", "ba", "jw", "su" ], "language_detection": true, "translation": true, "timestamps": ["segment", "word"], "streaming": false, "speaker_diarization": false }, "upstream_benchmarks": [ { "dataset": "LibriSpeech test-clean", "language": "en", "metric": "wer", "score": 7.54, "score_unit": "percent", "source": "https://huggingface.co/openai/whisper-tiny", "notes": "Model card reports WER for the multilingual tiny variant evaluated on LibriSpeech test-clean." }, { "dataset": "LibriSpeech test-other", "language": "en", "metric": "wer", "score": 17.15, "score_unit": "percent", "source": "https://huggingface.co/openai/whisper-tiny", "notes": "Model card." }, { "dataset": "Common Voice 11.0", "language": null, "metric": "wer", "score": 141.0, "score_unit": "percent", "source": "https://huggingface.co/openai/whisper-tiny", "notes": "Model-card number for the multilingual tiny variant; unusually high, recorded as reported." } ], "reference_framework": "transformers", "reference_rationale": "The openai/whisper-tiny model card shows direct use of `WhisperProcessor` + `WhisperForConditionalGeneration` from mainline `transformers` (no `trust_remote_code`). The Whisper classes are first-party in transformers and carry the publisher-maintained generation config, tokenizer, and feature extractor — they are the canonical instrumentable reference. The original `openai/whisper` PyPI package and `refs/ggml-org/whisper.cpp` are useful cross-checks but are not the published entrypoint for this HF repo.", "architecture_pattern": "encoder-decoder", "known_risks": [ "Whisper-specific log-mel post-processing: `log10(max(mel, 1e-10)) -> max(x, x.max()-8.0) -> (x + 4.0) / 4.0`. This is a per-utterance dynamic-range compression, not any of the standard stat normalizations (mean/std, per-feature). Must be replicated exactly or WER degrades silently.", "Input is always pad-or-trimmed to exactly 30s (480000 samples → 3000 mel frames → 1500 encoder frames after two conv1d subsamples, the second stride=2). Short audio is zero-padded; audio longer than 30s requires chunking in the decoding loop. First port should target single-chunk transcription.", "Mixed positional encoding: encoder uses sinusoidal positional embedding (precomputed for max_source_positions=1500, added to the post-conv embedding), decoder uses a learned positional embedding matrix of size max_target_positions=448. Two different code paths for positional info.", "Decoder generation requires a forced prompt prefix: `<|startoftranscript|> <|lang|> <|task|> <|no_timestamps|>` (4 tokens). Skipping or mis-ordering these yields either the wrong language, translation instead of transcription, or timestamp-token noise in the output. These are configured via generation_config.forced_decoder_ids.", "Tokenizer vs model vocab size mismatch: tokenizer base vocab is 50258 (GPT-2 BPE + 1); model output dim is 51865. The extra 1607 tokens are added special tokens (99 language tokens 50259–50357, 2 task tokens 50358/50359, ~4 auxiliary control tokens, and 1501 timestamp tokens 50364–51864 at 20ms granularity). Converter must export the full added_tokens list so the tokenizer rebuilt from GGUF matches the model's output space.", "generation_config.suppress_tokens (87 ids including most punctuation and special tokens) must be applied at decode time; dropping it causes the decoder to emit junk.", "Timestamp emission: segment timestamps come from generation of timestamp tokens (50364+); word-level timestamps come from DTW over selected cross-attention heads (generation_config.alignment_heads). Timestamp extraction is a post-processing step over the token stream + cross-attn, not a separate head." ], "intake_gaps": [] }