From 55e59c30dbecb524467ae42514a24b8bdc2f232a Mon Sep 17 00:00:00 2001 From: Hongyi Wu <62729549+Aharrypotter@users.noreply.github.com> Date: Wed, 12 Aug 2026 16:36:57 +0800 Subject: [PATCH] feat(cpu): add Qwen3.5 4B multimodal support --- examples/qwen3_5/README.md | 49 +++++++++- .../config_4B_multimodal_w4a32_kai.json | 97 +++++++++++++++++++ .../quant_cfg_4B_multimodal_w4a32_kai.json | 21 ++++ examples/qwen3_5/test_validators.py | 40 ++++++++ examples/qwen3_5/validate_checkpoint.py | 94 ++++++++++++------ mllm/models/qwen3_5/configuration_qwen3_5.hpp | 50 +++++++--- tests/cpu/Qwen35ConfigTest.cpp | 48 ++++++--- 7 files changed, 342 insertions(+), 57 deletions(-) create mode 100644 examples/qwen3_5/config_4B_multimodal_w4a32_kai.json create mode 100644 examples/qwen3_5/quant_cfg_4B_multimodal_w4a32_kai.json diff --git a/examples/qwen3_5/README.md b/examples/qwen3_5/README.md index 82c270ff4..0b5f75d47 100644 --- a/examples/qwen3_5/README.md +++ b/examples/qwen3_5/README.md @@ -2,13 +2,14 @@ This example supports the text towers of [`Qwen/Qwen3.5-0.8B`](https://huggingface.co/Qwen/Qwen3.5-0.8B) and -[`Qwen/Qwen3.5-4B`](https://huggingface.co/Qwen/Qwen3.5-4B), plus single-image, -ordered multi-image, and bounded short-video inference for Qwen3.5-0.8B. +[`Qwen/Qwen3.5-4B`](https://huggingface.co/Qwen/Qwen3.5-4B), including +single-image, ordered multi-image, and bounded short-video inference for both +model sizes. | Model | Text | Images | Decoded RGB-frame API | Local H.264 MP4 | Hidden size | GDN / full-attention layers | | --- | --- | --- | --- | --- | ---: | ---: | | 0.8B | Yes | Yes | Yes | Yes, portable backend only | 1024 | 18 / 6 | -| 4B | Yes | No | No | No | 2560 | 24 / 8 | +| 4B | Yes | Yes | Yes | Yes, portable backend only | 2560 | 24 / 8 | All configurations support batch size 1 and a maximum cache length of 2048 tokens. `Qwen3_5ForCausalLM::resetState()` clears the full-attention KV cache @@ -112,7 +113,7 @@ python examples/qwen3_5/validate_converted_model.py \ --model-name Qwen3.5-0.8B ``` -### Qwen3.5-0.8B single-image +### Qwen3.5-0.8B multimodal ```bash python examples/qwen3_5/validate_checkpoint.py \ @@ -163,6 +164,33 @@ python examples/qwen3_5/validate_converted_model.py \ --model-name Qwen3.5-4B ``` +### Qwen3.5-4B multimodal + +```bash +python examples/qwen3_5/validate_checkpoint.py \ + /path/to/Qwen3.5-4B \ + --quant-config examples/qwen3_5/quant_cfg_4B_multimodal_w4a32_kai.json \ + --runtime-config examples/qwen3_5/config_4B_multimodal_w4a32_kai.json + +python -m pymllm.mobile.utils.mllm_convertor \ + --input_path /path/to/Qwen3.5-4B \ + --output_path /path/to/qwen3.5-4b-multimodal-w4a32-kai.mllm \ + --model_name Qwen3.5-4B-Multimodal \ + --cfg_path examples/qwen3_5/quant_cfg_4B_multimodal_w4a32_kai.json \ + --pipeline w4a32_kai_pipeline \ + --include_prefix model.language_model. \ + --include_prefix model.visual. \ + --format v2 \ + --verbose + +python examples/qwen3_5/validate_converted_model.py \ + /path/to/qwen3.5-4b-multimodal-w4a32-kai.mllm \ + /path/to/Qwen3.5-4B \ + --quant-config examples/qwen3_5/quant_cfg_4B_multimodal_w4a32_kai.json \ + --runtime-config examples/qwen3_5/config_4B_multimodal_w4a32_kai.json \ + --model-name Qwen3.5-4B-Multimodal +``` + The 4B converted tensor data exceeds 4 GiB. Use model-file V2 so descriptor sizes and offsets remain 64-bit, and provide at least 32 GiB of available host memory plus working disk space for conversion. @@ -209,12 +237,23 @@ mllm-qwen3-5-runner \ --config_path examples/qwen3_5/config_4B_w4a32_kai.json \ --prompt "Give a one-sentence introduction." \ --max_new_tokens 32 + +# Qwen3.5-4B multimodal; repeat --image_path for ordered multi-image input, +# or replace it with --video_path in a decoder-enabled build. +mllm-qwen3-5-runner \ + --model_path /path/to/qwen3.5-4b-multimodal-w4a32-kai.mllm \ + --model_version v2 \ + --tokenizer_path /path/to/Qwen3.5-4B/tokenizer.json \ + --config_path examples/qwen3_5/config_4B_multimodal_w4a32_kai.json \ + --image_path /path/to/image.jpg \ + --prompt "Describe the image." \ + --max_new_tokens 32 ``` Omit `--prompt` for the interactive loop. Repeat `--image_path` to attach multiple still images in order; the same ordered image list is used for each independent prompt in that process. Omit `--image_path` to run text-only -inference with either the text-only or multimodal 0.8B model. `--video_path` +inference with either the text-only or multimodal model of the selected size. `--video_path` requires a decoder-enabled build and cannot be combined with `--image_path`. Use `--video_fps`, `--video_max_frames`, `--video_max_bytes`, `--video_max_tokens`, `--video_max_decoded_pixels`, and diff --git a/examples/qwen3_5/config_4B_multimodal_w4a32_kai.json b/examples/qwen3_5/config_4B_multimodal_w4a32_kai.json new file mode 100644 index 000000000..5428cf6f5 --- /dev/null +++ b/examples/qwen3_5/config_4B_multimodal_w4a32_kai.json @@ -0,0 +1,97 @@ +{ + "architectures": [ + "Qwen3_5ForConditionalGeneration" + ], + "model_type": "qwen3_5", + "image_token_id": 248056, + "video_token_id": 248057, + "vision_start_token_id": 248053, + "vision_end_token_id": 248054, + "image_min_pixels": 65536, + "image_max_pixels": 262144, + "text_config": { + "attention_bias": false, + "attn_output_gate": true, + "eos_token_id": 248044, + "full_attention_interval": 4, + "head_dim": 256, + "hidden_act": "silu", + "hidden_size": 2560, + "intermediate_size": 9216, + "layer_types": [ + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention", + "linear_attention", + "linear_attention", + "linear_attention", + "full_attention" + ], + "linear_conv_kernel_dim": 4, + "linear_key_head_dim": 128, + "linear_num_key_heads": 16, + "linear_num_value_heads": 32, + "linear_value_head_dim": 128, + "mamba_ssm_dtype": "float32", + "max_position_embeddings": 262144, + "num_attention_heads": 16, + "num_hidden_layers": 32, + "num_key_value_heads": 4, + "rms_norm_eps": 1e-06, + "rope_parameters": { + "mrope_interleaved": true, + "mrope_section": [ + 11, + 11, + 10 + ], + "partial_rotary_factor": 0.25, + "rope_theta": 10000000, + "rope_type": "default" + }, + "tie_word_embeddings": true, + "vocab_size": 248320 + }, + "vision_config": { + "deepstack_visual_indexes": [], + "depth": 24, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1024, + "in_channels": 3, + "intermediate_size": 4096, + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2560, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2 + }, + "tie_word_embeddings": true, + "max_cache_length": 2048, + "linear_impl_type": "KaiLinear_f32_qai8dxp_qsi4c32p_mxk_nxk_qai8dxp1x8_qsi4c32p8x8_1x8x32" +} diff --git a/examples/qwen3_5/quant_cfg_4B_multimodal_w4a32_kai.json b/examples/qwen3_5/quant_cfg_4B_multimodal_w4a32_kai.json new file mode 100644 index 000000000..11d19215b --- /dev/null +++ b/examples/qwen3_5/quant_cfg_4B_multimodal_w4a32_kai.json @@ -0,0 +1,21 @@ +{ + "^model\\.language_model\\.layers\\.\\d+\\.self_attn\\.q_proj\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [8192, 2560], "replace": true}}, + "^model\\.language_model\\.layers\\.\\d+\\.self_attn\\.k_proj\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [1024, 2560], "replace": true}}, + "^model\\.language_model\\.layers\\.\\d+\\.self_attn\\.v_proj\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [1024, 2560], "replace": true}}, + "^model\\.language_model\\.layers\\.\\d+\\.self_attn\\.o_proj\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [2560, 4096], "replace": true}}, + "^model\\.language_model\\.layers\\.\\d+\\.linear_attn\\.in_proj_qkv\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [8192, 2560], "replace": true}}, + "^model\\.language_model\\.layers\\.\\d+\\.linear_attn\\.in_proj_z\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [4096, 2560], "replace": true}}, + "^model\\.language_model\\.layers\\.\\d+\\.linear_attn\\.in_proj_a\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [32, 2560], "replace": true}}, + "^model\\.language_model\\.layers\\.\\d+\\.linear_attn\\.in_proj_b\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [32, 2560], "replace": true}}, + "^model\\.language_model\\.layers\\.\\d+\\.linear_attn\\.out_proj\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [2560, 4096], "replace": true}}, + "^model\\.language_model\\.layers\\.\\d+\\.mlp\\.gate_proj\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [9216, 2560], "replace": true}}, + "^model\\.language_model\\.layers\\.\\d+\\.mlp\\.up_proj\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [9216, 2560], "replace": true}}, + "^model\\.language_model\\.layers\\.\\d+\\.mlp\\.down_proj\\.(bias|weight)": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [2560, 9216], "replace": true}}, + "^model\\.language_model\\.embed_tokens\\.weight$": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [248320, 2560], "replace": false, "rename": "lm_head_out.weight"}}, + "^model\\.visual\\.blocks\\.\\d+\\.attn\\.qkv\\.(bias|weight)$": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [3072, 1024], "replace": true}}, + "^model\\.visual\\.blocks\\.\\d+\\.attn\\.proj\\.(bias|weight)$": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [1024, 1024], "replace": true}}, + "^model\\.visual\\.blocks\\.\\d+\\.mlp\\.linear_fc1\\.(bias|weight)$": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [4096, 1024], "replace": true}}, + "^model\\.visual\\.blocks\\.\\d+\\.mlp\\.linear_fc2\\.(bias|weight)$": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [1024, 4096], "replace": true}}, + "^model\\.visual\\.merger\\.linear_fc1\\.(bias|weight)$": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [4096, 4096], "replace": true}}, + "^model\\.visual\\.merger\\.linear_fc2\\.(bias|weight)$": {"hints": {"quant_method": "kai", "kai_matmul_triplet": "f32_qai8dxp_qsi4c32p", "kai_matmul_layout": "mxk_nxk", "kai_matmul_tile_cfg": "qai8dxp1x8_qsi4c32p8x8_1x8x32", "shape": [2560, 4096], "replace": true}} +} diff --git a/examples/qwen3_5/test_validators.py b/examples/qwen3_5/test_validators.py index c48d29ed4..b7b050a7c 100644 --- a/examples/qwen3_5/test_validators.py +++ b/examples/qwen3_5/test_validators.py @@ -161,6 +161,35 @@ def test_official_08b_multimodal_recipe_and_descriptors(self) -> None: 103, ) + def test_official_4b_multimodal_recipe_and_descriptors(self) -> None: + config = _load_json("config_4B_multimodal_w4a32_kai.json") + quant_config = _load_json("quant_cfg_4B_multimodal_w4a32_kai.json") + + checkpoint_config = copy.deepcopy(config) + checkpoint_config["vision_config"].update( + {"initializer_range": 0.02, "model_type": "qwen3_5"} + ) + validate_multimodal_config_contract(checkpoint_config, config) + quantized_names, _ = validate_kai_recipe_contract( + checkpoint_config["text_config"], + quant_config, + config, + checkpoint_config["vision_config"], + ) + self.assertEqual(len(quantized_names), 445) + self.assertEqual(len(expected_vision_shapes(config["vision_config"])), 297) + + expected = _expected_descriptors(config, quant_config) + dtype_counts = Counter(dtype for dtype, _, _ in expected.values()) + self.assertEqual(len(expected), 626) + self.assertEqual(dtype_counts[BYTE], 347) + self.assertEqual(dtype_counts[FLOAT32], 279) + self.assertEqual( + sum(name.startswith("model.visual.") for name in expected), + 199, + ) + self.assertEqual(sum(size for _, _, size in expected.values()), 5_126_041_600) + def test_multimodal_recipe_rejects_text_runtime_and_contract_drift(self) -> None: config = _load_json("config_0.8B_multimodal_w4a32_kai.json") text_runtime = _load_json("config_0.8B_w4a32_kai.json") @@ -179,6 +208,17 @@ def test_multimodal_recipe_rejects_text_runtime_and_contract_drift(self) -> None with self.assertRaisesRegex(AssertionError, "image_max_pixels"): validate_multimodal_config_contract(config, mutated) + config_4b = _load_json("config_4B_multimodal_w4a32_kai.json") + wrong_vision = copy.deepcopy(config_4b) + wrong_vision["vision_config"]["depth"] = 12 + with self.assertRaisesRegex( + AssertionError, "official Qwen3.5-4B" + ): + validate_multimodal_config_contract(config_4b, wrong_vision) + + with self.assertRaisesRegex(AssertionError, "runtime is Qwen3.5-0.8B"): + validate_multimodal_config_contract(config_4b, config) + def test_variant_resolution_rejects_runtime_semantic_mismatches(self) -> None: text_config = _load_json("config_4B_w4a32_kai.json")["text_config"] mutations = ( diff --git a/examples/qwen3_5/validate_checkpoint.py b/examples/qwen3_5/validate_checkpoint.py index 0bc951d95..7b8f39dd2 100644 --- a/examples/qwen3_5/validate_checkpoint.py +++ b/examples/qwen3_5/validate_checkpoint.py @@ -98,19 +98,35 @@ "KaiLinear_f32_qai8dxp_qsi4c32p_mxk_nxk_" "qai8dxp1x8_qsi4c32p8x8_1x8x32" ) TIED_EMBEDDING_WEIGHT = "model.language_model.embed_tokens.weight" -OFFICIAL_08B_VISION_CONTRACT = { - "deepstack_visual_indexes": [], - "depth": 12, - "hidden_act": "gelu_pytorch_tanh", - "hidden_size": 768, - "in_channels": 3, - "intermediate_size": 3072, - "num_heads": 12, - "num_position_embeddings": 2304, - "out_hidden_size": 1024, - "patch_size": 16, - "spatial_merge_size": 2, - "temporal_patch_size": 2, +OFFICIAL_VISION_CONTRACTS = { + "0.8B": { + "deepstack_visual_indexes": [], + "depth": 12, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 768, + "in_channels": 3, + "intermediate_size": 3072, + "num_heads": 12, + "num_position_embeddings": 2304, + "out_hidden_size": 1024, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2, + }, + "4B": { + "deepstack_visual_indexes": [], + "depth": 24, + "hidden_act": "gelu_pytorch_tanh", + "hidden_size": 1024, + "in_channels": 3, + "intermediate_size": 4096, + "num_heads": 16, + "num_position_embeddings": 2304, + "out_hidden_size": 2560, + "patch_size": 16, + "spatial_merge_size": 2, + "temporal_patch_size": 2, + }, } MULTIMODAL_TOKEN_CONTRACT = { "image_token_id": 248056, @@ -120,10 +136,13 @@ } -def _vision_contract_mismatches(vision_config: dict) -> list[str]: +def _vision_contract_mismatches( + vision_config: dict, model_size: str +) -> list[str]: + contract = OFFICIAL_VISION_CONTRACTS[model_size] return [ f"{field}={vision_config.get(field)!r} (expected {expected!r})" - for field, expected in OFFICIAL_08B_VISION_CONTRACT.items() + for field, expected in contract.items() if vision_config.get(field) != expected ] @@ -372,24 +391,24 @@ def validate_kai_recipe_contract( "KAI recipe and runtime config must either both include vision or both be text-only" ) if include_vision: - if checkpoint_size != "0.8B": - raise AssertionError( - "Qwen3.5 multimodal conversion currently supports only 0.8B" - ) if not isinstance(vision_config, dict): raise AssertionError( "Multimodal KAI validation requires checkpoint vision_config" ) - checkpoint_vision_mismatches = _vision_contract_mismatches(vision_config) + checkpoint_vision_mismatches = _vision_contract_mismatches( + vision_config, checkpoint_size + ) if checkpoint_vision_mismatches: raise AssertionError( - "Checkpoint vision_config does not match official Qwen3.5-0.8B: " + f"Checkpoint vision_config does not match official Qwen3.5-{checkpoint_size}: " + "; ".join(checkpoint_vision_mismatches) ) - runtime_vision_mismatches = _vision_contract_mismatches(runtime_vision_config) + runtime_vision_mismatches = _vision_contract_mismatches( + runtime_vision_config, checkpoint_size + ) if runtime_vision_mismatches: raise AssertionError( - "Runtime vision_config does not match official Qwen3.5-0.8B: " + f"Runtime vision_config does not match official Qwen3.5-{checkpoint_size}: " + "; ".join(runtime_vision_mismatches) ) @@ -536,7 +555,22 @@ def expected_vision_shapes(vision_config: dict) -> dict[str, list[int]]: def validate_multimodal_config_contract( checkpoint_config: dict, runtime_config: dict ) -> None: - """Validate the official 0.8B single-image checkpoint/runtime boundary.""" + """Validate a supported official multimodal checkpoint/runtime boundary.""" + + checkpoint_text_config = checkpoint_config.get("text_config") + runtime_text_config = runtime_config.get("text_config") + if not isinstance(checkpoint_text_config, dict) or not isinstance( + runtime_text_config, dict + ): + raise AssertionError( + "Multimodal checkpoint and runtime must declare object text_config values" + ) + checkpoint_size = resolve_model_size(checkpoint_text_config) + runtime_size = resolve_model_size(runtime_text_config) + if checkpoint_size != runtime_size: + raise AssertionError( + f"Multimodal runtime is Qwen3.5-{runtime_size}, checkpoint is Qwen3.5-{checkpoint_size}" + ) for field, expected in MULTIMODAL_TOKEN_CONTRACT.items(): if checkpoint_config.get(field) != expected: @@ -550,19 +584,23 @@ def validate_multimodal_config_contract( checkpoint_vision_config = checkpoint_config.get("vision_config") if not isinstance(checkpoint_vision_config, dict): raise AssertionError("Checkpoint must declare vision_config") - checkpoint_vision_mismatches = _vision_contract_mismatches(checkpoint_vision_config) + checkpoint_vision_mismatches = _vision_contract_mismatches( + checkpoint_vision_config, checkpoint_size + ) if checkpoint_vision_mismatches: raise AssertionError( - "Checkpoint vision_config does not match official Qwen3.5-0.8B: " + f"Checkpoint vision_config does not match official Qwen3.5-{checkpoint_size}: " + "; ".join(checkpoint_vision_mismatches) ) runtime_vision_config = runtime_config.get("vision_config") if not isinstance(runtime_vision_config, dict): raise AssertionError("Runtime must declare vision_config") - runtime_vision_mismatches = _vision_contract_mismatches(runtime_vision_config) + runtime_vision_mismatches = _vision_contract_mismatches( + runtime_vision_config, checkpoint_size + ) if runtime_vision_mismatches: raise AssertionError( - "Runtime vision_config does not match official Qwen3.5-0.8B: " + f"Runtime vision_config does not match official Qwen3.5-{checkpoint_size}: " + "; ".join(runtime_vision_mismatches) ) if runtime_config.get("image_min_pixels") != 256 * 256: diff --git a/mllm/models/qwen3_5/configuration_qwen3_5.hpp b/mllm/models/qwen3_5/configuration_qwen3_5.hpp index 9e078d226..a456963ee 100644 --- a/mllm/models/qwen3_5/configuration_qwen3_5.hpp +++ b/mllm/models/qwen3_5/configuration_qwen3_5.hpp @@ -152,7 +152,7 @@ struct Qwen3_5Config : protected ConfigFile { throw std::invalid_argument("Qwen3.5 CPU only supports gelu_pytorch_tanh in vision blocks"); } if (!vision_deepstack_visual_indexes.empty()) { - throw std::invalid_argument("Qwen3.5 CPU single-image support does not implement deep-stack visual features"); + throw std::invalid_argument("Qwen3.5 CPU multimodal support does not implement deep-stack visual features"); } if (!mrope_interleaved || mrope_section.size() != 3 || std::any_of(mrope_section.begin(), mrope_section.end(), [](int32_t section) { return section <= 0; }) @@ -310,7 +310,7 @@ inline auto isOfficialQwen35_08BTextRuntimeConfig(const Qwen3_5Config& config) - return hasOfficialQwen35_08BTextGeometry(config) && !config.vision_enabled; } -/// Checks the single-image Qwen3.5-0.8B vision and deployment contract. +/// Checks the Qwen3.5-0.8B vision and bounded mobile deployment contract. inline auto isOfficialQwen35_08BMultimodalRuntimeConfig(const Qwen3_5Config& config) -> bool { return hasOfficialQwen35_08BTextGeometry(config) && config.vision_enabled && config.vision_depth == 12 && config.vision_hidden_size == 768 && config.vision_intermediate_size == 3072 && config.vision_in_channels == 3 @@ -327,14 +327,39 @@ inline auto isOfficialQwen35_08BRuntimeConfig(const Qwen3_5Config& config) -> bo return isOfficialQwen35_08BTextRuntimeConfig(config) || isOfficialQwen35_08BMultimodalRuntimeConfig(config); } -/// Checks whether \p config is the official Qwen3.5-4B runtime configuration: the shared -/// contract plus the 4B hidden/intermediate sizes, layer count, and head counts. -/// \param config Qwen3.5 text-tower configuration to inspect. -/// \return true only for the supported 4B variant. -inline auto isOfficialQwen35_4BRuntimeConfig(const Qwen3_5Config& config) -> bool { +inline auto hasOfficialQwen35_4BTextGeometry(const Qwen3_5Config& config) -> bool { return hasOfficialCommonRuntimeContract(config) && config.hidden_size == 2560 && config.intermediate_size == 9216 && config.num_hidden_layers == 32 && config.num_attention_heads == 16 && config.num_key_value_heads == 4 - && config.linear_num_value_heads == 32 && !config.vision_enabled; + && config.linear_num_value_heads == 32; +} + +/// Checks whether \p config is the official text-only Qwen3.5-4B runtime configuration. +/// \param config Qwen3.5 text-tower configuration to inspect. +/// \return true only for the supported text-only 4B variant. +inline auto isOfficialQwen35_4BTextRuntimeConfig(const Qwen3_5Config& config) -> bool { + return hasOfficialQwen35_4BTextGeometry(config) && !config.vision_enabled; +} + +/// Checks the Qwen3.5-4B vision and bounded mobile deployment contract. +inline auto isOfficialQwen35_4BMultimodalRuntimeConfig(const Qwen3_5Config& config) -> bool { + return hasOfficialQwen35_4BTextGeometry(config) && config.vision_enabled && config.vision_depth == 24 + && config.vision_hidden_size == 1024 && config.vision_intermediate_size == 4096 && config.vision_in_channels == 3 + && config.vision_num_heads == 16 && config.vision_num_position_embeddings == 2304 + && config.vision_out_hidden_size == 2560 && config.vision_patch_size == 16 && config.vision_spatial_merge_size == 2 + && config.vision_temporal_patch_size == 2 && config.vision_hidden_act == "gelu_pytorch_tanh" + && config.vision_deepstack_visual_indexes.empty() && config.image_token_id == 248056 && config.video_token_id == 248057 + && config.vision_start_token_id == 248053 && config.vision_end_token_id == 248054 + && config.image_min_pixels == 256 * 256 && config.image_max_pixels == 512 * 512; +} + +/// Checks whether \p config is either supported Qwen3.5-4B runtime contract. +inline auto isOfficialQwen35_4BRuntimeConfig(const Qwen3_5Config& config) -> bool { + return isOfficialQwen35_4BTextRuntimeConfig(config) || isOfficialQwen35_4BMultimodalRuntimeConfig(config); +} + +/// Checks whether \p config is either supported multimodal runtime contract. +inline auto isOfficialQwen35MultimodalRuntimeConfig(const Qwen3_5Config& config) -> bool { + return isOfficialQwen35_08BMultimodalRuntimeConfig(config) || isOfficialQwen35_4BMultimodalRuntimeConfig(config); } /// Checks whether \p config is one of the runtime contracts this CPU runner supports. @@ -346,12 +371,13 @@ inline auto matchesOfficialRuntimeContract(const Qwen3_5Config& config) -> bool /// Resolves a human-readable model name for diagnostics and error messages. /// \param config Qwen3.5 text-tower configuration to inspect. -/// \return "Qwen3.5-0.8B" or "Qwen3.5-4B" for a supported variant, otherwise the generic -/// fallback "Qwen3.5 text model". +/// \return A size- and modality-specific name for a supported variant, otherwise the +/// generic fallback "Qwen3.5 text model". inline auto modelNameForConfig(const Qwen3_5Config& config) -> std::string { if (isOfficialQwen35_08BMultimodalRuntimeConfig(config)) { return "Qwen3.5-0.8B Multimodal"; } + if (isOfficialQwen35_4BMultimodalRuntimeConfig(config)) { return "Qwen3.5-4B Multimodal"; } if (isOfficialQwen35_08BTextRuntimeConfig(config)) { return "Qwen3.5-0.8B"; } - if (isOfficialQwen35_4BRuntimeConfig(config)) { return "Qwen3.5-4B"; } + if (isOfficialQwen35_4BTextRuntimeConfig(config)) { return "Qwen3.5-4B"; } return "Qwen3.5 text model"; } @@ -381,7 +407,7 @@ inline void validateModelConfigMatch(const Qwen3_5Config& config, const Paramete if (embedding.dtype() != kFloat32) { throw std::invalid_argument("Qwen3.5 model/config mismatch: " + model_name + " requires a float32 embedding tensor"); } - const bool multimodal = isOfficialQwen35_08BMultimodalRuntimeConfig(config); + const bool multimodal = isOfficialQwen35MultimodalRuntimeConfig(config); if (multimodal && parameter_file->version() != ModelFileVersion::kV2) { throw std::invalid_argument("Qwen3.5 model/config mismatch: multimodal inference requires model-file V2"); } diff --git a/tests/cpu/Qwen35ConfigTest.cpp b/tests/cpu/Qwen35ConfigTest.cpp index 810c2dea3..4b0e9bc40 100644 --- a/tests/cpu/Qwen35ConfigTest.cpp +++ b/tests/cpu/Qwen35ConfigTest.cpp @@ -20,8 +20,8 @@ auto loadConfig(const std::string& model_size) -> mllm::models::qwen3_5::Qwen3_5 return mllm::models::qwen3_5::Qwen3_5Config(exampleDir() + "/config_" + model_size + "_w4a32_kai.json"); } -auto loadMultimodalConfig() -> mllm::models::qwen3_5::Qwen3_5Config { - return mllm::models::qwen3_5::Qwen3_5Config(exampleDir() + "/config_0.8B_multimodal_w4a32_kai.json"); +auto loadMultimodalConfig(const std::string& model_size = "0.8B") -> mllm::models::qwen3_5::Qwen3_5Config { + return mllm::models::qwen3_5::Qwen3_5Config(exampleDir() + "/config_" + model_size + "_multimodal_w4a32_kai.json"); } constexpr auto kKaiLinearImpl = @@ -106,6 +106,27 @@ TEST(Qwen35ConfigTest, Official08BMultimodalConfigBuildsVisionContract) { EXPECT_EQ(config.image_max_pixels, 512 * 512); } +TEST(Qwen35ConfigTest, Official4BMultimodalConfigBuildsVisionContract) { + const auto config = loadMultimodalConfig("4B"); + + EXPECT_EQ(mllm::models::qwen3_5::modelNameForConfig(config), "Qwen3.5-4B Multimodal"); + EXPECT_TRUE(mllm::models::qwen3_5::isOfficialQwen35_4BMultimodalRuntimeConfig(config)); + EXPECT_TRUE(mllm::models::qwen3_5::isOfficialQwen35MultimodalRuntimeConfig(config)); + EXPECT_TRUE(config.vision_enabled); + EXPECT_EQ(config.vision_depth, 24); + EXPECT_EQ(config.vision_hidden_size, 1024); + EXPECT_EQ(config.vision_intermediate_size, 4096); + EXPECT_EQ(config.vision_num_heads, 16); + EXPECT_EQ(config.vision_num_position_embeddings, 2304); + EXPECT_EQ(config.vision_patch_size, 16); + EXPECT_EQ(config.vision_temporal_patch_size, 2); + EXPECT_EQ(config.vision_spatial_merge_size, 2); + EXPECT_EQ(config.vision_out_hidden_size, config.hidden_size); + EXPECT_EQ(config.mrope_section, (std::vector{11, 11, 10})); + EXPECT_EQ(config.image_min_pixels, 256 * 256); + EXPECT_EQ(config.image_max_pixels, 512 * 512); +} + TEST(Qwen35ConfigTest, RejectsModelConfigEmbeddingMismatchWithoutAllocatingWeights) { const auto config = loadConfig("0.8B"); const auto expected_numel = @@ -134,17 +155,20 @@ TEST(Qwen35ConfigTest, RejectsModelConfigEmbeddingMismatchWithoutAllocatingWeigh } TEST(Qwen35ConfigTest, MultimodalConfigRequiresMatchingV2VisionDescriptors) { - const auto config = loadMultimodalConfig(); - auto parameter_file = parameterFileWithEmbedding(mllm::ModelFileVersion::kV2, {config.vocab_size, config.hidden_size}); - - EXPECT_THROW(mllm::models::qwen3_5::validateModelConfigMatch(config, parameter_file), std::invalid_argument); - pushDescriptor(parameter_file, "model.visual.patch_embed.proj.weight", - {config.vision_hidden_size, config.vision_in_channels, config.vision_temporal_patch_size, - config.vision_patch_size, config.vision_patch_size}); - pushDescriptor(parameter_file, "model.visual.pos_embed.weight", - {config.vision_num_position_embeddings, config.vision_hidden_size}); - EXPECT_NO_THROW(mllm::models::qwen3_5::validateModelConfigMatch(config, parameter_file)); + for (const auto& model_size : {std::string("0.8B"), std::string("4B")}) { + const auto config = loadMultimodalConfig(model_size); + auto parameter_file = parameterFileWithEmbedding(mllm::ModelFileVersion::kV2, {config.vocab_size, config.hidden_size}); + + EXPECT_THROW(mllm::models::qwen3_5::validateModelConfigMatch(config, parameter_file), std::invalid_argument) << model_size; + pushDescriptor(parameter_file, "model.visual.patch_embed.proj.weight", + {config.vision_hidden_size, config.vision_in_channels, config.vision_temporal_patch_size, + config.vision_patch_size, config.vision_patch_size}); + pushDescriptor(parameter_file, "model.visual.pos_embed.weight", + {config.vision_num_position_embeddings, config.vision_hidden_size}); + EXPECT_NO_THROW(mllm::models::qwen3_5::validateModelConfigMatch(config, parameter_file)) << model_size; + } + const auto config = loadMultimodalConfig(); const auto v1 = parameterFileWithEmbedding(mllm::ModelFileVersion::kV1, {config.vocab_size * config.hidden_size}); EXPECT_THROW(mllm::models::qwen3_5::validateModelConfigMatch(config, v1), std::invalid_argument); }