Skip to content

model: add glm-asr support #17901

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
piDack wants to merge 3 commits into
base: master
Choose a base branch
from
+123 −7
Open

model: add glm-asr support #17901

Changes from 1 commit
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f432a5c
[model] add glm-asr support
piDack Dec 10, 2025
c382d64
fix format for ci
piDack Dec 10, 2025
e8a1ec5
fix convert format for ci
piDack Dec 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
fix format for ci
  • Loading branch information
@piDack
piDack committed Dec 10, 2025
commit c382d64af4e264c02ed9cc67e0cd446a274861c9
9 changes: 4 additions & 5 deletions convert_hf_to_gguf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2411,7 +2411,6 @@ def set_vocab(self):
# Apply to granite small models only
if self.hparams.get("vocab_size", 32000) == 49152:
self.gguf_writer.add_add_bos_token(False)

if isinstance(self.hparams.get("eos_token_id"), list):
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained(self.dir_model, trust_remote_code=True)
Expand Down Expand Up @@ -9014,6 +9013,7 @@ def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
raise NotImplementedError("Ultravox does not have text decoder. Instead, it uses Llama or other models for text. If you want to get the audio encoder, please use --mmproj argument")


@ModelBase.register("GlmasrModel")
class GlmASRWhisperEncoderModel(MmprojModel):
has_vision_encoder = False
Expand All @@ -9025,7 +9025,7 @@ def __init__(self, *args, **kwargs):
self.hparams["hidden_size"] = self.hparams["d_model"]
self.hparams["intermediate_size"] = self.hparams["encoder_ffn_dim"]
self.hparams["num_attention_heads"] = self.hparams["encoder_attention_heads"]

def set_gguf_parameters(self):
super().set_gguf_parameters()
self.gguf_writer.add_clip_projector_type(gguf.VisionProjectorType.GLMA)
Expand All @@ -9043,12 +9043,12 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
if name.startswith("model.") or name.startswith("lm_head."):
# skip language model tensors
return []

if name.startswith("audio_encoder.whisper."):
name = name.replace("audio_encoder.whisper.","audio_tower.")
if "audio_encoder.layer_norm." in name or "audio_encoder.proj." in name:
name = name.replace("audio_encoder.", "audio_encoder.adapting.")

if name.startswith("audio_encoder.audio_bos_eos_token."):
return [(self.map_tensor_name("model.vision.boi"), data_torch[0]), (self.map_tensor_name("model.vision.eoi"), data_torch[1])]

Expand All @@ -9061,7 +9061,6 @@ def modify_tensors(self, data_torch: Tensor, name: str, bid: int | None) -> Iter
if ".2." in name:
name = name.replace(".2.", ".linear_2.")
if ".proj." in name:
print("skip proj")
return []

if "conv1.bias" in name or "conv2.bias" in name:
Expand Down
Loading