Add llava 34b template (#3034)

Author: merrymercy

fastchat/conversation.py

@@ -172,6 +172,9 @@ def get_prompt(self) -> str:
             ret = "" if system_prompt == "" else system_prompt + self.sep + "\n"
             for role, message in self.messages:
                 if message:
+                    if type(message) is tuple:
+                        message, images = message
+                        message = IMAGE_PLACEHOLDER_STR * len(images) + message
                     ret += role + "\n" + message + self.sep + "\n"
                 else:
                     ret += role + "\n"
@@ -1562,6 +1565,21 @@ def get_conv_template(name: str) -> Conversation:
     )
 )
 
+# Llava-chatml
+# reference: https://github.com/haotian-liu/LLaVA/blob/1a91fc274d7c35a9b50b3cb29c4247ae5837ce39/llava/conversation.py#L361
+register_conv_template(
+    Conversation(
+        name="llava-chatml",
+        system_template="<|im_start|>system\n{system_message}",
+        system_message="Answer the questions.",
+        roles=("<|im_start|>user", "<|im_start|>assistant"),
+        sep_style=SeparatorStyle.CHATML,
+        sep="<|im_end|>",
+        stop_str="<|im_end|>",
+    )
+)
+
+
 if __name__ == "__main__":
     from fastchat.conversation import get_conv_template
 

fastchat/model/model_adapter.py

@@ -2217,6 +2217,10 @@ def match(self, model_path: str):
         return "llava" in model_path.lower()
 
     def get_default_conv_template(self, model_path: str) -> Conversation:
+        model_path = model_path.lower()
+        if "34b" in model_path:
+            return get_conv_template("llava-chatml")
+
         return get_conv_template("vicuna_v1.1")
 
 

fastchat/model/model_registry.py

@@ -645,8 +645,10 @@ def get_model_info(name: str) -> ModelInfo:
 
 register_model_info(
     [
+        "llava-v1.6-34b",
         "llava-v1.6-vicuna-13b",
         "llava-v1.6-vicuna-7b",
+        "llava-v1.6-mistral-7b",
         "llava-v1.5-13b",
         "llava-v1.5-7b",
     ],