diff --git a/docs/model_support.md b/docs/model_support.md index 4a3b703c3..ba5f5b79b 100644 --- a/docs/model_support.md +++ b/docs/model_support.md @@ -100,13 +100,18 @@ After these steps, the new model should be compatible with most FastChat feature setting the environment variable `PEFT_SHARE_BASE_WEIGHTS=true` in any model worker. + ## API-Based Models -1. Implement an API-based streaming generator in [fastchat/serve/api_provider.py](https://github.com/lm-sys/FastChat/blob/main/fastchat/serve/api_provider.py). You can learn from the OpenAI example. -2. Specify your endpoint info in a JSON configuration file -``` +To support an API-based model, consider learning from the existing OpenAI example. +If the model is compatible with OpenAI APIs, then a configuration file is all that's needed without any additional code. +For custom protocols, implementation of a streaming generator in [fastchat/serve/api_provider.py](https://github.com/lm-sys/FastChat/blob/main/fastchat/serve/api_provider.py) is required, following the provided examples. Currently, FastChat is compatible with OpenAI, Anthropic, Google Vertex AI, Mistral, and Nvidia NGC. + +### Steps to Launch a WebUI with an API Model +1. Specify the endpoint information in a JSON configuration file. For instance, create a file named `api_endpoints.json`: +```json { - "gpt-3.5-turbo-0613": { - "model_name": "gpt-3.5-turbo-0613", + "gpt-3.5-turbo": { + "model_name": "gpt-3.5-turbo", "api_type": "openai", "api_base": "https://api.openai.com/v1", "api_key": "sk-******", @@ -114,13 +119,12 @@ After these steps, the new model should be compatible with most FastChat feature } } ``` - - "api_type" can be one of the following: openai, anthropic, gemini, mistral. For you own API, you can add a new type and implement it. - - "anony_only" means whether to show this model in anonymous mode only. -3. Launch the gradio web server with argument `--register [JSON-file]`. + - "api_type" can be one of the following: openai, anthropic, gemini, or mistral. For custom APIs, add a new type and implement it accordingly. + - "anony_only" indicates whether to display this model in anonymous mode only. +2. Launch the Gradio web server with the argument `--register api_endpoints.json`: ``` -python3 -m fastchat.serve.gradio_web_server --controller "" --share --register [JSON-file] +python3 -m fastchat.serve.gradio_web_server --controller "" --share --register api_endpoints.json ``` -You should be able to chat with your API-based model! -Currently, FastChat supports OpenAI, Anthropic, Google Vertex AI, Mistral, and Nvidia NGC. +Now, you can open a browser and interact with the model. diff --git a/fastchat/model/model_registry.py b/fastchat/model/model_registry.py index 4d7c73e09..1673bfe3f 100644 --- a/fastchat/model/model_registry.py +++ b/fastchat/model/model_registry.py @@ -239,9 +239,9 @@ def get_model_info(name: str) -> ModelInfo: "vicuna-33b", "vicuna-33b-v1.3", "vicuna-13b", - "vicuna-13b-v1.3", + "vicuna-13b-v1.5", "vicuna-7b", - "vicuna-7b-v1.3", + "vicuna-7b-v1.5", ], "Vicuna", "https://lmsys.org/blog/2023-03-30-vicuna/", diff --git a/fastchat/serve/gradio_block_arena_vision.py b/fastchat/serve/gradio_block_arena_vision.py index ebfcdd6fa..b5c60b3c3 100644 --- a/fastchat/serve/gradio_block_arena_vision.py +++ b/fastchat/serve/gradio_block_arena_vision.py @@ -21,6 +21,8 @@ add_text, clear_history, regenerate, + get_ip, + disable_btn, ) from fastchat.utils import ( build_logger, @@ -29,6 +31,13 @@ logger = build_logger("gradio_web_server_multi", "gradio_web_server_multi.log") +def clear_history_example(request: gr.Request): + ip = get_ip(request) + logger.info(f"clear_history_example. ip: {ip}") + state = None + return (state, []) + (disable_btn,) * 5 + + def build_single_vision_language_model_ui(models, add_promotion_links=False): promotion = ( """ @@ -71,7 +80,7 @@ def build_single_vision_language_model_ui(models, add_promotion_links=False): render=False, elem_id="input_box", ) - imagebox = gr.Image(type="pil") + imagebox = gr.Image(type="pil", sources=["upload", "clipboard"]) cur_dir = os.path.dirname(os.path.abspath(__file__)) @@ -101,7 +110,7 @@ def build_single_vision_language_model_ui(models, add_promotion_links=False): label="Max output tokens", ) - gr.Examples( + examples = gr.Examples( examples=[ [ f"{cur_dir}/example_images/city.jpeg", @@ -160,6 +169,7 @@ def build_single_vision_language_model_ui(models, add_promotion_links=False): [state, chatbot] + btn_list, ) clear_btn.click(clear_history, None, [state, chatbot, textbox, imagebox] + btn_list) + examples.dataset.click(clear_history_example, None, [state, chatbot] + btn_list) model_selector.change( clear_history, None, [state, chatbot, textbox, imagebox] + btn_list diff --git a/fastchat/serve/gradio_web_server.py b/fastchat/serve/gradio_web_server.py index dda57e42d..06c2af06b 100644 --- a/fastchat/serve/gradio_web_server.py +++ b/fastchat/serve/gradio_web_server.py @@ -61,7 +61,8 @@ The service is a research preview. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes. -The service collects user dialogue data and reserves the right to distribute it under a Creative Commons Attribution (CC-BY) or a similar license. +Please do not upload any private information. +The service collects user dialogue data, including both text and images, and reserves the right to distribute it under a Creative Commons Attribution (CC-BY) or a similar license. Additionally, Bard is offered on LMSys for research purposes only. To access the Bard product, please visit its [website](http://bard.google.com). ### Acknowledgment @@ -79,8 +80,8 @@ # JSON file format of API-based models: # { -# "gpt-3.5-turbo-0613": { -# "model_name": "gpt-3.5-turbo-0613", +# "gpt-3.5-turbo": { +# "model_name": "gpt-3.5-turbo", # "api_type": "openai", # "api_base": "https://api.openai.com/v1", # "api_key": "sk-******", diff --git a/fastchat/serve/gradio_web_server_multi.py b/fastchat/serve/gradio_web_server_multi.py index fe366c18f..72d8aef75 100644 --- a/fastchat/serve/gradio_web_server_multi.py +++ b/fastchat/serve/gradio_web_server_multi.py @@ -128,9 +128,7 @@ def build_demo(models, vl_models, elo_results_file, leaderboard_table_file): models, add_promotion_links=True ) - with gr.Tab( - "Vision-Language Model Direct Chat", id=3, visible=args.multimodal - ): + with gr.Tab("Vision Direct Chat", id=3, visible=args.multimodal): single_vision_language_model_list = ( build_single_vision_language_model_ui( vl_models, add_promotion_links=True diff --git a/fastchat/utils.py b/fastchat/utils.py index 5f14abce8..70f61202f 100644 --- a/fastchat/utils.py +++ b/fastchat/utils.py @@ -233,7 +233,7 @@ def pretty_print_semaphore(semaphore): url_params = Object.fromEntries(params); console.log("url_params", url_params); - msg = "Users of this website are required to agree to the following terms:\\n\\nThe service is a research preview. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes.\\nThe service collects user dialogue data and reserves the right to distribute it under a Creative Commons Attribution (CC-BY) or a similar license." + msg = "Users of this website are required to agree to the following terms:\\n\\nThe service is a research preview. It only provides limited safety measures and may generate offensive content. It must not be used for any illegal, harmful, violent, racist, or sexual purposes.\\nPlease do not upload any private information.\\nThe service collects user dialogue data, including both text and images, and reserves the right to distribute it under a Creative Commons Attribution (CC-BY) or a similar license." alert(msg); return url_params;