server.py: hacky code

examples/openai/README.md

@@ -0,0 +1,53 @@
+# examples.openai: OpenAI API-compatible server
+
+A simple Python server that sits above the C++ [../server](examples/server) and offers improved OAI compatibility.
+
+## Usage
+
+```bash
+python -m examples.openai -m some-model.gguf
+
+
+```
+
+## Features
+
+The new examples/openai/server.py:
+
+- Uses llama.cpp C++ server as a backend (spawns it or connects to existing)
+
+- Uses actual jinja2 chat templates read from the models
+
+- Supports grammar-constrained output for both JSON response format and tool calls
+
+- Tool calling “works” w/ all models (even non-specialized ones like Mixtral 7x8B)
+
+    - Optimised support for Functionary & Nous Hermes, easy to extend to other tool-calling fine-tunes
+
+## TODO
+
+- Embedding endpoint w/ distinct server subprocess
+
+- Automatic/manual session caching
+
+    - Spawns the main C++ CLI under the hood
+
+    - Support precaching long prompts from CLI
+
+    - Instant incremental inference in long threads
+
+- Improve examples/agent:
+
+    - Interactive agent CLI that auto-discovers tools from OpenAPI endpoints
+
+    - Script that wraps any Python source as a container-sandboxed OpenAPI endpoint (allowing running ~unsafe code w/ tools)
+
+    - Basic memory / RAG / python interpreter tools
+
+- Follow-ups
+
+    - Remove OAI support from server
+
+    - Remove non-Python json schema to grammar converters
+
+    - Reach out to frameworks to advertise new option. 

examples/openai/__main__.py

@@ -0,0 +1,8 @@
+
+from jsonargparse import CLI
+
+from examples.openai.server import main
+
+if __name__ == "__main__":
+    CLI(main)
+

examples/openai/api.py

@@ -0,0 +1,27 @@
+from typing import Any, Optional
+from pydantic import BaseModel, Json
+
+class Message(BaseModel):
+    role: str
+    content: str
+
+class ToolFunction(BaseModel):
+    name: str
+    description: str
+    parameters: Any
+
+class Tool(BaseModel):
+    type: str
+    function: ToolFunction
+
+class ResponseFormat(BaseModel):
+    type: str
+    json_schema: Optional[Any] = None
+
+class ChatCompletionRequest(BaseModel):
+    model: str
+    tools: Optional[list[Tool]] = None
+    messages: list[Message]
+    response_format: Optional[ResponseFormat] = None
+    temperature: float = 1.0
+    stream: bool = False

examples/openai/chat_format.py

@@ -0,0 +1,59 @@
+from enum import StrEnum
+import jinja2
+
+from examples.openai.gguf_kvs import GGUFKeyValues, Keys
+
+def raise_exception(msg: str):
+    raise Exception(msg)
+
+class ToolStyle(StrEnum):
+    # https://cookbook.openai.com/examples/how_to_call_functions_with_chat_models
+    DEFAULT="Default",
+    # https://github.com/MeetKai/functionary
+    # TODO: look at https://github.com/ggerganov/llama.cpp/pull/5695
+    # https://github.com/MeetKai/functionary/blob/main/functionary/prompt_template/prompt_template_v2.py
+    FUNCTIONARY_V2="Functionary V2",
+    # https://github.com/NousResearch/Hermes-Function-Calling
+    NOUS_RESEARCH_HERMES="Nous-Research-Hermes-Function-Calling",
+
+class ChatFormat: #(BaseModel):
+    def __init__(self, template: str, eos_token: str, bos_token: str):
+        env = jinja2.Environment(loader=jinja2.BaseLoader(), trim_blocks=True, lstrip_blocks=True)
+        self.template = env.from_string(template)
+        self.eos_token = eos_token
+        self.bos_token = bos_token
+
+        self.strict_user_assistant_alternation = "{% if (message['role'] == 'user') != (loop.index0 % 2 == 0) %}{{ raise_exception" in template
+
+        if "<|recipient|>' + tool_call['function']['name']" in template:
+            self.tool_style = ToolStyle.FUNCTIONARY_V2
+        else:
+            self.tool_style = ToolStyle.DEFAULT
+
+
+    def __str__(self):
+        return f"ChatFormat(template={self.template}, eos_token={self.eos_token}, bos_token={self.bos_token})"
+
+
+    @staticmethod
+    def from_gguf(metadata: GGUFKeyValues):
+        return ChatFormat(
+            template = metadata[Keys.Tokenizer.CHAT_TEMPLATE],
+            bos_token = metadata[Keys.Tokenizer.BOS_ID],
+            eos_token = metadata[Keys.Tokenizer.EOS_ID])
+    # @staticmethod
+    # def from_gguf(model: Path):
+    #     reader = GGUFReader(model.as_posix())
+    #     return ChatFormat(
+    #         template = reader.fields[Keys.Tokenizer.CHAT_TEMPLATE].read(),
+    #         bos_token = reader.fields[Keys.Tokenizer.BOS_ID].read(),
+    #         eos_token = reader.fields[Keys.Tokenizer.EOS_ID].read())
+
+    def render(self, messages: list[dict], add_generation_prompt: bool, omit_bos: bool = False):
+        return self.template.render(
+            messages=messages,
+            eos_token=self.eos_token,
+            bos_token='' if omit_bos else self.bos_token,
+            raise_exception=raise_exception,
+            add_generation_prompt=add_generation_prompt,
+        )

examples/openai/gguf_kvs.py

@@ -0,0 +1,20 @@
+from pathlib import Path
+import sys
+
+sys.path.insert(0, str(Path(__file__).parent.parent.parent / "gguf-py"))
+
+from gguf.gguf_reader import GGUFReader
+from gguf.constants import Keys
+
+class GGUFKeyValues:
+    def __init__(self, model: Path):
+        reader = GGUFReader(model.as_posix())
+        self.fields = reader.fields
+    def __getitem__(self, key: str):
+        if '{arch}' in key:
+            key = key.replace('{arch}', self[Keys.General.ARCHITECTURE])
+        return self.fields[key].read()
+    def __contains__(self, key: str):
+        return key in self.fields
+    def keys(self):
+        return self.fields.keys()

examples/openai/llama_cpp_server_api.py

@@ -0,0 +1,28 @@
+from typing import Optional
+from pydantic import BaseModel, Json
+
+class LlamaCppServerCompletionRequest(BaseModel):
+    prompt: str
+    stream: Optional[bool] = None
+    cache_prompt: Optional[bool] = None
+    n_predict: Optional[int] = None
+    top_k: Optional[int] = None
+    top_p: Optional[float] = None
+    min_p: Optional[float] = None
+    tfs_z: Optional[float] = None
+    typical_p: Optional[float] = None
+    temperature: Optional[float] = None
+    dynatemp_range: Optional[float] = None
+    dynatemp_exponent: Optional[float] = None
+    repeat_last_n: Optional[int] = None
+    repeat_penalty: Optional[float] = None
+    frequency_penalty: Optional[float] = None
+    presence_penalty: Optional[float] = None
+    mirostat: Optional[bool] = None
+    mirostat_tau: Optional[float] = None
+    mirostat_eta: Optional[float] = None
+    penalize_nl: Optional[bool] = None
+    n_keep: Optional[int] = None
+    seed: Optional[int] = None
+    grammar: Optional[str] = None
+    json_schema: Optional[Json] = None

examples/openai/requirements.txt

@@ -0,0 +1,7 @@
+fastapi[all]
+gguf
+jinja2
+jsonargparse
+pydantic
+sse-starlette
+uvicorn[all]