bug fix

manishshettym · manishshettym · commit 90e274bea28e · 2023-09-17T21:25:10.000-07:00
diff --git a/llama_cpp/llama.py b/llama_cpp/llama.py
@@ -430,7 +430,7 @@ def tokenize(self, text: bytes, add_bos: bool = True) -> List[int]:
         n_tokens = llama_cpp.llama_tokenize_with_model(
             self.model,
             text,
-            len(text)
+            len(text),
             tokens,
             n_ctx,
             add_bos,
