From 5c2c2b7806072cf53c587d56426260f6e694e710 Mon Sep 17 00:00:00 2001 From: Antonio Cheong Date: Thu, 29 Feb 2024 03:29:59 +0000 Subject: [PATCH 1/5] luarocks version requires revision --- tiktoken.lua-0.2.0.rockspec | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 tiktoken.lua-0.2.0.rockspec diff --git a/tiktoken.lua-0.2.0.rockspec b/tiktoken.lua-0.2.0.rockspec deleted file mode 100644 index 04444da..0000000 --- a/tiktoken.lua-0.2.0.rockspec +++ /dev/null @@ -1,28 +0,0 @@ -package = "tiktoken_core" -version = "0.2.0" - -source = { - url = "git+https://github.com/gptlang/lua-tiktoken", - tag = "0.2.0", -} - -description = { - summary = "An experimental port of OpenAI's Tokenizer to lua", - detailed = [[ - The Lua module written in Rust that provides Tiktoken support for Lua. - ]], - homepage = "https://github.com/gptlang/lua-tiktoken", - license = "MIT" -} - -dependencies = { - "lua >= 5.1", - "luarocks-build-rust-mlua", -} - -build = { - type = "rust-mlua", - modules = { - "tiktoken_core" - }, -} From b39416a12c83770fc77a7eb2d5845314438d0e23 Mon Sep 17 00:00:00 2001 From: Antonio Cheong Date: Thu, 29 Feb 2024 03:30:41 +0000 Subject: [PATCH 2/5] remember to rename to tiktoken_core --- tiktoken_core-0.2.0-1.rockspec | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) create mode 100644 tiktoken_core-0.2.0-1.rockspec diff --git a/tiktoken_core-0.2.0-1.rockspec b/tiktoken_core-0.2.0-1.rockspec new file mode 100644 index 0000000..0ae6836 --- /dev/null +++ b/tiktoken_core-0.2.0-1.rockspec @@ -0,0 +1,28 @@ +package = "tiktoken_core" +version = "0.2.0-1" + +source = { + url = "git+https://github.com/gptlang/lua-tiktoken", + tag = "0.2.0", +} + +description = { + summary = "An experimental port of OpenAI's Tokenizer to lua", + detailed = [[ + The Lua module written in Rust that provides Tiktoken support for Lua. + ]], + homepage = "https://github.com/gptlang/lua-tiktoken", + license = "MIT" +} + +dependencies = { + "lua >= 5.1", + "luarocks-build-rust-mlua", +} + +build = { + type = "rust-mlua", + modules = { + "tiktoken_core" + }, +} From ec70e41ade6e2e69f6e364bcee97394c695d37c4 Mon Sep 17 00:00:00 2001 From: Antonio Cheong Date: Sat, 2 Mar 2024 23:46:16 +0000 Subject: [PATCH 3/5] add cargo build flags for MacOS --- .cargo/config | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .cargo/config diff --git a/.cargo/config b/.cargo/config new file mode 100644 index 0000000..d47f983 --- /dev/null +++ b/.cargo/config @@ -0,0 +1,11 @@ +[target.x86_64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] + +[target.aarch64-apple-darwin] +rustflags = [ + "-C", "link-arg=-undefined", + "-C", "link-arg=dynamic_lookup", +] From 33faaa0a7fd522351c3d3629d20dd2ef5b2c0997 Mon Sep 17 00:00:00 2001 From: Antonio Cheong Date: Sun, 3 Mar 2024 23:22:42 +0000 Subject: [PATCH 4/5] add luajit support --- .gitignore | 1 + Cargo.toml | 1 + 2 files changed, 2 insertions(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..eb5a316 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +target diff --git a/Cargo.toml b/Cargo.toml index 114dd41..855426b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,3 +22,4 @@ lua54 = ["mlua/lua54"] lua53 = ["mlua/lua53"] lua52 = ["mlua/lua52"] lua51 = ["mlua/lua51"] +luajit = ["mlua/luajit"] From 61d831a81a01ba095f8c94e148b721b3aa7b1e45 Mon Sep 17 00:00:00 2001 From: Antonio Cheong Date: Sun, 3 Mar 2024 23:24:06 +0000 Subject: [PATCH 5/5] bump tiktoken_core to 0.2.1 --- ...en_core-0.2.0-1.rockspec => tiktoken_core-0.2.1-1.rockspec | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename tiktoken_core-0.2.0-1.rockspec => tiktoken_core-0.2.1-1.rockspec (93%) diff --git a/tiktoken_core-0.2.0-1.rockspec b/tiktoken_core-0.2.1-1.rockspec similarity index 93% rename from tiktoken_core-0.2.0-1.rockspec rename to tiktoken_core-0.2.1-1.rockspec index 0ae6836..78df676 100644 --- a/tiktoken_core-0.2.0-1.rockspec +++ b/tiktoken_core-0.2.1-1.rockspec @@ -1,9 +1,9 @@ package = "tiktoken_core" -version = "0.2.0-1" +version = "0.2.1-1" source = { url = "git+https://github.com/gptlang/lua-tiktoken", - tag = "0.2.0", + tag = "0.2.1", } description = {